├── README.md ├── clone ├── CodeBERT │ ├── construct_exemplar.py │ ├── ewc.py │ ├── model.py │ ├── run.py │ ├── run_cl.py │ ├── run_emr.sh │ ├── run_ewc.sh │ ├── run_finetune.sh │ ├── run_multitask.sh │ ├── run_ours.sh │ └── test.sh └── CodeT5 │ ├── bleu.py │ ├── construct_exemplar.py │ ├── evaluate.py │ ├── ewc.py │ ├── my_lib.py │ ├── run.py │ ├── run_cl.py │ ├── run_emr.sh │ ├── run_ewc.sh │ ├── run_finetune.sh │ ├── run_multitask.sh │ ├── run_ours.sh │ └── test.sh ├── result.xlsx ├── sum ├── CodeBERT │ ├── bleu.py │ ├── construct_exemplar.py │ ├── evall │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-36.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ └── __init__.cpython-39.pyc │ │ ├── bleu │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ │ ├── __init__.cpython-36.pyc │ │ │ │ ├── __init__.cpython-37.pyc │ │ │ │ ├── __init__.cpython-39.pyc │ │ │ │ ├── bleu.cpython-36.pyc │ │ │ │ ├── bleu.cpython-37.pyc │ │ │ │ ├── bleu.cpython-39.pyc │ │ │ │ ├── bleu_scorer.cpython-36.pyc │ │ │ │ ├── bleu_scorer.cpython-37.pyc │ │ │ │ ├── bleu_scorer.cpython-39.pyc │ │ │ │ ├── google_bleu.cpython-36.pyc │ │ │ │ ├── google_bleu.cpython-37.pyc │ │ │ │ ├── google_bleu.cpython-39.pyc │ │ │ │ ├── nltk_bleu.cpython-36.pyc │ │ │ │ ├── nltk_bleu.cpython-37.pyc │ │ │ │ └── nltk_bleu.cpython-39.pyc │ │ │ ├── bleu.py │ │ │ ├── bleu_scorer.py │ │ │ ├── google_bleu.py │ │ │ └── nltk_bleu.py │ │ ├── meteor │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ │ ├── __init__.cpython-36.pyc │ │ │ │ ├── __init__.cpython-37.pyc │ │ │ │ ├── __init__.cpython-39.pyc │ │ │ │ ├── meteor.cpython-36.pyc │ │ │ │ ├── meteor.cpython-37.pyc │ │ │ │ └── meteor.cpython-39.pyc │ │ │ ├── data │ │ │ │ └── paraphrase-en.gz │ │ │ ├── meteor-1.5.jar │ │ │ └── meteor.py │ │ └── rouge │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-36.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── __init__.cpython-39.pyc │ │ │ ├── rouge.cpython-36.pyc │ │ │ ├── rouge.cpython-37.pyc │ │ │ └── rouge.cpython-39.pyc │ │ │ └── rouge.py │ ├── evaluate.py │ ├── ewc.py │ ├── model.py │ ├── run.py │ ├── run_cl.py │ ├── run_emr.sh │ ├── run_ewc.sh │ ├── run_finetune.sh │ ├── run_multitask.sh │ ├── run_ours.py │ └── run_ours.sh └── CodeT5 │ ├── bleu.py │ ├── construct_exemplar.py │ ├── evall │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ └── __init__.cpython-39.pyc │ ├── bleu │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-36.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── __init__.cpython-39.pyc │ │ │ ├── bleu.cpython-36.pyc │ │ │ ├── bleu.cpython-37.pyc │ │ │ ├── bleu.cpython-39.pyc │ │ │ ├── bleu_scorer.cpython-36.pyc │ │ │ ├── bleu_scorer.cpython-37.pyc │ │ │ ├── bleu_scorer.cpython-39.pyc │ │ │ ├── google_bleu.cpython-36.pyc │ │ │ ├── google_bleu.cpython-37.pyc │ │ │ ├── google_bleu.cpython-39.pyc │ │ │ ├── nltk_bleu.cpython-36.pyc │ │ │ ├── nltk_bleu.cpython-37.pyc │ │ │ └── nltk_bleu.cpython-39.pyc │ │ ├── bleu.py │ │ ├── bleu_scorer.py │ │ ├── google_bleu.py │ │ └── nltk_bleu.py │ ├── meteor │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-36.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── __init__.cpython-39.pyc │ │ │ ├── meteor.cpython-36.pyc │ │ │ ├── meteor.cpython-37.pyc │ │ │ └── meteor.cpython-39.pyc │ │ ├── data │ │ │ └── paraphrase-en.gz │ │ ├── meteor-1.5.jar │ │ └── meteor.py │ └── rouge │ │ ├── __init__.py │ │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── __init__.cpython-39.pyc │ │ ├── rouge.cpython-36.pyc │ │ ├── rouge.cpython-37.pyc │ │ └── rouge.cpython-39.pyc │ │ └── rouge.py │ ├── evaluate.py │ ├── ewc.py │ ├── my_lib.py │ ├── run.py │ ├── run_cl.py │ ├── run_emr.sh │ ├── run_ewc.sh │ ├── run_finetune.sh │ ├── run_multitask.sh │ ├── run_ours.py │ └── run_ours.sh └── svd ├── CodeBERT ├── construct_exemplar.py ├── ewc.py ├── model.py ├── run.py ├── run_cl.py ├── run_emr.sh ├── run_ewc.sh ├── run_finetune.sh ├── run_multitask.sh ├── run_ours.sh └── test.sh └── CodeT5 ├── bleu.py ├── construct_exemplar.py ├── evaluate.py ├── ewc.py ├── my_lib.py ├── run.py ├── run_cl.py ├── run_emr.sh ├── run_ewc.sh ├── run_finetune.sh ├── run_multitask.sh ├── run_ours.sh └── test.sh /README.md: -------------------------------------------------------------------------------- 1 | # REPEAT 2 | This is an implemention for our ICSE 2023 paper based on pytorch 3 | 4 | Keeping Pace with Ever-Increasing Data: Towards Continual Learning of Code Intelligence Models 5 | 6 | by Shuzheng Gao, Hongyu Zhang, Cuiyun Gao and Chaozheng Wang 7 | 8 | 9 | # Introduction 10 | REPEAT is a general training method for continual learning of code intelligence models. 11 | 12 | 13 | 14 | # Data 15 | Our processed datasets can be downloaded in [Zenodo](https://zenodo.org/record/7827136#.ZDjMEnZByUl). 16 | 17 | 18 | # Run the code 19 | Reproduce the results of our method and each baseline. 20 | 21 | 22 | ```markdown 23 | |-- REPEAT 24 | |-- sum 25 | | |-- CodeBERT 26 | | | |-- run_fineune.sh 27 | | | |-- run_emr.sh 28 | | | |-- run_ewc.sh 29 | | | |-- run_multitask.sh 30 | | | |-- run_ours.sh 31 | | | |-- ... 32 | | |-- CodeT5 33 | | | |-- run_fineune.sh 34 | | | |-- run_emr.sh 35 | | | |-- run_ewc.sh 36 | | | |-- run_multitask.sh 37 | | | |-- run_ours.sh 38 | | | |-- ... 39 | |-- svd 40 | | |-- CodeBERT 41 | | | |-- ... 42 | | |-- CodeT5 43 | | |-- ... 44 | |-- clone 45 | | |-- CodeBERT 46 | | | |-- ... 47 | | |-- CodeT5 48 | | |-- ... 49 | ``` 50 | 51 | For example, if you want to reproduce the results of code summarization on CodeBERT, you can first move to the direcotory 52 | 53 | ```bash 54 | cd sum/CodeBERT 55 | ``` 56 | 57 | Please first modify the data and model directory. You can also change the model's hyperparameter in each bash file. 58 | 59 | 60 | Normal Finetune: 61 | 62 | ```bash 63 | bash run_finetune.sh 64 | ``` 65 | 66 | 67 | EMR method: 68 | 69 | ```bash 70 | bash run_emr.sh 71 | ``` 72 | 73 | 74 | EWC method: 75 | 76 | ```bash 77 | bash run_ewc.sh 78 | ``` 79 | 80 | 81 | Upper bound: 82 | 83 | ```bash 84 | bash run_multitask.sh 85 | ``` 86 | 87 | 88 | Upper bound: 89 | 90 | ```bash 91 | bash run_ous.sh 92 | ``` 93 | 94 | 95 | ## Citation 96 | 97 | If you use our code, please kindly cite: 98 | 99 | ``` 100 | @inproceedings{Gao2023repeat, 101 | title={Keeping Pace with Ever-Increasing Data: Towards Continual Learning of Code Intelligence Models}, 102 | author={Shuzheng Gao, Hongyu Zhang, Cuiyun Gao, and Chaozheng Wang}, 103 | booktitle={ICSE}, 104 | year={2023}, 105 | publisher={IEEE} 106 | } 107 | ``` 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | -------------------------------------------------------------------------------- /clone/CodeBERT/ewc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | from torch.autograd import Variable 4 | from torch.nn import CrossEntropyLoss, MSELoss 5 | import copy 6 | #from model.masked_cross_entropy import * 7 | def variable(t: torch.Tensor, use_cuda=True, **kwargs): 8 | if torch.cuda.is_available() and use_cuda: 9 | t = t.cuda() 10 | return Variable(t, **kwargs) 11 | 12 | class EWC(object): 13 | def __init__(self, model, dataset, device, length): 14 | 15 | self.model = model 16 | # the data we use to compute fisher information of ewc (old_exemplars) 17 | self.dataset = dataset 18 | self.device = device 19 | self.length = length 20 | 21 | self.params = {n: p for n, p in self.model.named_parameters() if p.requires_grad} 22 | self._means = {} # previous parameters 23 | self._precision_matrices = self._diag_fisher() # approximated diagnal fisher information matrix 24 | 25 | for n, p in copy.deepcopy(self.params).items(): 26 | self._means[n] = variable(p.data) 27 | 28 | def _diag_fisher(self): 29 | 30 | self.model.train() 31 | precision_matrices = {} 32 | for n, p in copy.deepcopy(self.params).items(): 33 | p.data.zero_() 34 | precision_matrices[n] = variable(p.data) 35 | 36 | for step,batch in enumerate(self.dataset): 37 | self.model.zero_grad() 38 | 39 | inputs = batch[0].to(self.device) 40 | labels=batch[1].to(self.device) 41 | self.model.train() 42 | loss,logits = self.model(inputs,labels) 43 | 44 | loss.backward() 45 | 46 | for n, p in self.model.named_parameters(): 47 | if p.grad is None: 48 | continue 49 | precision_matrices[n].data += p.grad.data ** 2 / self.length 50 | 51 | precision_matrices = {n: p for n, p in precision_matrices.items()} 52 | self.model.zero_grad() 53 | return precision_matrices 54 | 55 | def penalty(self, model: nn.Module): 56 | loss = 0 57 | for n, p in model.named_parameters(): 58 | _loss = self._precision_matrices[n] * (p - self._means[n]) ** 2 59 | loss += _loss.sum() 60 | 61 | return loss 62 | -------------------------------------------------------------------------------- /clone/CodeBERT/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch 4 | from torch.autograd import Variable 5 | import copy 6 | from torch.nn import CrossEntropyLoss, MSELoss 7 | 8 | 9 | 10 | class Model(nn.Module): 11 | def __init__(self, encoder,config,tokenizer,args): 12 | super(Model, self).__init__() 13 | self.encoder = encoder 14 | self.config=config 15 | self.tokenizer=tokenizer 16 | self.args=args 17 | self.weight=1 18 | 19 | 20 | def forward(self, input_ids=None,labels=None,relay=None): 21 | outputs=self.encoder(input_ids,attention_mask=input_ids.ne(1))[0] 22 | logits=outputs 23 | prob=torch.sigmoid(logits) 24 | if relay is True: 25 | labels=labels.float() 26 | loss=torch.log(prob[:,0]+1e-10)*labels+torch.log((1-prob)[:,0]+1e-10)*(1-labels) 27 | return -loss,prob 28 | if labels is not None: 29 | labels=labels.float() 30 | loss=torch.log(prob[:,0]+1e-10)*labels+torch.log((1-prob)[:,0]+1e-10)*(1-labels) 31 | loss=-loss.mean() 32 | return loss,prob 33 | else: 34 | return prob 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /clone/CodeBERT/run_emr.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=0 2 | name=emr 3 | train_replay_size=2000 4 | train_examplar_path=./saved_models/$name/train_examplar.jsonl 5 | dev_examplar_path=./saved_models/$name/dev_examplar.jsonl 6 | pretrained_model=/microsoft/codebert-base 7 | 8 | #train 9 | train_data_file=/POJ_clone/binary/train_0.jsonl 10 | eval_data_file=/POJ_clone/binary/dev_0.jsonl 11 | load_model_path=./saved_models/finetune/task_0/checkpoint-best-acc/model.bin 12 | output_dir=./saved_models/$name/task_0 13 | python run_cl.py \ 14 | --output_dir=$output_dir \ 15 | --model_type=roberta \ 16 | --tokenizer_name=$pretrained_model \ 17 | --model_name_or_path=$pretrained_model \ 18 | --load_model_path $load_model_path\ 19 | --do_train \ 20 | --train_data_file=$train_data_file \ 21 | --eval_data_file=$eval_data_file \ 22 | --mode $name \ 23 | --task_id 0 \ 24 | --train_replay_size=$train_replay_size \ 25 | --train_examplar_path=$train_examplar_path \ 26 | --eval_examplar_path=$dev_examplar_path \ 27 | --epoch 5 \ 28 | --block_size 400 \ 29 | --train_batch_size 16 \ 30 | --eval_batch_size 64 \ 31 | --learning_rate 2e-5 \ 32 | --max_grad_norm 1.0 \ 33 | --evaluate_during_training \ 34 | --seed 123456 2>&1 | tee train.log 35 | train_data_file=/POJ_clone/binary/train_1.jsonl 36 | eval_data_file=/POJ_clone/binary/dev_1.jsonl 37 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 38 | output_dir=./saved_models/$name/task_1 39 | python run_cl.py \ 40 | --output_dir=$output_dir \ 41 | --model_type=roberta \ 42 | --tokenizer_name=$pretrained_model \ 43 | --model_name_or_path=$pretrained_model \ 44 | --load_model_path $load_model_path\ 45 | --do_train \ 46 | --train_data_file=$train_data_file \ 47 | --eval_data_file=$eval_data_file \ 48 | --mode $name \ 49 | --task_id 1 \ 50 | --train_replay_size=$train_replay_size \ 51 | --train_examplar_path=$train_examplar_path \ 52 | --eval_examplar_path=$dev_examplar_path \ 53 | --epoch 5 \ 54 | --block_size 400 \ 55 | --train_batch_size 16 \ 56 | --eval_batch_size 64 \ 57 | --learning_rate 2e-5 \ 58 | --max_grad_norm 1.0 \ 59 | --evaluate_during_training \ 60 | --seed 123456 2>&1 | tee train.log 61 | train_data_file=/POJ_clone/binary/train_2.jsonl 62 | eval_data_file=/POJ_clone/binary/dev_2.jsonl 63 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 64 | output_dir=./saved_models/$name/task_2 65 | python run_cl.py \ 66 | --output_dir=$output_dir \ 67 | --model_type=roberta \ 68 | --tokenizer_name=$pretrained_model \ 69 | --model_name_or_path=$pretrained_model \ 70 | --load_model_path $load_model_path\ 71 | --do_train \ 72 | --train_data_file=$train_data_file \ 73 | --eval_data_file=$eval_data_file \ 74 | --mode $name \ 75 | --task_id 2 \ 76 | --train_replay_size=$train_replay_size \ 77 | --train_examplar_path=$train_examplar_path \ 78 | --eval_examplar_path=$dev_examplar_path \ 79 | --epoch 5 \ 80 | --block_size 400 \ 81 | --train_batch_size 16 \ 82 | --eval_batch_size 64 \ 83 | --learning_rate 2e-5 \ 84 | --max_grad_norm 1.0 \ 85 | --evaluate_during_training \ 86 | --seed 123456 2>&1 | tee train.log 87 | train_data_file=/POJ_clone/binary/train_3.jsonl 88 | eval_data_file=/POJ_clone/binary/dev_3.jsonl 89 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 90 | output_dir=./saved_models/$name/task_3 91 | python run_cl.py \ 92 | --output_dir=$output_dir \ 93 | --model_type=roberta \ 94 | --tokenizer_name=$pretrained_model \ 95 | --model_name_or_path=$pretrained_model \ 96 | --load_model_path $load_model_path\ 97 | --do_train \ 98 | --train_data_file=$train_data_file \ 99 | --eval_data_file=$eval_data_file \ 100 | --mode $name \ 101 | --task_id 3 \ 102 | --train_replay_size=$train_replay_size \ 103 | --train_examplar_path=$train_examplar_path \ 104 | --eval_examplar_path=$dev_examplar_path \ 105 | --epoch 5 \ 106 | --block_size 400 \ 107 | --train_batch_size 16 \ 108 | --eval_batch_size 64 \ 109 | --learning_rate 2e-5 \ 110 | --max_grad_norm 1.0 \ 111 | --evaluate_during_training \ 112 | --seed 123456 2>&1 | tee train.log 113 | train_data_file=/POJ_clone/binary/train_4.jsonl 114 | eval_data_file=/POJ_clone/binary/dev_4.jsonl 115 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 116 | output_dir=./saved_models/$name/task_4 117 | python run_cl.py \ 118 | --output_dir=$output_dir \ 119 | --model_type=roberta \ 120 | --tokenizer_name=$pretrained_model \ 121 | --model_name_or_path=$pretrained_model \ 122 | --load_model_path $load_model_path\ 123 | --do_train \ 124 | --train_data_file=$train_data_file \ 125 | --eval_data_file=$eval_data_file \ 126 | --mode $name \ 127 | --task_id 4 \ 128 | --train_replay_size=$train_replay_size \ 129 | --train_examplar_path=$train_examplar_path \ 130 | --eval_examplar_path=$dev_examplar_path \ 131 | --epoch 5 \ 132 | --block_size 400 \ 133 | --train_batch_size 16 \ 134 | --eval_batch_size 64 \ 135 | --learning_rate 2e-5 \ 136 | --max_grad_norm 1.0 \ 137 | --evaluate_during_training \ 138 | --seed 123456 2>&1 | tee train.log 139 | 140 | 141 | #generate 142 | cp -r ./saved_models/finetune/task_0 ./saved_models/$name 143 | 144 | data_dir=/POJ_clone/binary 145 | test_data_file=$data_dir/test_0.jsonl,$data_dir/test_1.jsonl,$data_dir/test_2.jsonl,$data_dir/test_3.jsonl,$data_dir/test_4.jsonl 146 | output_dir=./saved_models/$name/task_0,./saved_models/$name/task_1,./saved_models/$name/task_2,./saved_models/$name/task_3,./saved_models/$name/task_4 147 | python run.py \ 148 | --output_dir=$output_dir \ 149 | --model_type=roberta \ 150 | --tokenizer_name=$pretrained_model \ 151 | --model_name_or_path=$pretrained_model \ 152 | --do_test \ 153 | --train_data_file=$test_data_file \ 154 | --test_data_file=$test_data_file \ 155 | --epoch 5 \ 156 | --block_size 400 \ 157 | --train_batch_size 32 \ 158 | --eval_batch_size 64 \ 159 | --learning_rate 2e-5 \ 160 | --max_grad_norm 1.0 \ 161 | --evaluate_during_training \ 162 | --seed 123456 2>&1 | tee test.log 163 | -------------------------------------------------------------------------------- /clone/CodeBERT/run_ewc.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=0 2 | name=ewc 3 | train_replay_size=2000 4 | weight=2000 5 | train_examplar_path=./saved_models/$name/train_examplar.jsonl 6 | dev_examplar_path=./saved_models/$name/dev_examplar.jsonl 7 | pretrained_model=/microsoft/codebert-base 8 | 9 | #train 10 | train_data_file=/POJ_clone/binary/train_0.jsonl 11 | eval_data_file=/POJ_clone/binary/dev_0.jsonl 12 | load_model_path=./saved_models/finetune/task_0/checkpoint-best-acc/model.bin 13 | output_dir=./saved_models/$name/task_0 14 | python run_cl.py \ 15 | --output_dir=$output_dir \ 16 | --model_type=roberta \ 17 | --tokenizer_name=$pretrained_model \ 18 | --model_name_or_path=$pretrained_model \ 19 | --load_model_path $load_model_path\ 20 | --do_train \ 21 | --train_data_file=$train_data_file \ 22 | --eval_data_file=$eval_data_file \ 23 | --mode $name \ 24 | --task_id 0 \ 25 | --ewc_weight $weight \ 26 | --train_replay_size=$train_replay_size \ 27 | --train_examplar_path=$train_examplar_path \ 28 | --eval_examplar_path=$dev_examplar_path \ 29 | --epoch 5 \ 30 | --block_size 400 \ 31 | --train_batch_size 16 \ 32 | --eval_batch_size 64 \ 33 | --learning_rate 2e-5 \ 34 | --max_grad_norm 1.0 \ 35 | --evaluate_during_training \ 36 | --seed 123456 2>&1 | tee train.log 37 | train_data_file=/POJ_clone/binary/train_1.jsonl 38 | eval_data_file=/POJ_clone/binary/dev_1.jsonl 39 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 40 | output_dir=./saved_models/$name/task_1 41 | python run_cl.py \ 42 | --output_dir=$output_dir \ 43 | --model_type=roberta \ 44 | --tokenizer_name=$pretrained_model \ 45 | --model_name_or_path=$pretrained_model \ 46 | --load_model_path $load_model_path\ 47 | --do_train \ 48 | --train_data_file=$train_data_file \ 49 | --eval_data_file=$eval_data_file \ 50 | --mode $name \ 51 | --task_id 1 \ 52 | --ewc_weight $weight \ 53 | --train_replay_size=$train_replay_size \ 54 | --train_examplar_path=$train_examplar_path \ 55 | --eval_examplar_path=$dev_examplar_path \ 56 | --epoch 5 \ 57 | --block_size 400 \ 58 | --train_batch_size 16 \ 59 | --eval_batch_size 64 \ 60 | --learning_rate 2e-5 \ 61 | --max_grad_norm 1.0 \ 62 | --evaluate_during_training \ 63 | --seed 123456 2>&1 | tee train.log 64 | train_data_file=/POJ_clone/binary/train_2.jsonl 65 | eval_data_file=/POJ_clone/binary/dev_2.jsonl 66 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 67 | output_dir=./saved_models/$name/task_2 68 | python run_cl.py \ 69 | --output_dir=$output_dir \ 70 | --model_type=roberta \ 71 | --tokenizer_name=$pretrained_model \ 72 | --model_name_or_path=$pretrained_model \ 73 | --load_model_path $load_model_path\ 74 | --do_train \ 75 | --train_data_file=$train_data_file \ 76 | --eval_data_file=$eval_data_file \ 77 | --mode $name \ 78 | --task_id 2 \ 79 | --ewc_weight $weight \ 80 | --train_replay_size=$train_replay_size \ 81 | --train_examplar_path=$train_examplar_path \ 82 | --eval_examplar_path=$dev_examplar_path \ 83 | --epoch 5 \ 84 | --block_size 400 \ 85 | --train_batch_size 16 \ 86 | --eval_batch_size 64 \ 87 | --learning_rate 2e-5 \ 88 | --max_grad_norm 1.0 \ 89 | --evaluate_during_training \ 90 | --seed 123456 2>&1 | tee train.log 91 | train_data_file=/POJ_clone/binary/train_3.jsonl 92 | eval_data_file=/POJ_clone/binary/dev_3.jsonl 93 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 94 | output_dir=./saved_models/$name/task_3 95 | python run_cl.py \ 96 | --output_dir=$output_dir \ 97 | --model_type=roberta \ 98 | --tokenizer_name=$pretrained_model \ 99 | --model_name_or_path=$pretrained_model \ 100 | --load_model_path $load_model_path\ 101 | --do_train \ 102 | --train_data_file=$train_data_file \ 103 | --eval_data_file=$eval_data_file \ 104 | --mode $name \ 105 | --task_id 3 \ 106 | --ewc_weight $weight \ 107 | --train_replay_size=$train_replay_size \ 108 | --train_examplar_path=$train_examplar_path \ 109 | --eval_examplar_path=$dev_examplar_path \ 110 | --epoch 5 \ 111 | --block_size 400 \ 112 | --train_batch_size 16 \ 113 | --eval_batch_size 64 \ 114 | --learning_rate 2e-5 \ 115 | --max_grad_norm 1.0 \ 116 | --evaluate_during_training \ 117 | --seed 123456 2>&1 | tee train.log 118 | train_data_file=/POJ_clone/binary/train_4.jsonl 119 | eval_data_file=/POJ_clone/binary/dev_4.jsonl 120 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 121 | output_dir=./saved_models/$name/task_4 122 | python run_cl.py \ 123 | --output_dir=$output_dir \ 124 | --model_type=roberta \ 125 | --tokenizer_name=$pretrained_model \ 126 | --model_name_or_path=$pretrained_model \ 127 | --load_model_path $load_model_path\ 128 | --do_train \ 129 | --train_data_file=$train_data_file \ 130 | --eval_data_file=$eval_data_file \ 131 | --mode $name \ 132 | --task_id 4 \ 133 | --ewc_weight $weight \ 134 | --train_replay_size=$train_replay_size \ 135 | --train_examplar_path=$train_examplar_path \ 136 | --eval_examplar_path=$dev_examplar_path \ 137 | --epoch 5 \ 138 | --block_size 400 \ 139 | --train_batch_size 16 \ 140 | --eval_batch_size 64 \ 141 | --learning_rate 2e-5 \ 142 | --max_grad_norm 1.0 \ 143 | --evaluate_during_training \ 144 | --seed 123456 2>&1 | tee train.log 145 | 146 | 147 | #generate 148 | cp -r ./saved_models/finetune/task_0 ./saved_models/$name 149 | 150 | data_dir=/POJ_clone/binary 151 | test_data_file=$data_dir/test_0.jsonl,$data_dir/test_1.jsonl,$data_dir/test_2.jsonl,$data_dir/test_3.jsonl,$data_dir/test_4.jsonl 152 | output_dir=./saved_models/$name/task_0,./saved_models/$name/task_1,./saved_models/$name/task_2,./saved_models/$name/task_3,./saved_models/$name/task_4 153 | python run.py \ 154 | --output_dir=$output_dir \ 155 | --model_type=roberta \ 156 | --tokenizer_name=$pretrained_model \ 157 | --model_name_or_path=$pretrained_model \ 158 | --do_test \ 159 | --train_data_file=$test_data_file \ 160 | --test_data_file=$test_data_file \ 161 | --epoch 5 \ 162 | --block_size 400 \ 163 | --train_batch_size 32 \ 164 | --eval_batch_size 64 \ 165 | --learning_rate 2e-5 \ 166 | --max_grad_norm 1.0 \ 167 | --evaluate_during_training \ 168 | --seed 123456 2>&1 | tee test.log -------------------------------------------------------------------------------- /clone/CodeBERT/run_finetune.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=0 2 | name=finetune 3 | pretrained_model=/microsoft/codebert-base 4 | 5 | #train 6 | train_data_file=/POJ_clone/binary/train_0.jsonl 7 | eval_data_file=/POJ_clone/binary/dev_0.jsonl 8 | output_dir=./saved_models/$name/task_0 9 | python run.py \ 10 | --output_dir=$output_dir \ 11 | --model_type=roberta \ 12 | --tokenizer_name=$pretrained_model \ 13 | --model_name_or_path=$pretrained_model \ 14 | --do_train \ 15 | --train_data_file=$train_data_file \ 16 | --eval_data_file=$eval_data_file \ 17 | --epoch 5 \ 18 | --block_size 400 \ 19 | --train_batch_size 16 \ 20 | --eval_batch_size 64 \ 21 | --learning_rate 2e-5 \ 22 | --max_grad_norm 1.0 \ 23 | --evaluate_during_training \ 24 | --seed 123456 2>&1 | tee train.log 25 | train_data_file=/POJ_clone/binary/train_1.jsonl 26 | eval_data_file=/POJ_clone/binary/dev_1.jsonl 27 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 28 | output_dir=./saved_models/$name/task_1 29 | python run.py \ 30 | --output_dir=$output_dir \ 31 | --model_type=roberta \ 32 | --tokenizer_name=$pretrained_model \ 33 | --model_name_or_path=$pretrained_model \ 34 | --do_train \ 35 | --train_data_file=$train_data_file \ 36 | --eval_data_file=$eval_data_file \ 37 | --load_model_path=$load_model_path \ 38 | --epoch 5 \ 39 | --block_size 400 \ 40 | --train_batch_size 16 \ 41 | --eval_batch_size 64 \ 42 | --learning_rate 2e-5 \ 43 | --max_grad_norm 1.0 \ 44 | --evaluate_during_training \ 45 | --seed 123456 2>&1 | tee train.log 46 | train_data_file=/POJ_clone/binary/train_2.jsonl 47 | eval_data_file=/POJ_clone/binary/dev_2.jsonl 48 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 49 | output_dir=./saved_models/$name/task_2 50 | python run.py \ 51 | --output_dir=$output_dir \ 52 | --model_type=roberta \ 53 | --tokenizer_name=$pretrained_model \ 54 | --model_name_or_path=$pretrained_model \ 55 | --do_train \ 56 | --train_data_file=$train_data_file \ 57 | --eval_data_file=$eval_data_file \ 58 | --load_model_path=$load_model_path \ 59 | --epoch 5 \ 60 | --block_size 400 \ 61 | --train_batch_size 16 \ 62 | --eval_batch_size 64 \ 63 | --learning_rate 2e-5 \ 64 | --max_grad_norm 1.0 \ 65 | --evaluate_during_training \ 66 | --seed 123456 2>&1 | tee train.log 67 | train_data_file=/POJ_clone/binary/train_3.jsonl 68 | eval_data_file=/POJ_clone/binary/dev_3.jsonl 69 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 70 | output_dir=./saved_models/$name/task_3 71 | python run.py \ 72 | --output_dir=$output_dir \ 73 | --model_type=roberta \ 74 | --tokenizer_name=$pretrained_model \ 75 | --model_name_or_path=$pretrained_model \ 76 | --do_train \ 77 | --train_data_file=$train_data_file \ 78 | --eval_data_file=$eval_data_file \ 79 | --load_model_path=$load_model_path \ 80 | --epoch 5 \ 81 | --block_size 400 \ 82 | --train_batch_size 16 \ 83 | --eval_batch_size 64 \ 84 | --learning_rate 2e-5 \ 85 | --max_grad_norm 1.0 \ 86 | --evaluate_during_training \ 87 | --seed 123456 2>&1 | tee train.log 88 | train_data_file=/POJ_clone/binary/train_4.jsonl 89 | eval_data_file=/POJ_clone/binary/dev_4.jsonl 90 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 91 | output_dir=./saved_models/$name/task_4 92 | python run.py \ 93 | --output_dir=$output_dir \ 94 | --model_type=roberta \ 95 | --tokenizer_name=$pretrained_model \ 96 | --model_name_or_path=$pretrained_model \ 97 | --do_train \ 98 | --train_data_file=$train_data_file \ 99 | --eval_data_file=$eval_data_file \ 100 | --load_model_path=$load_model_path \ 101 | --epoch 5 \ 102 | --block_size 400 \ 103 | --train_batch_size 16 \ 104 | --eval_batch_size 64 \ 105 | --learning_rate 2e-5 \ 106 | --max_grad_norm 1.0 \ 107 | --evaluate_during_training \ 108 | --seed 123456 2>&1 | tee train.log 109 | 110 | #generate 111 | data_dir=/POJ_clone/binary 112 | test_data_file=$data_dir/test_0.jsonl,$data_dir/test_1.jsonl,$data_dir/test_2.jsonl,$data_dir/test_3.jsonl,$data_dir/test_4.jsonl 113 | output_dir=./saved_models/$name/task_0,./saved_models/$name/task_1,./saved_models/$name/task_2,./saved_models/$name/task_3,./saved_models/$name/task_4 114 | python run.py \ 115 | --output_dir=$output_dir \ 116 | --model_type=roberta \ 117 | --tokenizer_name=$pretrained_model \ 118 | --model_name_or_path=$pretrained_model \ 119 | --do_test \ 120 | --train_data_file=$test_data_file \ 121 | --test_data_file=$test_data_file \ 122 | --epoch 5 \ 123 | --block_size 400 \ 124 | --train_batch_size 32 \ 125 | --eval_batch_size 64 \ 126 | --learning_rate 2e-5 \ 127 | --max_grad_norm 1.0 \ 128 | --evaluate_during_training \ 129 | --seed 123456 2>&1 | tee test.log 130 | -------------------------------------------------------------------------------- /clone/CodeBERT/run_multitask.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=0 2 | name=multi_task 3 | pretrained_model=/microsoft/codebert-base 4 | 5 | #train 6 | train_data_file=/POJ_clone/binary/train.jsonl 7 | eval_data_file=/POJ_clone/binary/dev.jsonl 8 | output_dir=./saved_models/$name/task_4 9 | python run.py \ 10 | --output_dir=$output_dir \ 11 | --model_type=roberta \ 12 | --tokenizer_name=$pretrained_model \ 13 | --model_name_or_path=$pretrained_model \ 14 | --do_train \ 15 | --train_data_file=$train_data_file \ 16 | --eval_data_file=$eval_data_file \ 17 | --epoch 5 \ 18 | --block_size 400 \ 19 | --train_batch_size 16 \ 20 | --eval_batch_size 64 \ 21 | --learning_rate 2e-5 \ 22 | --max_grad_norm 1.0 \ 23 | --evaluate_during_training \ 24 | --seed 123456 2>&1 | tee train.log 25 | 26 | 27 | train_data_file=/POJ_clone/binary/train_0123.jsonl 28 | eval_data_file=/POJ_clone/binary/dev_0123.jsonl 29 | output_dir=./saved_models/$name/task_3 30 | python run.py \ 31 | --output_dir=$output_dir \ 32 | --model_type=roberta \ 33 | --tokenizer_name=$pretrained_model \ 34 | --model_name_or_path=$pretrained_model \ 35 | --do_train \ 36 | --train_data_file=$train_data_file \ 37 | --eval_data_file=$eval_data_file \ 38 | --epoch 5 \ 39 | --block_size 400 \ 40 | --train_batch_size 16 \ 41 | --eval_batch_size 64 \ 42 | --learning_rate 2e-5 \ 43 | --max_grad_norm 1.0 \ 44 | --evaluate_during_training \ 45 | --seed 123456 2>&1 | tee train.log 46 | 47 | 48 | train_data_file=/POJ_clone/binary/train_012.jsonl 49 | eval_data_file=/POJ_clone/binary/dev_012.jsonl 50 | output_dir=./saved_models/$name/task_2 51 | python run.py \ 52 | --output_dir=$output_dir \ 53 | --model_type=roberta \ 54 | --tokenizer_name=$pretrained_model \ 55 | --model_name_or_path=$pretrained_model \ 56 | --do_train \ 57 | --train_data_file=$train_data_file \ 58 | --eval_data_file=$eval_data_file \ 59 | --epoch 5 \ 60 | --block_size 400 \ 61 | --train_batch_size 16 \ 62 | --eval_batch_size 64 \ 63 | --learning_rate 2e-5 \ 64 | --max_grad_norm 1.0 \ 65 | --evaluate_during_training \ 66 | --seed 123456 2>&1 | tee train.log 67 | 68 | 69 | train_data_file=/POJ_clone/binary/train_01.jsonl 70 | eval_data_file=/POJ_clone/binary/dev_01.jsonl 71 | output_dir=./saved_models/$name/task_1 72 | python run.py \ 73 | --output_dir=$output_dir \ 74 | --model_type=roberta \ 75 | --tokenizer_name=$pretrained_model \ 76 | --model_name_or_path=$pretrained_model \ 77 | --do_train \ 78 | --train_data_file=$train_data_file \ 79 | --eval_data_file=$eval_data_file \ 80 | --epoch 5 \ 81 | --block_size 400 \ 82 | --train_batch_size 16 \ 83 | --eval_batch_size 64 \ 84 | --learning_rate 2e-5 \ 85 | --max_grad_norm 1.0 \ 86 | --evaluate_during_training \ 87 | --seed 123456 2>&1 | tee train.log 88 | -------------------------------------------------------------------------------- /clone/CodeBERT/run_ours.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=1 2 | name=emr_ours_adaewc 3 | train_replay_size=2000 #java:1448, python:2245, go:1462. php:2146, javascript:822, ruby:221, clone:2000, svd:1509 4 | weight=2000 5 | k=5 6 | mu=5 7 | train_examplar_path=./saved_models/$name/train_examplar.jsonl 8 | dev_examplar_path=./saved_models/$name/dev_examplar.jsonl 9 | pretrained_model=/microsoft/codebert-base 10 | 11 | #train 12 | train_data_file=/POJ_clone/binary/train_0.jsonl 13 | eval_data_file=/POJ_clone/binary/dev_0.jsonl 14 | load_model_path=./saved_models/finetune/task_0/checkpoint-best-acc/model.bin 15 | output_dir=./saved_models/$name/task_0 16 | python run_cl.py \ 17 | --output_dir=$output_dir \ 18 | --model_type=roberta \ 19 | --tokenizer_name=$pretrained_model \ 20 | --model_name_or_path=$pretrained_model \ 21 | --load_model_path $load_model_path\ 22 | --do_train \ 23 | --train_data_file=$train_data_file \ 24 | --eval_data_file=$eval_data_file \ 25 | --mode $name \ 26 | --task_id 0 \ 27 | --k $k \ 28 | --mu $mu \ 29 | --ewc_weight $weight \ 30 | --train_replay_size=$train_replay_size \ 31 | --train_examplar_path=$train_examplar_path \ 32 | --eval_examplar_path=$dev_examplar_path \ 33 | --epoch 5 \ 34 | --block_size 256 \ 35 | --train_batch_size 16 \ 36 | --eval_batch_size 64 \ 37 | --learning_rate 2e-5 \ 38 | --max_grad_norm 1.0 \ 39 | --evaluate_during_training \ 40 | --seed 123456 2>&1 | tee train.log 41 | train_data_file=/POJ_clone/binary/train_1.jsonl 42 | eval_data_file=/POJ_clone/binary/dev_1.jsonl 43 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 44 | output_dir=./saved_models/$name/task_1 45 | python run_cl.py \ 46 | --output_dir=$output_dir \ 47 | --model_type=roberta \ 48 | --tokenizer_name=$pretrained_model \ 49 | --model_name_or_path=$pretrained_model \ 50 | --load_model_path $load_model_path\ 51 | --do_train \ 52 | --train_data_file=$train_data_file \ 53 | --eval_data_file=$eval_data_file \ 54 | --mode $name \ 55 | --task_id 1 \ 56 | --k $k \ 57 | --mu $mu \ 58 | --ewc_weight $weight \ 59 | --train_replay_size=$train_replay_size \ 60 | --train_examplar_path=$train_examplar_path \ 61 | --eval_examplar_path=$dev_examplar_path \ 62 | --epoch 5 \ 63 | --block_size 256 \ 64 | --train_batch_size 16 \ 65 | --eval_batch_size 64 \ 66 | --learning_rate 2e-5 \ 67 | --max_grad_norm 1.0 \ 68 | --evaluate_during_training \ 69 | --seed 123456 2>&1 | tee train.log 70 | train_data_file=/POJ_clone/binary/train_2.jsonl 71 | eval_data_file=/POJ_clone/binary/dev_2.jsonl 72 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 73 | output_dir=./saved_models/$name/task_2 74 | python run_cl.py \ 75 | --output_dir=$output_dir \ 76 | --model_type=roberta \ 77 | --tokenizer_name=$pretrained_model \ 78 | --model_name_or_path=$pretrained_model \ 79 | --load_model_path $load_model_path\ 80 | --do_train \ 81 | --train_data_file=$train_data_file \ 82 | --eval_data_file=$eval_data_file \ 83 | --mode $name \ 84 | --task_id 2 \ 85 | --k $k \ 86 | --mu $mu \ 87 | --ewc_weight $weight \ 88 | --train_replay_size=$train_replay_size \ 89 | --train_examplar_path=$train_examplar_path \ 90 | --eval_examplar_path=$dev_examplar_path \ 91 | --epoch 5 \ 92 | --block_size 256 \ 93 | --train_batch_size 16 \ 94 | --eval_batch_size 64 \ 95 | --learning_rate 2e-5 \ 96 | --max_grad_norm 1.0 \ 97 | --evaluate_during_training \ 98 | --seed 123456 2>&1 | tee train.log 99 | train_data_file=/POJ_clone/binary/train_3.jsonl 100 | eval_data_file=/POJ_clone/binary/dev_3.jsonl 101 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 102 | output_dir=./saved_models/$name/task_3 103 | python run_cl.py \ 104 | --output_dir=$output_dir \ 105 | --model_type=roberta \ 106 | --tokenizer_name=$pretrained_model \ 107 | --model_name_or_path=$pretrained_model \ 108 | --load_model_path $load_model_path\ 109 | --do_train \ 110 | --train_data_file=$train_data_file \ 111 | --eval_data_file=$eval_data_file \ 112 | --mode $name \ 113 | --task_id 3 \ 114 | --k $k \ 115 | --mu $mu \ 116 | --ewc_weight $weight \ 117 | --train_replay_size=$train_replay_size \ 118 | --train_examplar_path=$train_examplar_path \ 119 | --eval_examplar_path=$dev_examplar_path \ 120 | --epoch 5 \ 121 | --block_size 256 \ 122 | --train_batch_size 16 \ 123 | --eval_batch_size 64 \ 124 | --learning_rate 2e-5 \ 125 | --max_grad_norm 1.0 \ 126 | --evaluate_during_training \ 127 | --seed 123456 2>&1 | tee train.log 128 | train_data_file=/POJ_clone/binary/train_4.jsonl 129 | eval_data_file=/POJ_clone/binary/dev_4.jsonl 130 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 131 | output_dir=./saved_models/$name/task_4 132 | python run_cl.py \ 133 | --output_dir=$output_dir \ 134 | --model_type=roberta \ 135 | --tokenizer_name=$pretrained_model \ 136 | --model_name_or_path=$pretrained_model \ 137 | --load_model_path $load_model_path\ 138 | --do_train \ 139 | --train_data_file=$train_data_file \ 140 | --eval_data_file=$eval_data_file \ 141 | --mode $name \ 142 | --task_id 4 \ 143 | --k $k \ 144 | --mu $mu \ 145 | --ewc_weight $weight \ 146 | --train_replay_size=$train_replay_size \ 147 | --train_examplar_path=$train_examplar_path \ 148 | --eval_examplar_path=$dev_examplar_path \ 149 | --epoch 5 \ 150 | --block_size 256 \ 151 | --train_batch_size 16 \ 152 | --eval_batch_size 64 \ 153 | --learning_rate 2e-5 \ 154 | --max_grad_norm 1.0 \ 155 | --evaluate_during_training \ 156 | --seed 123456 2>&1 | tee train.log 157 | 158 | 159 | #generate 160 | cp -r ./saved_models/finetune/task_0 ./saved_models/multi_task 161 | data_dir=/POJ_clone/binary 162 | test_data_file=$data_dir/test_0.jsonl,$data_dir/test_1.jsonl,$data_dir/test_2.jsonl,$data_dir/test_3.jsonl,$data_dir/test_4.jsonl 163 | output_dir=./saved_models/$name/task_0,./saved_models/$name/task_1,./saved_models/$name/task_2,./saved_models/$name/task_3,./saved_models/$name/task_4 164 | python run.py \ 165 | --output_dir=$output_dir \ 166 | --model_type=roberta \ 167 | --tokenizer_name=$pretrained_model \ 168 | --model_name_or_path=$pretrained_model \ 169 | --do_test \ 170 | --train_data_file=$test_data_file \ 171 | --test_data_file=$test_data_file \ 172 | --epoch 5 \ 173 | --block_size 256 \ 174 | --train_batch_size 32 \ 175 | --eval_batch_size 64 \ 176 | --learning_rate 2e-5 \ 177 | --max_grad_norm 1.0 \ 178 | --evaluate_during_training \ 179 | --seed 123456 2>&1 | tee test.log 180 | -------------------------------------------------------------------------------- /clone/CodeBERT/test.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=1 2 | name=finetune 3 | pretrained_model=/microsoft/codebert-base 4 | 5 | data_dir=/POJ_clone/binary 6 | test_data_file=$data_dir/test_0.jsonl,$data_dir/test_1.jsonl,$data_dir/test_2.jsonl,$data_dir/test_3.jsonl,$data_dir/test_4.jsonl 7 | output_dir=./saved_models/$name/task_0,./saved_models/$name/task_1,./saved_models/$name/task_2,./saved_models/$name/task_3,./saved_models/$name/task_4 8 | python run.py \ 9 | --output_dir=$output_dir \ 10 | --model_type=roberta \ 11 | --tokenizer_name=$pretrained_model \ 12 | --model_name_or_path=$pretrained_model \ 13 | --do_test \ 14 | --train_data_file=$test_data_file \ 15 | --test_data_file=$test_data_file \ 16 | --epoch 5 \ 17 | --block_size 256 \ 18 | --train_batch_size 32 \ 19 | --eval_batch_size 64 \ 20 | --learning_rate 2e-5 \ 21 | --max_grad_norm 1.0 \ 22 | --evaluate_during_training \ 23 | --seed 123456 2>&1 | tee test.log 24 | -------------------------------------------------------------------------------- /clone/CodeT5/evaluate.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import pandas as pd 3 | import json 4 | from collections import OrderedDict, Counter 5 | import os 6 | import argparse 7 | 8 | 9 | def read_file(base_dir, task_id): 10 | f = csv.reader(open(os.path.join(base_dir, 'test_'+task_id+'.gold'),'r')) 11 | ref = {} 12 | count = 0 13 | for i in f: 14 | ref[count] = [i[0].split('\t')[1].strip()] 15 | count+=1 16 | f = csv.reader(open(os.path.join(base_dir, 'test_'+task_id+'.output'),'r')) 17 | hypo = {} 18 | count = 0 19 | for i in f: 20 | hypo[count] = [i[0].split('\t')[1].strip()] 21 | if len(hypo[count][0].split()) == 0: 22 | hypo[count] = ['Get'] 23 | count+=1 24 | return hypo,ref 25 | 26 | parser = argparse.ArgumentParser() 27 | parser.add_argument("--mode", default=None, type=str, required=True, help="mode") 28 | parser.add_argument("--lang", default=None, type=str, required=True, help="lang") 29 | args = parser.parse_args() 30 | 31 | acc_sum=0 32 | p_sum=0 33 | r_sum=0 34 | f1_sum=0 35 | acc_list=[[] for _ in range(5)] 36 | f1_list=[[] for _ in range(5)] 37 | for i in range(5): 38 | for j in range(i+1): 39 | hypo,ref = read_file('model/'+args.mode+'/task_'+str(i)+'/', str(j)) 40 | tp,tn,fp,fn=0,0,0,0 41 | for k in hypo: 42 | prediction=hypo[k][0] 43 | refrence=ref[k][0] 44 | if prediction=='true' and refrence =='true': 45 | tp+=1 46 | elif prediction=='true' and refrence =='false': 47 | fn+=1 48 | elif prediction=='false' and refrence =='false': 49 | tn+=1 50 | elif prediction=='false' and refrence =='true': 51 | fp+=1 52 | p = tp/(tp+fp) 53 | r = tp/(tp+fn) 54 | f1 = 2*p*r/(p+r) 55 | acc = (tp+tn)/(tp+tn+fp+fn) 56 | #print('('+str(round(100*f1,2))+','+str(round(100*acc,2))+')', end=' ') 57 | print(round(f1*100,2), end=' ') 58 | f1_sum+=f1/(i+1) 59 | acc_sum+=acc/(i+1) 60 | p_sum+=p/(i+1) 61 | r_sum+=r/(i+1) 62 | print('') 63 | 64 | print('Avg_F1:',round(100*f1_sum/5,2)) 65 | print('Avg_P:',round(100*p_sum/5,2)) 66 | print('Avg_R:',round(100*r_sum/5,2)) 67 | print('Avg_ACC:',round(100*acc_sum/5,2)) -------------------------------------------------------------------------------- /clone/CodeT5/ewc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | from torch.autograd import Variable 4 | import copy 5 | #from model.masked_cross_entropy import * 6 | 7 | def variable(t: torch.Tensor, use_cuda=True, **kwargs): 8 | if torch.cuda.is_available() and use_cuda: 9 | t = t.cuda() 10 | return Variable(t, **kwargs) 11 | 12 | class EWC(object): 13 | def __init__(self, model, dataset, device, length, tokenizer): 14 | 15 | self.model = model 16 | # the data we use to compute fisher information of ewc (old_exemplars) 17 | self.dataset = dataset 18 | self.device = device 19 | self.length = length 20 | self.tokenizer = tokenizer 21 | 22 | self.params = {n: p for n, p in self.model.named_parameters() if p.requires_grad} 23 | self._means = {} # previous parameters 24 | self._precision_matrices = self._diag_fisher() # approximated diagnal fisher information matrix 25 | 26 | for n, p in copy.deepcopy(self.params).items(): 27 | self._means[n] = variable(p.data) 28 | 29 | def _diag_fisher(self): 30 | 31 | self.model.train() 32 | precision_matrices = {} 33 | for n, p in copy.deepcopy(self.params).items(): 34 | p.data.zero_() 35 | precision_matrices[n] = variable(p.data) 36 | 37 | for batch in self.dataset: 38 | self.model.zero_grad() 39 | 40 | batch = tuple(t.to(self.device) for t in batch) 41 | source_ids, source_mask, target_ids, target_mask = batch 42 | labels = [ 43 | [(label if label != self.tokenizer.pad_token_id else -100) for label in labels_example] for 44 | labels_example in target_ids 45 | ] 46 | labels = torch.tensor(labels).to(self.device) 47 | out = self.model(input_ids=source_ids, attention_mask=source_mask, labels=labels) 48 | loss = out.loss 49 | 50 | loss.backward() 51 | 52 | for n, p in self.model.named_parameters(): 53 | if p.grad is None: 54 | continue 55 | precision_matrices[n].data += p.grad.data ** 2 / self.length 56 | 57 | precision_matrices = {n: p for n, p in precision_matrices.items()} 58 | self.model.zero_grad() 59 | return precision_matrices 60 | 61 | def penalty(self, model: nn.Module): 62 | loss = 0 63 | for n, p in model.named_parameters(): 64 | _loss = self._precision_matrices[n] * (p - self._means[n]) ** 2 65 | loss += _loss.sum() 66 | 67 | return loss 68 | -------------------------------------------------------------------------------- /clone/CodeT5/my_lib.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | 4 | from openprompt.data_utils import InputExample 5 | 6 | 7 | def get_elapse_time(t0): 8 | elapse_time = time.time() - t0 9 | if elapse_time > 3600: 10 | hour = int(elapse_time // 3600) 11 | minute = int((elapse_time % 3600) // 60) 12 | return "{}h{}m".format(hour, minute) 13 | else: 14 | minute = int((elapse_time % 3600) // 60) 15 | return "{}m".format(minute) 16 | 17 | 18 | class Example(object): 19 | """A single training/test example.""" 20 | 21 | def __init__(self, 22 | idx, 23 | source, 24 | target, 25 | original_source, 26 | original_target 27 | ): 28 | self.idx = idx 29 | self.source = source 30 | self.target = target 31 | self.original_source = original_source 32 | self.original_target = original_target 33 | 34 | 35 | def read_examples(filename, args): 36 | """Read examples from filename.""" 37 | if args.add_task_prefix: 38 | task_prefix = f"Generating comments for {args.lang}: " 39 | else: 40 | task_prefix = "" 41 | 42 | if args.add_lang_ids: 43 | language_prefix = " " 44 | else: 45 | language_prefix = "" 46 | 47 | examples = [] 48 | with open(filename, encoding="utf-8") as f: 49 | for idx, line in enumerate(f): 50 | line = line.strip() 51 | js = json.loads(line) 52 | if 'idx' not in js: 53 | js['idx'] = idx 54 | code = ' '.join(js['code_tokens']).replace('\n', ' ') 55 | code = ' '.join(code.strip().split()) 56 | 57 | nl = ' '.join(js['docstring_tokens']).replace('\n', '') 58 | nl = ' '.join(nl.strip().split()) 59 | examples.append( 60 | Example( 61 | idx=idx, 62 | source=task_prefix + code, 63 | target=language_prefix + nl, 64 | original_source=js['code_tokens'], 65 | original_target=js['docstring_tokens'], 66 | ) 67 | ) 68 | 69 | return examples 70 | 71 | 72 | class InputFeatures(object): 73 | """A single training/test features for an example.""" 74 | 75 | def __init__(self, 76 | example_id, 77 | source_ids, 78 | target_ids, 79 | source_mask, 80 | target_mask, 81 | 82 | ): 83 | self.example_id = example_id 84 | self.source_ids = source_ids 85 | self.target_ids = target_ids 86 | self.source_mask = source_mask 87 | self.target_mask = target_mask 88 | 89 | 90 | def convert_examples_to_features(examples, tokenizer, args, stage=None): 91 | # collect texts 92 | codes = [] 93 | target_nl = [] 94 | for example_id, example in enumerate(examples): 95 | codes.append(example.source) 96 | 97 | if stage == "test": 98 | target_nl.append("None") 99 | else: 100 | target_nl.append(example.target) 101 | 102 | # begin tokenizing 103 | encoded_codes = tokenizer( 104 | codes, padding=True, verbose=False, add_special_tokens=True, 105 | truncation=True, max_length=args.max_source_length, return_tensors='pt') 106 | 107 | encoded_targets = tokenizer( 108 | target_nl, padding=True, verbose=False, add_special_tokens=True, 109 | truncation=True, max_length=args.max_source_length, return_tensors='pt') 110 | 111 | return {'source_ids':encoded_codes['input_ids'], 'target_ids':encoded_targets['input_ids'], 112 | 'source_mask':encoded_codes['attention_mask'], 'target_mask':encoded_targets['attention_mask']} 113 | 114 | 115 | def read_prompt_examples(filename): 116 | """Read examples from filename.""" 117 | examples = [] 118 | with open(filename, encoding="utf-8") as f: 119 | for idx, line in enumerate(f): 120 | line = line.strip() 121 | js = json.loads(line) 122 | if 'idx' not in js: 123 | js['idx'] = idx 124 | code = ' '.join(js['code_tokens']).replace('\n', ' ') 125 | code = ' '.join(code.strip().split()) 126 | nl = ' '.join(js['docstring_tokens']).replace('\n', '') 127 | nl = ' '.join(nl.strip().split()) 128 | # code = js['code'].replace('\n', ' ').strip() 129 | # nl = js['docstring'].replace('\n', ' ').strip() 130 | examples.append( 131 | InputExample( 132 | guid=idx, 133 | text_a= code, 134 | tgt_text= nl, 135 | ) 136 | ) 137 | 138 | return examples -------------------------------------------------------------------------------- /clone/CodeT5/run_emr.sh: -------------------------------------------------------------------------------- 1 | gpu=0 2 | lang=clone 3 | name=emr 4 | train_replay_size=2000 5 | pretrained_model=/Salesforce/codet5-base 6 | num_train_epochs=10 7 | batch_size=16 8 | beam_size=1 9 | train_examplar_path=model/$name/train_examplar.jsonl 10 | dev_examplar_path=model/$name/dev_examplar.jsonl 11 | 12 | #train 13 | train_file=/POJ_clone/binary/train_0.jsonl 14 | dev_file=/POJ_clone/binary/dev_0.jsonl 15 | load_model_path=./model/finetune/task_0/checkpoint-best-f1/pytorch_model.bin 16 | output_dir=model/$name/task_0 17 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 18 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 19 | --mode_name $name --task_id 0 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 20 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 21 | 22 | train_file=/POJ_clone/binary/train_1.jsonl 23 | dev_file=/POJ_clone/binary/dev_1.jsonl 24 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 25 | output_dir=model/$name/task_1 26 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 27 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 28 | --mode_name $name --task_id 1 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 29 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 30 | 31 | train_file=/POJ_clone/binary/train_2.jsonl 32 | dev_file=/POJ_clone/binary/dev_2.jsonl 33 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 34 | output_dir=model/$name/task_2 35 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 36 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 37 | --mode_name $name --task_id 2 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 38 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 39 | 40 | train_file=/POJ_clone/binary/train_3.jsonl 41 | dev_file=/POJ_clone/binary/dev_3.jsonl 42 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 43 | output_dir=model/$name/task_3 44 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 45 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 46 | --mode_name $name --task_id 3 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 47 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 48 | 49 | train_file=/POJ_clone/binary/train_4.jsonl 50 | dev_file=/POJ_clone/binary/dev_4.jsonl 51 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 52 | output_dir=model/$name/task_4 53 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 54 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 55 | --mode_name $name --task_id 4 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 56 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 57 | 58 | 59 | #generate 60 | beam_size=1 61 | batch_size=32 62 | 63 | mkdir -p model/$name/task_0 64 | cp ./model/finetune/task_0/test_0.gold ./model/$name/task_0 65 | cp ./model/finetune/task_0/test_0.output ./model/$name/task_0 66 | 67 | output_dir=model/$name/task_1 68 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 69 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl 70 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 71 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir \ 72 | --eval_batch_size $batch_size --beam_size $beam_size 73 | 74 | output_dir=model/$name/task_2 75 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 76 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl 77 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 78 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir \ 79 | --eval_batch_size $batch_size --beam_size $beam_size 80 | 81 | output_dir=model/$name/task_3 82 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 83 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl,/POJ_clone/binary/test_3.jsonl 84 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 85 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir \ 86 | --eval_batch_size $batch_size --beam_size $beam_size 87 | 88 | output_dir=model/$name/task_4 89 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 90 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl,/POJ_clone/binary/test_3.jsonl,/POJ_clone/binary/test_4.jsonl 91 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 92 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir \ 93 | --eval_batch_size $batch_size --beam_size $beam_size 94 | 95 | 96 | #evaluate 97 | python evaluate.py --mode $name --lang $lang -------------------------------------------------------------------------------- /clone/CodeT5/run_ewc.sh: -------------------------------------------------------------------------------- 1 | gpu=0 2 | lang=clone 3 | name=ewc 4 | train_replay_size=2000 5 | weight=2000 6 | pretrained_model=/Salesforce/codet5-base 7 | num_train_epochs=10 8 | batch_size=16 9 | beam_size=1 10 | train_examplar_path=model/$name/train_examplar.jsonl 11 | dev_examplar_path=model/$name/dev_examplar.jsonl 12 | 13 | #train 14 | train_file=/POJ_clone/binary/train_0.jsonl 15 | dev_file=/POJ_clone/binary/dev_0.jsonl 16 | load_model_path=./model/finetune/task_0/checkpoint-best-f1/pytorch_model.bin 17 | output_dir=model/$name/task_0 18 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 19 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 20 | --mode_name $name --task_id 0 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size --ewc_weight $weight\ 21 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 22 | 23 | train_file=/POJ_clone/binary/train_1.jsonl 24 | dev_file=/POJ_clone/binary/dev_1.jsonl 25 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 26 | output_dir=model/$name/task_1 27 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 28 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 29 | --mode_name $name --task_id 1 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size --ewc_weight $weight\ 30 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 31 | 32 | train_file=/POJ_clone/binary/train_2.jsonl 33 | dev_file=/POJ_clone/binary/dev_2.jsonl 34 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 35 | output_dir=model/$name/task_2 36 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 37 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 38 | --mode_name $name --task_id 2 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 39 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 40 | 41 | train_file=/POJ_clone/binary/train_3.jsonl 42 | dev_file=/POJ_clone/binary/dev_3.jsonl 43 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 44 | output_dir=model/$name/task_3 45 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 46 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 47 | --mode_name $name --task_id 3 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size --ewc_weight $weight\ 48 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 49 | 50 | train_file=/POJ_clone/binary/train_4.jsonl 51 | dev_file=/POJ_clone/binary/dev_4.jsonl 52 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 53 | output_dir=model/$name/task_4 54 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 55 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 56 | --mode_name $name --task_id 4 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size --ewc_weight $weight\ 57 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 58 | 59 | 60 | #generate 61 | beam_size=1 62 | batch_size=32 63 | 64 | mkdir -p model/$name/task_0 65 | cp ./model/$lang/finetune/task_0/test_0.gold ./model/$name/task_0 66 | cp ./model/$lang/finetune/task_0/test_0.output ./model/$name/task_0 67 | 68 | output_dir=model/$name/task_1 69 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 70 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl 71 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 72 | --do_test --model_name $pretrained_model --output_dir $output_dir \ 73 | --eval_batch_size $batch_size --beam_size $beam_size 74 | 75 | output_dir=model/$name/task_2 76 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 77 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl 78 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 79 | --do_test --model_name $pretrained_model --output_dir $output_dir \ 80 | --eval_batch_size $batch_size --beam_size $beam_size 81 | 82 | output_dir=model/$name/task_3 83 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 84 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl,/POJ_clone/binary/test_3.jsonl 85 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 86 | --do_test --model_name $pretrained_model --output_dir $output_dir \ 87 | --eval_batch_size $batch_size --beam_size $beam_size 88 | 89 | output_dir=model/$name/task_4 90 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 91 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl,/POJ_clone/binary/test_3.jsonl,/POJ_clone/binary/test_4.jsonl 92 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 93 | --do_test --model_name $pretrained_model --output_dir $output_dir \ 94 | --eval_batch_size $batch_size --beam_size $beam_size 95 | 96 | 97 | #evaluate 98 | python evaluate.py --mode $name --lang $lang -------------------------------------------------------------------------------- /clone/CodeT5/run_finetune.sh: -------------------------------------------------------------------------------- 1 | gpu=0 2 | lang=clone 3 | name=finetune 4 | pretrained_model=/Salesforce/codet5-base 5 | num_train_epochs=10 6 | batch_size=16 7 | beam_size=1 8 | 9 | #train 10 | train_file=/POJ_clone/binary/train_0.jsonl 11 | dev_file=/POJ_clone/binary/dev_0.jsonl 12 | output_dir=model/$name/task_0 13 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 14 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir \ 15 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 16 | 17 | train_file=/POJ_clone/binary/train_1.jsonl 18 | dev_file=/POJ_clone/binary/dev_1.jsonl 19 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 20 | output_dir=model/$name/task_1 21 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 22 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 23 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 24 | 25 | train_file=/POJ_clone/binary/train_2.jsonl 26 | dev_file=/POJ_clone/binary/dev_2.jsonl 27 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 28 | output_dir=model/$name/task_2 29 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 30 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 31 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 32 | 33 | train_file=/POJ_clone/binary/train_3.jsonl 34 | dev_file=/POJ_clone/binary/dev_3.jsonl 35 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 36 | output_dir=model/$name/task_3 37 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 38 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 39 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 40 | 41 | train_file=/POJ_clone/binary/train_4.jsonl 42 | dev_file=/POJ_clone/binary/dev_4.jsonl 43 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 44 | output_dir=model/$name/task_4 45 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 46 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 47 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 48 | 49 | 50 | #generate 51 | beam_size=10 52 | batch_size=32 53 | 54 | output_dir=model/$name/task_0 55 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 56 | test_file=/POJ_clone/binary/test_0.jsonl 57 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 58 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model\ 59 | --eval_batch_size $batch_size --beam_size $beam_size 60 | 61 | output_dir=model/$name/task_1 62 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 63 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl 64 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 65 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 66 | --eval_batch_size $batch_size --beam_size $beam_size 67 | 68 | output_dir=model/$name/task_2 69 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 70 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl 71 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 72 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 73 | --eval_batch_size $batch_size --beam_size $beam_size 74 | 75 | output_dir=model/$name/task_3 76 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 77 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl,/POJ_clone/binary/test_3.jsonl 78 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 79 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 80 | --eval_batch_size $batch_size --beam_size $beam_size 81 | 82 | output_dir=model/$name/task_4 83 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 84 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl,/POJ_clone/binary/test_3.jsonl,/POJ_clone/binary/test_4.jsonl 85 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 86 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 87 | --eval_batch_size $batch_size --beam_size $beam_size 88 | 89 | 90 | #evaluate 91 | python evaluate.py --mode $name --lang $lang -------------------------------------------------------------------------------- /clone/CodeT5/run_multitask.sh: -------------------------------------------------------------------------------- 1 | gpu=0 2 | lang=clone 3 | name=multi_task 4 | pretrained_model=/Salesforce/codet5-base 5 | num_train_epochs=10 6 | batch_size=16 7 | beam_size=1 8 | 9 | #train 10 | 11 | train_file=/POJ_clone/binaryT5/train_01.jsonl 12 | dev_file=/POJ_clone/binaryT5/dev_01.jsonl 13 | output_dir=model/$name/task_1 14 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 15 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 16 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 17 | 18 | train_file=/POJ_clone/binaryT5/train_012.jsonl 19 | dev_file=/POJ_clone/binaryT5/dev_012.jsonl 20 | output_dir=model/$name/task_2 21 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 22 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 23 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 24 | 25 | train_file=/POJ_clone/binaryT5/train_0123.jsonl 26 | dev_file=/POJ_clone/binaryT5/dev_0123.jsonl 27 | output_dir=model/$name/task_3 28 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 29 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 30 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 31 | 32 | train_file=/POJ_clone/binaryT5/train.jsonl 33 | dev_file=/POJ_clone/binaryT5/dev.jsonl 34 | output_dir=model/$name/task_4 35 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 36 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 37 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 38 | -------------------------------------------------------------------------------- /clone/CodeT5/test.sh: -------------------------------------------------------------------------------- 1 | gpu=0 2 | lang=clone 3 | name=finetune 4 | pretrained_model=/Salesforce/codet5-base 5 | data_dir=/CodeSearchNet 6 | batch_size=32 7 | beam_size=1 8 | 9 | 10 | #generate 11 | beam_size=10 12 | batch_size=32 13 | 14 | mkdir -p model/$name/task_0 15 | cp ./model/finetune/task_0/test_0.gold ./model/$name/task_0 16 | cp ./model/finetune/task_0/test_0.output ./model/$name/task_0 17 | 18 | output_dir=model/$name/task_1 19 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 20 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl 21 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 22 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $test_model\ 23 | --eval_batch_size $batch_size --beam_size $beam_size 24 | 25 | output_dir=model/$name/task_2 26 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 27 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl 28 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 29 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $test_model\ 30 | --eval_batch_size $batch_size --beam_size $beam_size 31 | 32 | output_dir=model/$name/task_3 33 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 34 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl,/POJ_clone/binary/test_3.jsonl 35 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 36 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $test_model\ 37 | --eval_batch_size $batch_size --beam_size $beam_size 38 | 39 | output_dir=model/$name/task_4 40 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 41 | test_file=/POJ_clone/binary/test_0.jsonl,/POJ_clone/binary/test_1.jsonl,/POJ_clone/binary/test_2.jsonl,/POJ_clone/binary/test_3.jsonl,/POJ_clone/binary/test_4.jsonl 42 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 512 --max_target_length 128 --test_file $test_file \ 43 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $test_model\ 44 | --eval_batch_size $batch_size --beam_size $beam_size 45 | 46 | 47 | #evaluate 48 | python evaluate.py --mode $name --lang $lang -------------------------------------------------------------------------------- /result.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/result.xlsx -------------------------------------------------------------------------------- /sum/CodeBERT/evall/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'wasi' 2 | 3 | from . import bleu 4 | from . import rouge 5 | from . import meteor 6 | -------------------------------------------------------------------------------- /sum/CodeBERT/evall/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'wasi' 2 | 3 | from .bleu import * 4 | from .bleu_scorer import * 5 | from .nltk_bleu import * 6 | from .google_bleu import * 7 | -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/bleu.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/bleu.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/bleu.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/bleu.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/bleu.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/bleu.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/bleu_scorer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/bleu_scorer.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/bleu_scorer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/bleu_scorer.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/bleu_scorer.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/bleu_scorer.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/google_bleu.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/google_bleu.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/google_bleu.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/google_bleu.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/google_bleu.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/google_bleu.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/nltk_bleu.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/nltk_bleu.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/nltk_bleu.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/nltk_bleu.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/__pycache__/nltk_bleu.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/bleu/__pycache__/nltk_bleu.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/bleu.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # File Name : bleu.py 4 | # 5 | # Description : Wrapper for BLEU scorer. 6 | # 7 | # Creation Date : 06-01-2015 8 | # Last Modified : Thu 19 Mar 2015 09:13:28 PM PDT 9 | # Authors : Hao Fang and Tsung-Yi Lin 10 | 11 | from .bleu_scorer import BleuScorer 12 | 13 | 14 | class Bleu: 15 | def __init__(self, n=4): 16 | # default compute Blue score up to 4 17 | self._n = n 18 | self._hypo_for_image = {} 19 | self.ref_for_image = {} 20 | 21 | def compute_score(self, gts, res, verbose): 22 | assert (sorted(gts.keys()) == sorted(res.keys())) 23 | imgIds = list(gts.keys()) 24 | 25 | bleu_scorer = BleuScorer(n=self._n) 26 | for id in imgIds: 27 | hypo = res[id] 28 | ref = gts[id] 29 | 30 | # Sanity check. 31 | assert (type(hypo) is list) 32 | assert (len(hypo) == 1) 33 | assert (type(ref) is list) 34 | assert (len(ref) >= 1) 35 | 36 | bleu_scorer += (hypo[0], ref) 37 | 38 | # score, scores = bleu_scorer.compute_score(option='shortest') 39 | score, scores, bleu = bleu_scorer.compute_score(option='closest', verbose=verbose) 40 | # score, scores = bleu_scorer.compute_score(option='average', verbose=1) 41 | 42 | return score, scores, bleu 43 | 44 | def method(self): 45 | return "Bleu" 46 | -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/google_bleu.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Python implementation of BLEU and smooth-BLEU. 17 | This module provides a Python implementation of BLEU and smooth-BLEU. 18 | Smooth BLEU is computed following the method outlined in the paper: 19 | Chin-Yew Lin, Franz Josef Och. ORANGE: a method for evaluating automatic 20 | evaluation metrics for machine translation. COLING 2004. 21 | """ 22 | 23 | import collections 24 | import math 25 | 26 | 27 | def _get_ngrams(segment, max_order): 28 | """Extracts all n-grams upto a given maximum order from an input segment. 29 | Args: 30 | segment: text segment from which n-grams will be extracted. 31 | max_order: maximum length in tokens of the n-grams returned by this 32 | methods. 33 | Returns: 34 | The Counter containing all n-grams upto max_order in segment 35 | with a count of how many times each n-gram occurred. 36 | """ 37 | ngram_counts = collections.Counter() 38 | for order in range(1, max_order + 1): 39 | for i in range(0, len(segment) - order + 1): 40 | ngram = tuple(segment[i:i + order]) 41 | ngram_counts[ngram] += 1 42 | return ngram_counts 43 | 44 | 45 | def compute_bleu(reference_corpus, translation_corpus, max_order=4, 46 | smooth=False): 47 | """Computes BLEU score of translated segments against one or more references. 48 | Args: 49 | reference_corpus: list of lists of references for each translation. Each 50 | reference should be tokenized into a list of tokens. 51 | translation_corpus: list of translations to score. Each translation 52 | should be tokenized into a list of tokens. 53 | max_order: Maximum n-gram order to use when computing BLEU score. 54 | smooth: Whether or not to apply Lin et al. 2004 smoothing. 55 | Returns: 56 | 3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram 57 | precisions and brevity penalty. 58 | """ 59 | matches_by_order = [0] * max_order 60 | possible_matches_by_order = [0] * max_order 61 | reference_length = 0 62 | translation_length = 0 63 | for (references, translation) in zip(reference_corpus, 64 | translation_corpus): 65 | reference_length += min(len(r) for r in references) 66 | translation_length += len(translation) 67 | 68 | merged_ref_ngram_counts = collections.Counter() 69 | for reference in references: 70 | merged_ref_ngram_counts |= _get_ngrams(reference, max_order) 71 | translation_ngram_counts = _get_ngrams(translation, max_order) 72 | overlap = translation_ngram_counts & merged_ref_ngram_counts 73 | for ngram in overlap: 74 | matches_by_order[len(ngram) - 1] += overlap[ngram] 75 | for order in range(1, max_order + 1): 76 | possible_matches = len(translation) - order + 1 77 | if possible_matches > 0: 78 | possible_matches_by_order[order - 1] += possible_matches 79 | 80 | precisions = [0] * max_order 81 | for i in range(0, max_order): 82 | if smooth: 83 | precisions[i] = ((matches_by_order[i] + 1.) / 84 | (possible_matches_by_order[i] + 1.)) 85 | else: 86 | if possible_matches_by_order[i] > 0: 87 | precisions[i] = (float(matches_by_order[i]) / 88 | possible_matches_by_order[i]) 89 | else: 90 | precisions[i] = 0.0 91 | 92 | if min(precisions) > 0: 93 | p_log_sum = sum((1. / max_order) * math.log(p) for p in precisions) 94 | geo_mean = math.exp(p_log_sum) 95 | else: 96 | geo_mean = 0 97 | 98 | ratio = float(translation_length) / reference_length 99 | 100 | if ratio > 1.0: 101 | bp = 1. 102 | else: 103 | bp = math.exp(1 - 1. / ratio) 104 | 105 | bleu = geo_mean * bp 106 | 107 | return (bleu, precisions, bp, ratio, translation_length, reference_length) 108 | 109 | 110 | def corpus_bleu(hypotheses, references): 111 | refs = [] 112 | hyps = [] 113 | count = 0 114 | total_score = 0.0 115 | 116 | assert (sorted(hypotheses.keys()) == sorted(references.keys())) 117 | Ids = list(hypotheses.keys()) 118 | ind_score = dict() 119 | 120 | for id in Ids: 121 | hyp = hypotheses[id][0].split() 122 | ref = [r.split() for r in references[id]] 123 | hyps.append(hyp) 124 | refs.append(ref) 125 | 126 | score = compute_bleu([ref], [hyp], smooth=True)[0] 127 | total_score += score 128 | count += 1 129 | ind_score[id] = score 130 | 131 | avg_score = total_score / count 132 | corpus_bleu = compute_bleu(refs, hyps, smooth=True)[0] 133 | return corpus_bleu, avg_score, ind_score 134 | -------------------------------------------------------------------------------- /sum/CodeBERT/evall/bleu/nltk_bleu.py: -------------------------------------------------------------------------------- 1 | import nltk 2 | from nltk.translate.bleu_score import SmoothingFunction 3 | 4 | 5 | def nltk_sentence_bleu(hypothesis, reference, order=4): 6 | cc = SmoothingFunction() 7 | return nltk.translate.bleu([reference], hypothesis, smoothing_function=cc.method4) 8 | 9 | 10 | def nltk_corpus_bleu(hypotheses, references, order=4): 11 | refs = [] 12 | hyps = [] 13 | count = 0 14 | total_score = 0.0 15 | 16 | cc = SmoothingFunction() 17 | 18 | assert (sorted(hypotheses.keys()) == sorted(references.keys())) 19 | Ids = list(hypotheses.keys()) 20 | ind_score = dict() 21 | 22 | for id in Ids: 23 | hyp = hypotheses[id][0].split() 24 | ref = [r.split() for r in references[id]] 25 | hyps.append(hyp) 26 | refs.append(ref) 27 | 28 | score = nltk.translate.bleu(ref, hyp, smoothing_function=cc.method4) 29 | total_score += score 30 | count += 1 31 | ind_score[id] = score 32 | 33 | avg_score = total_score / count 34 | corpus_bleu = nltk.translate.bleu_score.corpus_bleu(refs, hyps) 35 | return corpus_bleu, avg_score, ind_score 36 | -------------------------------------------------------------------------------- /sum/CodeBERT/evall/meteor/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'wasi' 2 | 3 | from .meteor import * 4 | -------------------------------------------------------------------------------- /sum/CodeBERT/evall/meteor/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/meteor/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/meteor/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/meteor/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/meteor/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/meteor/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/meteor/__pycache__/meteor.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/meteor/__pycache__/meteor.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/meteor/__pycache__/meteor.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/meteor/__pycache__/meteor.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/meteor/__pycache__/meteor.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/meteor/__pycache__/meteor.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/meteor/data/paraphrase-en.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/meteor/data/paraphrase-en.gz -------------------------------------------------------------------------------- /sum/CodeBERT/evall/meteor/meteor-1.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/meteor/meteor-1.5.jar -------------------------------------------------------------------------------- /sum/CodeBERT/evall/meteor/meteor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Python wrapper for METEOR implementation, by Xinlei Chen 4 | # Acknowledge Michael Denkowski for the generous discussion and help 5 | from __future__ import division 6 | 7 | import atexit 8 | import logging 9 | import os 10 | import subprocess 11 | import sys 12 | import threading 13 | 14 | import psutil 15 | 16 | # Assumes meteor-1.5.jar is in the same directory as meteor.py. Change as needed. 17 | METEOR_JAR = 'meteor-1.5.jar' 18 | 19 | 20 | def enc(s): 21 | return s.encode('utf-8') 22 | 23 | 24 | def dec(s): 25 | return s.decode('utf-8') 26 | 27 | 28 | class Meteor: 29 | 30 | def __init__(self): 31 | # Used to guarantee thread safety 32 | self.lock = threading.Lock() 33 | 34 | mem = '2G' 35 | mem_available_G = psutil.virtual_memory().available / 1E9 36 | if mem_available_G < 2: 37 | logging.warning("There is less than 2GB of available memory.\n" 38 | "Will try with limiting Meteor to 1GB of memory but this might cause issues.\n" 39 | "If you have problems using Meteor, " 40 | "then you can try to lower the `mem` variable in meteor.py") 41 | mem = '1G' 42 | 43 | meteor_cmd = ['java', '-jar', '-Xmx{}'.format(mem), METEOR_JAR, 44 | '-', '-', '-stdio', '-l', 'en', '-norm'] 45 | env = os.environ.copy() 46 | env['LC_ALL'] = "C" 47 | self.meteor_p = subprocess.Popen(meteor_cmd, 48 | cwd=os.path.dirname(os.path.abspath(__file__)), 49 | env=env, 50 | stdin=subprocess.PIPE, 51 | stdout=subprocess.PIPE, 52 | stderr=subprocess.PIPE) 53 | 54 | atexit.register(self.close) 55 | 56 | def close(self): 57 | with self.lock: 58 | if self.meteor_p: 59 | self.meteor_p.kill() 60 | self.meteor_p.wait() 61 | self.meteor_p = None 62 | # if the user calls close() manually, remove the 63 | # reference from atexit so the object can be garbage-collected. 64 | if atexit is not None and atexit.unregister is not None: 65 | atexit.unregister(self.close) 66 | 67 | def compute_score(self, gts, res): 68 | assert (gts.keys() == res.keys()) 69 | imgIds = gts.keys() 70 | scores = [] 71 | 72 | eval_line = 'EVAL' 73 | with self.lock: 74 | for i in imgIds: 75 | assert (len(res[i]) == 1) 76 | stat = self._stat(res[i][0], gts[i]) 77 | eval_line += ' ||| {}'.format(stat) 78 | 79 | self.meteor_p.stdin.write(enc('{}\n'.format(eval_line))) 80 | self.meteor_p.stdin.flush() 81 | for i in range(0, len(imgIds)): 82 | v = self.meteor_p.stdout.readline() 83 | try: 84 | scores.append(float(dec(v.strip()))) 85 | except: 86 | sys.stderr.write("Error handling value: {}\n".format(v)) 87 | sys.stderr.write("Decoded value: {}\n".format(dec(v.strip()))) 88 | sys.stderr.write("eval_line: {}\n".format(eval_line)) 89 | # You can try uncommenting the next code line to show stderr from the Meteor JAR. 90 | # If the Meteor JAR is not writing to stderr, then the line will just hang. 91 | # sys.stderr.write("Error from Meteor:\n{}".format(self.meteor_p.stderr.read())) 92 | raise 93 | score = float(dec(self.meteor_p.stdout.readline()).strip()) 94 | 95 | return score, scores 96 | 97 | def method(self): 98 | return "METEOR" 99 | 100 | def _stat(self, hypothesis_str, reference_list): 101 | # SCORE ||| reference 1 words ||| reference n words ||| hypothesis words 102 | hypothesis_str = hypothesis_str.replace('|||', '').replace(' ', ' ') 103 | score_line = ' ||| '.join(('SCORE', ' ||| '.join(reference_list), hypothesis_str)) 104 | self.meteor_p.stdin.write(enc(score_line)) 105 | self.meteor_p.stdin.write(enc('\n')) 106 | self.meteor_p.stdin.flush() 107 | return dec(self.meteor_p.stdout.readline()).strip() 108 | 109 | def _score(self, hypothesis_str, reference_list): 110 | with self.lock: 111 | # SCORE ||| reference 1 words ||| reference n words ||| hypothesis words 112 | hypothesis_str = hypothesis_str.replace('|||', '').replace(' ', ' ') 113 | score_line = ' ||| '.join(('SCORE', ' ||| '.join(reference_list), hypothesis_str)) 114 | self.meteor_p.stdin.write(enc('{}\n'.format(score_line))) 115 | self.meteor_p.stdin.flush() 116 | stats = dec(self.meteor_p.stdout.readline()).strip() 117 | eval_line = 'EVAL ||| {}'.format(stats) 118 | # EVAL ||| stats 119 | self.meteor_p.stdin.write(enc('{}\n'.format(eval_line))) 120 | self.meteor_p.stdin.flush() 121 | score = float(dec(self.meteor_p.stdout.readline()).strip()) 122 | # bug fix: there are two values returned by the jar file, one average, and one all, so do it twice 123 | # thanks for Andrej for pointing this out 124 | score = float(dec(self.meteor_p.stdout.readline()).strip()) 125 | return score 126 | 127 | def __del__(self): 128 | self.close() 129 | -------------------------------------------------------------------------------- /sum/CodeBERT/evall/rouge/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'wasi' 2 | 3 | from .rouge import * 4 | -------------------------------------------------------------------------------- /sum/CodeBERT/evall/rouge/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/rouge/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/rouge/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/rouge/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/rouge/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/rouge/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/rouge/__pycache__/rouge.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/rouge/__pycache__/rouge.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/rouge/__pycache__/rouge.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/rouge/__pycache__/rouge.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/rouge/__pycache__/rouge.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/evall/rouge/__pycache__/rouge.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeBERT/evall/rouge/rouge.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # File Name : rouge.py 4 | # 5 | # Description : Computes ROUGE-L metric as described by Lin and Hovey (2004) 6 | # 7 | # Creation Date : 2015-01-07 06:03 8 | # Author : Ramakrishna Vedantam 9 | 10 | import numpy as np 11 | 12 | 13 | def my_lcs(string, sub): 14 | """ 15 | Calculates longest common subsequence for a pair of tokenized strings 16 | :param string : list of str : tokens from a string split using whitespace 17 | :param sub : list of str : shorter string, also split using whitespace 18 | :returns: length (list of int): length of the longest common subsequence between the two strings 19 | Note: my_lcs only gives length of the longest common subsequence, not the actual LCS 20 | """ 21 | if len(string) < len(sub): 22 | sub, string = string, sub 23 | 24 | lengths = [[0 for i in range(0, len(sub) + 1)] for j in range(0, len(string) + 1)] 25 | 26 | for j in range(1, len(sub) + 1): 27 | for i in range(1, len(string) + 1): 28 | if string[i - 1] == sub[j - 1]: 29 | lengths[i][j] = lengths[i - 1][j - 1] + 1 30 | else: 31 | lengths[i][j] = max(lengths[i - 1][j], lengths[i][j - 1]) 32 | 33 | return lengths[len(string)][len(sub)] 34 | 35 | 36 | class Rouge(): 37 | ''' 38 | Class for computing ROUGE-L score for a set of candidate sentences for the MS COCO test set 39 | ''' 40 | 41 | def __init__(self): 42 | # vrama91: updated the value below based on discussion with Hovey 43 | self.beta = 1.2 44 | 45 | def calc_score(self, candidate, refs): 46 | """ 47 | Compute ROUGE-L score given one candidate and references for an image 48 | :param candidate: str : candidate sentence to be evaluated 49 | :param refs: list of str : COCO reference sentences for the particular image to be evaluated 50 | :returns score: int (ROUGE-L score for the candidate evaluated against references) 51 | """ 52 | assert (len(candidate) == 1) 53 | assert (len(refs) > 0) 54 | prec = [] 55 | rec = [] 56 | 57 | # split into tokens 58 | token_c = candidate[0].split(" ") 59 | 60 | for reference in refs: 61 | # split into tokens 62 | token_r = reference.split(" ") 63 | # compute the longest common subsequence 64 | lcs = my_lcs(token_r, token_c) 65 | prec.append(lcs / float(len(token_c))) 66 | rec.append(lcs / float(len(token_r))) 67 | 68 | prec_max = max(prec) 69 | rec_max = max(rec) 70 | 71 | if prec_max != 0 and rec_max != 0: 72 | score = ((1 + self.beta ** 2) * prec_max * rec_max) / float(rec_max + self.beta ** 2 * prec_max) 73 | else: 74 | score = 0.0 75 | return score 76 | 77 | def compute_score(self, gts, res): 78 | """ 79 | Computes Rouge-L score given a set of reference and candidate sentences for the dataset 80 | Invoked by evaluate_captions.py 81 | :param gts: dict : candidate / test sentences with "image name" key and "tokenized sentences" as values 82 | :param res: dict : reference MS-COCO sentences with "image name" key and "tokenized sentences" as values 83 | :returns: average_score: float (mean ROUGE-L score computed by averaging scores for all the images) 84 | """ 85 | assert (sorted(gts.keys()) == sorted(res.keys())) 86 | imgIds = list(gts.keys()) 87 | 88 | score = dict() 89 | for id in imgIds: 90 | hypo = res[id] 91 | ref = gts[id] 92 | 93 | # Sanity check. 94 | assert (type(hypo) is list) 95 | assert (len(hypo) == 1) 96 | assert (type(ref) is list) 97 | assert (len(ref) > 0) 98 | 99 | score[id] = self.calc_score(hypo, ref) 100 | 101 | average_score = np.mean(np.array(list(score.values()))) 102 | return average_score, score 103 | 104 | def method(self): 105 | return "Rouge" 106 | -------------------------------------------------------------------------------- /sum/CodeBERT/evaluate.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import pandas as pd 3 | import json 4 | from collections import OrderedDict, Counter 5 | import os 6 | import argparse 7 | 8 | from evall.bleu import corpus_bleu 9 | from evall.rouge import Rouge 10 | from evall.meteor import Meteor 11 | 12 | def eval_accuracies(hypotheses, references, sources=None, 13 | filename=None, print_copy_info=False, mode='dev'): 14 | _, bleu, ind_bleu = corpus_bleu(hypotheses, references) 15 | 16 | # Compute ROUGE scores 17 | rouge_calculator = Rouge() 18 | rouge_l, ind_rouge = rouge_calculator.compute_score(references, hypotheses) 19 | meteor_calculator = Meteor() 20 | meteor, _ = meteor_calculator.compute_score(references, hypotheses) 21 | 22 | return bleu * 100, rouge_l * 100, meteor * 100 23 | 24 | def read_file(base_dir, task_id): 25 | f = csv.reader(open(os.path.join(base_dir, 'test_generate'+task_id+'.gold'),'r')) 26 | ref = {} 27 | count = 0 28 | for i in f: 29 | ref[count] = [i[0].split('\t')[1].strip()] 30 | count+=1 31 | f = csv.reader(open(os.path.join(base_dir, 'test_generate'+task_id+'.output'),'r')) 32 | hypo = {} 33 | count = 0 34 | for i in f: 35 | hypo[count] = [i[0].split('\t')[1].strip()] 36 | if len(hypo[count][0].split()) == 0: 37 | hypo[count] = ['Get'] 38 | count+=1 39 | return hypo,ref 40 | 41 | parser = argparse.ArgumentParser() 42 | parser.add_argument("--mode", default=None, type=str, required=True, help="mode") 43 | parser.add_argument("--lang", default=None, type=str, required=True, help="lang") 44 | args = parser.parse_args() 45 | 46 | bleu_sum=0 47 | meteor_sum=0 48 | rouge_sum=0 49 | first_bleu=0 50 | first_meteor=0 51 | first_rouge=0 52 | for i in range(5): 53 | for j in range(i+1): 54 | hypo,ref = read_file('model/'+args.lang+'/'+args.mode+'/task_'+str(i)+'/', str(j)) 55 | bleu,rouge,meteor = eval_accuracies(hypo,ref) 56 | #print('Model:',i,' Task:',j) 57 | #print('bleu:',bleu,' meteor:',meteor,' rouge:',rouge) 58 | print(round(bleu,2), end=' ') 59 | if j==0: 60 | first_bleu+=bleu 61 | first_meteor+=meteor 62 | first_rouge+=rouge 63 | bleu_sum+=bleu/(i+1) 64 | meteor_sum+=meteor/(i+1) 65 | rouge_sum+=rouge/(i+1) 66 | print('') 67 | 68 | #print('first_bleu:',first_bleu/5,' first_meteor:',first_meteor/5,' first_rouge:',first_rouge/5) 69 | #print('avg_bleu:',bleu_sum/5,' avg_meteor:',meteor_sum/5,' avg_rouge:',rouge_sum/5) 70 | print('Avg_BLEU:',round(bleu_sum/5,2)) 71 | print('Avg_METEOR:',round(meteor_sum/5,2)) 72 | print('Avg_ROUGE:',round(rouge_sum/5,2)) -------------------------------------------------------------------------------- /sum/CodeBERT/ewc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | from torch.autograd import Variable 4 | import copy 5 | #from model.masked_cross_entropy import * 6 | 7 | def variable(t: torch.Tensor, use_cuda=True, **kwargs): 8 | if torch.cuda.is_available() and use_cuda: 9 | t = t.cuda() 10 | return Variable(t, **kwargs) 11 | 12 | class EWC(object): 13 | def __init__(self, model, dataset, device, length): 14 | 15 | self.model = model 16 | # the data we use to compute fisher information of ewc (old_exemplars) 17 | self.dataset = dataset 18 | self.device = device 19 | self.length = length 20 | 21 | self.params = {n: p for n, p in self.model.named_parameters() if p.requires_grad} 22 | self._means = {} # previous parameters 23 | self._precision_matrices = self._diag_fisher() # approximated diagnal fisher information matrix 24 | 25 | for n, p in copy.deepcopy(self.params).items(): 26 | self._means[n] = variable(p.data) 27 | 28 | def _diag_fisher(self): 29 | 30 | self.model.train() 31 | precision_matrices = {} 32 | for n, p in copy.deepcopy(self.params).items(): 33 | p.data.zero_() 34 | precision_matrices[n] = variable(p.data) 35 | 36 | for batch in self.dataset: 37 | self.model.zero_grad() 38 | 39 | batch = tuple(t.to(self.device) for t in batch) 40 | source_ids,source_mask,target_ids,target_mask = batch 41 | loss,_,_ = self.model(source_ids=source_ids,source_mask=source_mask,target_ids=target_ids,target_mask=target_mask) 42 | 43 | loss.backward() 44 | 45 | for n, p in self.model.named_parameters(): 46 | if p.grad is None: 47 | continue 48 | precision_matrices[n].data += p.grad.data ** 2 / self.length 49 | 50 | precision_matrices = {n: p for n, p in precision_matrices.items()} 51 | self.model.zero_grad() 52 | return precision_matrices 53 | 54 | def penalty(self, model: nn.Module): 55 | loss = 0 56 | for n, p in model.named_parameters(): 57 | _loss = self._precision_matrices[n] * (p - self._means[n]) ** 2 58 | loss += _loss.sum() 59 | 60 | return loss 61 | -------------------------------------------------------------------------------- /sum/CodeBERT/run_finetune.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=1 2 | lang=java #programming language 3 | name=finetune 4 | lr=5e-5 5 | batch_size=32 6 | beam_size=10 7 | source_length=256 8 | target_length=128 9 | data_dir=/CodeSearchNet 10 | num_train_epochs=15 11 | train_file=$data_dir/$lang/cl/train_0.jsonl 12 | dev_file=$data_dir/$lang/cl/dev_0.jsonl 13 | pretrained_model=/microsoft/codebert-base 14 | output_dir=model/$lang/$name/task_0 15 | 16 | python run.py --do_train --do_eval --model_type roberta --model_name_or_path $pretrained_model --train_filename $train_file --dev_filename $dev_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --train_batch_size $batch_size --eval_batch_size $batch_size --learning_rate $lr --num_train_epochs $num_train_epochs 17 | 18 | train_file=$data_dir/$lang/cl/train_1.jsonl 19 | dev_file=$data_dir/$lang/cl/dev_1.jsonl 20 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 21 | output_dir=model/$lang/$name/task_1 22 | 23 | python run.py --do_train --do_eval --model_type roberta --model_name_or_path $pretrained_model --load_model_path $load_model_path --train_filename $train_file --dev_filename $dev_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --train_batch_size $batch_size --eval_batch_size $batch_size --learning_rate $lr --num_train_epochs $num_train_epochs 24 | 25 | train_file=$data_dir/$lang/cl/train_2.jsonl 26 | dev_file=$data_dir/$lang/cl/dev_2.jsonl 27 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 28 | output_dir=model/$lang/$name/task_2 29 | 30 | python run.py --do_train --do_eval --model_type roberta --model_name_or_path $pretrained_model --load_model_path $load_model_path --train_filename $train_file --dev_filename $dev_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --train_batch_size $batch_size --eval_batch_size $batch_size --learning_rate $lr --num_train_epochs $num_train_epochs 31 | 32 | train_file=$data_dir/$lang/cl/train_3.jsonl 33 | dev_file=$data_dir/$lang/cl/dev_3.jsonl 34 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 35 | output_dir=model/$lang/$name/task_3 36 | 37 | python run.py --do_train --do_eval --model_type roberta --model_name_or_path $pretrained_model --load_model_path $load_model_path --train_filename $train_file --dev_filename $dev_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --train_batch_size $batch_size --eval_batch_size $batch_size --learning_rate $lr --num_train_epochs $num_train_epochs 38 | 39 | train_file=$data_dir/$lang/cl/train_4.jsonl 40 | dev_file=$data_dir/$lang/cl/dev_4.jsonl 41 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 42 | output_dir=model/$lang/$name/task_4 43 | 44 | python run.py --do_train --do_eval --model_type roberta --model_name_or_path $pretrained_model --load_model_path $load_model_path --train_filename $train_file --dev_filename $dev_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --train_batch_size $batch_size --eval_batch_size $batch_size --learning_rate $lr --num_train_epochs $num_train_epochs 45 | 46 | #Generate 47 | beam_size=10 48 | batch_size=128 49 | 50 | output_dir=model/$lang/$name/task_0 51 | test_model=$output_dir/checkpoint-best-bleu/pytorch_model.bin #checkpoint for test 52 | test_file=$data_dir/$lang/cl/test_0.jsonl 53 | python run.py --do_test --model_type roberta --model_name_or_path /microsoft/codebert-base --load_model_path $test_model --dev_filename $test_file --test_filename $test_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --eval_batch_size $batch_size 54 | 55 | output_dir=model/$lang/$name/task_1 56 | test_model=$output_dir/checkpoint-best-bleu/pytorch_model.bin #checkpoint for test 57 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl 58 | python run.py --do_test --model_type roberta --model_name_or_path /microsoft/codebert-base --load_model_path $test_model --dev_filename $test_file --test_filename $test_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --eval_batch_size $batch_size 59 | 60 | output_dir=model/$lang/$name/task_2 61 | test_model=$output_dir/checkpoint-best-bleu/pytorch_model.bin #checkpoint for test 62 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl,$data_dir/$lang/cl/test_2.jsonl 63 | python run.py --do_test --model_type roberta --model_name_or_path /microsoft/codebert-base --load_model_path $test_model --dev_filename $test_file --test_filename $test_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --eval_batch_size $batch_size 64 | 65 | output_dir=model/$lang/$name/task_3 66 | test_model=$output_dir/checkpoint-best-bleu/pytorch_model.bin #checkpoint for test 67 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl,$data_dir/$lang/cl/test_2.jsonl,$data_dir/$lang/cl/test_3.jsonl 68 | python run.py --do_test --model_type roberta --model_name_or_path /microsoft/codebert-base --load_model_path $test_model --dev_filename $test_file --test_filename $test_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --eval_batch_size $batch_size 69 | 70 | output_dir=model/$lang/$name/task_4 71 | test_model=$output_dir/checkpoint-best-bleu/pytorch_model.bin #checkpoint for test 72 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl,$data_dir/$lang/cl/test_2.jsonl,$data_dir/$lang/cl/test_3.jsonl,$data_dir/$lang/cl/test_4.jsonl 73 | python run.py --do_test --model_type roberta --model_name_or_path /microsoft/codebert-base --load_model_path $test_model --dev_filename $test_file --test_filename $test_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --eval_batch_size $batch_size 74 | 75 | 76 | #Evaluate 77 | python evaluate.py --mode $name --lang $lang -------------------------------------------------------------------------------- /sum/CodeBERT/run_multitask.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=1 2 | lang=java #programming language 3 | lr=5e-5 4 | batch_size=32 5 | beam_size=10 6 | source_length=256 7 | target_length=128 8 | data_dir=/CodeSearchNet 9 | num_train_epochs=15 10 | name=multi_task 11 | 12 | output_dir=model/$lang/$name/task_4 13 | train_file=$data_dir/$lang/cl/train.jsonl 14 | dev_file=$data_dir/$lang/cl/dev.jsonl 15 | pretrained_model=/microsoft/codebert-base 16 | 17 | python run.py --do_train --do_eval --model_type roberta --model_name_or_path $pretrained_model --train_filename $train_file --dev_filename $dev_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --train_batch_size $batch_size --eval_batch_size $batch_size --learning_rate $lr --num_train_epochs $num_train_epochs 18 | 19 | output_dir=model/$lang/$name/task_3 20 | train_file=$data_dir/$lang/cl/train_0123.jsonl 21 | dev_file=$data_dir/$lang/cl/dev_0123.jsonl 22 | pretrained_model=/microsoft/codebert-base 23 | 24 | python run.py --do_train --do_eval --model_type roberta --model_name_or_path $pretrained_model --train_filename $train_file --dev_filename $dev_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --train_batch_size $batch_size --eval_batch_size $batch_size --learning_rate $lr --num_train_epochs $num_train_epochs 25 | 26 | output_dir=model/$lang/$name/task_2 27 | train_file=$data_dir/$lang/cl/train_012jsonl 28 | dev_file=$data_dir/$lang/cl/dev_012.jsonl 29 | pretrained_model=/microsoft/codebert-base 30 | 31 | python run.py --do_train --do_eval --model_type roberta --model_name_or_path $pretrained_model --train_filename $train_file --dev_filename $dev_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --train_batch_size $batch_size --eval_batch_size $batch_size --learning_rate $lr --num_train_epochs $num_train_epochs 32 | 33 | output_dir=model/$lang/$name/task_1 34 | train_file=$data_dir/$lang/cl/train_01.jsonl 35 | dev_file=$data_dir/$lang/cl/dev_01.jsonl 36 | pretrained_model=/microsoft/codebert-base 37 | 38 | python run.py --do_train --do_eval --model_type roberta --model_name_or_path $pretrained_model --train_filename $train_file --dev_filename $dev_file --output_dir $output_dir --max_source_length $source_length --max_target_length $target_length --beam_size $beam_size --train_batch_size $batch_size --eval_batch_size $batch_size --learning_rate $lr --num_train_epochs $num_train_epochs 39 | -------------------------------------------------------------------------------- /sum/CodeBERT/run_ours.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeBERT/run_ours.py -------------------------------------------------------------------------------- /sum/CodeT5/evall/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'wasi' 2 | 3 | from . import bleu 4 | from . import rouge 5 | from . import meteor 6 | -------------------------------------------------------------------------------- /sum/CodeT5/evall/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'wasi' 2 | 3 | from .bleu import * 4 | from .bleu_scorer import * 5 | from .nltk_bleu import * 6 | from .google_bleu import * 7 | -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/bleu.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/bleu.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/bleu.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/bleu.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/bleu.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/bleu.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/bleu_scorer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/bleu_scorer.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/bleu_scorer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/bleu_scorer.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/bleu_scorer.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/bleu_scorer.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/google_bleu.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/google_bleu.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/google_bleu.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/google_bleu.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/google_bleu.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/google_bleu.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/nltk_bleu.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/nltk_bleu.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/nltk_bleu.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/nltk_bleu.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/__pycache__/nltk_bleu.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/bleu/__pycache__/nltk_bleu.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/bleu.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # File Name : bleu.py 4 | # 5 | # Description : Wrapper for BLEU scorer. 6 | # 7 | # Creation Date : 06-01-2015 8 | # Last Modified : Thu 19 Mar 2015 09:13:28 PM PDT 9 | # Authors : Hao Fang and Tsung-Yi Lin 10 | 11 | from .bleu_scorer import BleuScorer 12 | 13 | 14 | class Bleu: 15 | def __init__(self, n=4): 16 | # default compute Blue score up to 4 17 | self._n = n 18 | self._hypo_for_image = {} 19 | self.ref_for_image = {} 20 | 21 | def compute_score(self, gts, res, verbose): 22 | assert (sorted(gts.keys()) == sorted(res.keys())) 23 | imgIds = list(gts.keys()) 24 | 25 | bleu_scorer = BleuScorer(n=self._n) 26 | for id in imgIds: 27 | hypo = res[id] 28 | ref = gts[id] 29 | 30 | # Sanity check. 31 | assert (type(hypo) is list) 32 | assert (len(hypo) == 1) 33 | assert (type(ref) is list) 34 | assert (len(ref) >= 1) 35 | 36 | bleu_scorer += (hypo[0], ref) 37 | 38 | # score, scores = bleu_scorer.compute_score(option='shortest') 39 | score, scores, bleu = bleu_scorer.compute_score(option='closest', verbose=verbose) 40 | # score, scores = bleu_scorer.compute_score(option='average', verbose=1) 41 | 42 | return score, scores, bleu 43 | 44 | def method(self): 45 | return "Bleu" 46 | -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/google_bleu.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Python implementation of BLEU and smooth-BLEU. 17 | This module provides a Python implementation of BLEU and smooth-BLEU. 18 | Smooth BLEU is computed following the method outlined in the paper: 19 | Chin-Yew Lin, Franz Josef Och. ORANGE: a method for evaluating automatic 20 | evaluation metrics for machine translation. COLING 2004. 21 | """ 22 | 23 | import collections 24 | import math 25 | 26 | 27 | def _get_ngrams(segment, max_order): 28 | """Extracts all n-grams upto a given maximum order from an input segment. 29 | Args: 30 | segment: text segment from which n-grams will be extracted. 31 | max_order: maximum length in tokens of the n-grams returned by this 32 | methods. 33 | Returns: 34 | The Counter containing all n-grams upto max_order in segment 35 | with a count of how many times each n-gram occurred. 36 | """ 37 | ngram_counts = collections.Counter() 38 | for order in range(1, max_order + 1): 39 | for i in range(0, len(segment) - order + 1): 40 | ngram = tuple(segment[i:i + order]) 41 | ngram_counts[ngram] += 1 42 | return ngram_counts 43 | 44 | 45 | def compute_bleu(reference_corpus, translation_corpus, max_order=4, 46 | smooth=False): 47 | """Computes BLEU score of translated segments against one or more references. 48 | Args: 49 | reference_corpus: list of lists of references for each translation. Each 50 | reference should be tokenized into a list of tokens. 51 | translation_corpus: list of translations to score. Each translation 52 | should be tokenized into a list of tokens. 53 | max_order: Maximum n-gram order to use when computing BLEU score. 54 | smooth: Whether or not to apply Lin et al. 2004 smoothing. 55 | Returns: 56 | 3-Tuple with the BLEU score, n-gram precisions, geometric mean of n-gram 57 | precisions and brevity penalty. 58 | """ 59 | matches_by_order = [0] * max_order 60 | possible_matches_by_order = [0] * max_order 61 | reference_length = 0 62 | translation_length = 0 63 | for (references, translation) in zip(reference_corpus, 64 | translation_corpus): 65 | reference_length += min(len(r) for r in references) 66 | translation_length += len(translation) 67 | 68 | merged_ref_ngram_counts = collections.Counter() 69 | for reference in references: 70 | merged_ref_ngram_counts |= _get_ngrams(reference, max_order) 71 | translation_ngram_counts = _get_ngrams(translation, max_order) 72 | overlap = translation_ngram_counts & merged_ref_ngram_counts 73 | for ngram in overlap: 74 | matches_by_order[len(ngram) - 1] += overlap[ngram] 75 | for order in range(1, max_order + 1): 76 | possible_matches = len(translation) - order + 1 77 | if possible_matches > 0: 78 | possible_matches_by_order[order - 1] += possible_matches 79 | 80 | precisions = [0] * max_order 81 | for i in range(0, max_order): 82 | if smooth: 83 | precisions[i] = ((matches_by_order[i] + 1.) / 84 | (possible_matches_by_order[i] + 1.)) 85 | else: 86 | if possible_matches_by_order[i] > 0: 87 | precisions[i] = (float(matches_by_order[i]) / 88 | possible_matches_by_order[i]) 89 | else: 90 | precisions[i] = 0.0 91 | 92 | if min(precisions) > 0: 93 | p_log_sum = sum((1. / max_order) * math.log(p) for p in precisions) 94 | geo_mean = math.exp(p_log_sum) 95 | else: 96 | geo_mean = 0 97 | 98 | ratio = float(translation_length) / reference_length 99 | 100 | if ratio > 1.0: 101 | bp = 1. 102 | else: 103 | bp = math.exp(1 - 1. / ratio) 104 | 105 | bleu = geo_mean * bp 106 | 107 | return (bleu, precisions, bp, ratio, translation_length, reference_length) 108 | 109 | 110 | def corpus_bleu(hypotheses, references): 111 | refs = [] 112 | hyps = [] 113 | count = 0 114 | total_score = 0.0 115 | 116 | assert (sorted(hypotheses.keys()) == sorted(references.keys())) 117 | Ids = list(hypotheses.keys()) 118 | ind_score = dict() 119 | 120 | for id in Ids: 121 | hyp = hypotheses[id][0].split() 122 | ref = [r.split() for r in references[id]] 123 | hyps.append(hyp) 124 | refs.append(ref) 125 | 126 | score = compute_bleu([ref], [hyp], smooth=True)[0] 127 | total_score += score 128 | count += 1 129 | ind_score[id] = score 130 | 131 | avg_score = total_score / count 132 | corpus_bleu = compute_bleu(refs, hyps, smooth=True)[0] 133 | return corpus_bleu, avg_score, ind_score 134 | -------------------------------------------------------------------------------- /sum/CodeT5/evall/bleu/nltk_bleu.py: -------------------------------------------------------------------------------- 1 | import nltk 2 | from nltk.translate.bleu_score import SmoothingFunction 3 | 4 | 5 | def nltk_sentence_bleu(hypothesis, reference, order=4): 6 | cc = SmoothingFunction() 7 | return nltk.translate.bleu([reference], hypothesis, smoothing_function=cc.method4) 8 | 9 | 10 | def nltk_corpus_bleu(hypotheses, references, order=4): 11 | refs = [] 12 | hyps = [] 13 | count = 0 14 | total_score = 0.0 15 | 16 | cc = SmoothingFunction() 17 | 18 | assert (sorted(hypotheses.keys()) == sorted(references.keys())) 19 | Ids = list(hypotheses.keys()) 20 | ind_score = dict() 21 | 22 | for id in Ids: 23 | hyp = hypotheses[id][0].split() 24 | ref = [r.split() for r in references[id]] 25 | hyps.append(hyp) 26 | refs.append(ref) 27 | 28 | score = nltk.translate.bleu(ref, hyp, smoothing_function=cc.method4) 29 | total_score += score 30 | count += 1 31 | ind_score[id] = score 32 | 33 | avg_score = total_score / count 34 | corpus_bleu = nltk.translate.bleu_score.corpus_bleu(refs, hyps) 35 | return corpus_bleu, avg_score, ind_score 36 | -------------------------------------------------------------------------------- /sum/CodeT5/evall/meteor/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'wasi' 2 | 3 | from .meteor import * 4 | -------------------------------------------------------------------------------- /sum/CodeT5/evall/meteor/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/meteor/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/meteor/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/meteor/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/meteor/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/meteor/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/meteor/__pycache__/meteor.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/meteor/__pycache__/meteor.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/meteor/__pycache__/meteor.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/meteor/__pycache__/meteor.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/meteor/__pycache__/meteor.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/meteor/__pycache__/meteor.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/meteor/data/paraphrase-en.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/meteor/data/paraphrase-en.gz -------------------------------------------------------------------------------- /sum/CodeT5/evall/meteor/meteor-1.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/meteor/meteor-1.5.jar -------------------------------------------------------------------------------- /sum/CodeT5/evall/meteor/meteor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Python wrapper for METEOR implementation, by Xinlei Chen 4 | # Acknowledge Michael Denkowski for the generous discussion and help 5 | from __future__ import division 6 | 7 | import atexit 8 | import logging 9 | import os 10 | import subprocess 11 | import sys 12 | import threading 13 | 14 | import psutil 15 | 16 | # Assumes meteor-1.5.jar is in the same directory as meteor.py. Change as needed. 17 | METEOR_JAR = 'meteor-1.5.jar' 18 | 19 | 20 | def enc(s): 21 | return s.encode('utf-8') 22 | 23 | 24 | def dec(s): 25 | return s.decode('utf-8') 26 | 27 | 28 | class Meteor: 29 | 30 | def __init__(self): 31 | # Used to guarantee thread safety 32 | self.lock = threading.Lock() 33 | 34 | mem = '2G' 35 | mem_available_G = psutil.virtual_memory().available / 1E9 36 | if mem_available_G < 2: 37 | logging.warning("There is less than 2GB of available memory.\n" 38 | "Will try with limiting Meteor to 1GB of memory but this might cause issues.\n" 39 | "If you have problems using Meteor, " 40 | "then you can try to lower the `mem` variable in meteor.py") 41 | mem = '1G' 42 | 43 | meteor_cmd = ['java', '-jar', '-Xmx{}'.format(mem), METEOR_JAR, 44 | '-', '-', '-stdio', '-l', 'en', '-norm'] 45 | env = os.environ.copy() 46 | env['LC_ALL'] = "C" 47 | self.meteor_p = subprocess.Popen(meteor_cmd, 48 | cwd=os.path.dirname(os.path.abspath(__file__)), 49 | env=env, 50 | stdin=subprocess.PIPE, 51 | stdout=subprocess.PIPE, 52 | stderr=subprocess.PIPE) 53 | 54 | atexit.register(self.close) 55 | 56 | def close(self): 57 | with self.lock: 58 | if self.meteor_p: 59 | self.meteor_p.kill() 60 | self.meteor_p.wait() 61 | self.meteor_p = None 62 | # if the user calls close() manually, remove the 63 | # reference from atexit so the object can be garbage-collected. 64 | if atexit is not None and atexit.unregister is not None: 65 | atexit.unregister(self.close) 66 | 67 | def compute_score(self, gts, res): 68 | assert (gts.keys() == res.keys()) 69 | imgIds = gts.keys() 70 | scores = [] 71 | 72 | eval_line = 'EVAL' 73 | with self.lock: 74 | for i in imgIds: 75 | assert (len(res[i]) == 1) 76 | stat = self._stat(res[i][0], gts[i]) 77 | eval_line += ' ||| {}'.format(stat) 78 | 79 | self.meteor_p.stdin.write(enc('{}\n'.format(eval_line))) 80 | self.meteor_p.stdin.flush() 81 | for i in range(0, len(imgIds)): 82 | v = self.meteor_p.stdout.readline() 83 | try: 84 | scores.append(float(dec(v.strip()))) 85 | except: 86 | sys.stderr.write("Error handling value: {}\n".format(v)) 87 | sys.stderr.write("Decoded value: {}\n".format(dec(v.strip()))) 88 | sys.stderr.write("eval_line: {}\n".format(eval_line)) 89 | # You can try uncommenting the next code line to show stderr from the Meteor JAR. 90 | # If the Meteor JAR is not writing to stderr, then the line will just hang. 91 | # sys.stderr.write("Error from Meteor:\n{}".format(self.meteor_p.stderr.read())) 92 | raise 93 | score = float(dec(self.meteor_p.stdout.readline()).strip()) 94 | 95 | return score, scores 96 | 97 | def method(self): 98 | return "METEOR" 99 | 100 | def _stat(self, hypothesis_str, reference_list): 101 | # SCORE ||| reference 1 words ||| reference n words ||| hypothesis words 102 | hypothesis_str = hypothesis_str.replace('|||', '').replace(' ', ' ') 103 | score_line = ' ||| '.join(('SCORE', ' ||| '.join(reference_list), hypothesis_str)) 104 | self.meteor_p.stdin.write(enc(score_line)) 105 | self.meteor_p.stdin.write(enc('\n')) 106 | self.meteor_p.stdin.flush() 107 | return dec(self.meteor_p.stdout.readline()).strip() 108 | 109 | def _score(self, hypothesis_str, reference_list): 110 | with self.lock: 111 | # SCORE ||| reference 1 words ||| reference n words ||| hypothesis words 112 | hypothesis_str = hypothesis_str.replace('|||', '').replace(' ', ' ') 113 | score_line = ' ||| '.join(('SCORE', ' ||| '.join(reference_list), hypothesis_str)) 114 | self.meteor_p.stdin.write(enc('{}\n'.format(score_line))) 115 | self.meteor_p.stdin.flush() 116 | stats = dec(self.meteor_p.stdout.readline()).strip() 117 | eval_line = 'EVAL ||| {}'.format(stats) 118 | # EVAL ||| stats 119 | self.meteor_p.stdin.write(enc('{}\n'.format(eval_line))) 120 | self.meteor_p.stdin.flush() 121 | score = float(dec(self.meteor_p.stdout.readline()).strip()) 122 | # bug fix: there are two values returned by the jar file, one average, and one all, so do it twice 123 | # thanks for Andrej for pointing this out 124 | score = float(dec(self.meteor_p.stdout.readline()).strip()) 125 | return score 126 | 127 | def __del__(self): 128 | self.close() 129 | -------------------------------------------------------------------------------- /sum/CodeT5/evall/rouge/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'wasi' 2 | 3 | from .rouge import * 4 | -------------------------------------------------------------------------------- /sum/CodeT5/evall/rouge/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/rouge/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/rouge/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/rouge/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/rouge/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/rouge/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/rouge/__pycache__/rouge.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/rouge/__pycache__/rouge.cpython-36.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/rouge/__pycache__/rouge.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/rouge/__pycache__/rouge.cpython-37.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/rouge/__pycache__/rouge.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ReliableCoding/REPEAT/2c933c25471371cd2532c066abbb3a7d14fc9ea0/sum/CodeT5/evall/rouge/__pycache__/rouge.cpython-39.pyc -------------------------------------------------------------------------------- /sum/CodeT5/evall/rouge/rouge.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # File Name : rouge.py 4 | # 5 | # Description : Computes ROUGE-L metric as described by Lin and Hovey (2004) 6 | # 7 | # Creation Date : 2015-01-07 06:03 8 | # Author : Ramakrishna Vedantam 9 | 10 | import numpy as np 11 | 12 | 13 | def my_lcs(string, sub): 14 | """ 15 | Calculates longest common subsequence for a pair of tokenized strings 16 | :param string : list of str : tokens from a string split using whitespace 17 | :param sub : list of str : shorter string, also split using whitespace 18 | :returns: length (list of int): length of the longest common subsequence between the two strings 19 | Note: my_lcs only gives length of the longest common subsequence, not the actual LCS 20 | """ 21 | if len(string) < len(sub): 22 | sub, string = string, sub 23 | 24 | lengths = [[0 for i in range(0, len(sub) + 1)] for j in range(0, len(string) + 1)] 25 | 26 | for j in range(1, len(sub) + 1): 27 | for i in range(1, len(string) + 1): 28 | if string[i - 1] == sub[j - 1]: 29 | lengths[i][j] = lengths[i - 1][j - 1] + 1 30 | else: 31 | lengths[i][j] = max(lengths[i - 1][j], lengths[i][j - 1]) 32 | 33 | return lengths[len(string)][len(sub)] 34 | 35 | 36 | class Rouge(): 37 | ''' 38 | Class for computing ROUGE-L score for a set of candidate sentences for the MS COCO test set 39 | ''' 40 | 41 | def __init__(self): 42 | # vrama91: updated the value below based on discussion with Hovey 43 | self.beta = 1.2 44 | 45 | def calc_score(self, candidate, refs): 46 | """ 47 | Compute ROUGE-L score given one candidate and references for an image 48 | :param candidate: str : candidate sentence to be evaluated 49 | :param refs: list of str : COCO reference sentences for the particular image to be evaluated 50 | :returns score: int (ROUGE-L score for the candidate evaluated against references) 51 | """ 52 | assert (len(candidate) == 1) 53 | assert (len(refs) > 0) 54 | prec = [] 55 | rec = [] 56 | 57 | # split into tokens 58 | token_c = candidate[0].split(" ") 59 | 60 | for reference in refs: 61 | # split into tokens 62 | token_r = reference.split(" ") 63 | # compute the longest common subsequence 64 | lcs = my_lcs(token_r, token_c) 65 | prec.append(lcs / float(len(token_c))) 66 | rec.append(lcs / float(len(token_r))) 67 | 68 | prec_max = max(prec) 69 | rec_max = max(rec) 70 | 71 | if prec_max != 0 and rec_max != 0: 72 | score = ((1 + self.beta ** 2) * prec_max * rec_max) / float(rec_max + self.beta ** 2 * prec_max) 73 | else: 74 | score = 0.0 75 | return score 76 | 77 | def compute_score(self, gts, res): 78 | """ 79 | Computes Rouge-L score given a set of reference and candidate sentences for the dataset 80 | Invoked by evaluate_captions.py 81 | :param gts: dict : candidate / test sentences with "image name" key and "tokenized sentences" as values 82 | :param res: dict : reference MS-COCO sentences with "image name" key and "tokenized sentences" as values 83 | :returns: average_score: float (mean ROUGE-L score computed by averaging scores for all the images) 84 | """ 85 | assert (sorted(gts.keys()) == sorted(res.keys())) 86 | imgIds = list(gts.keys()) 87 | 88 | score = dict() 89 | for id in imgIds: 90 | hypo = res[id] 91 | ref = gts[id] 92 | 93 | # Sanity check. 94 | assert (type(hypo) is list) 95 | assert (len(hypo) == 1) 96 | assert (type(ref) is list) 97 | assert (len(ref) > 0) 98 | 99 | score[id] = self.calc_score(hypo, ref) 100 | 101 | average_score = np.mean(np.array(list(score.values()))) 102 | return average_score, score 103 | 104 | def method(self): 105 | return "Rouge" 106 | -------------------------------------------------------------------------------- /sum/CodeT5/evaluate.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import pandas as pd 3 | import json 4 | from collections import OrderedDict, Counter 5 | import os 6 | import argparse 7 | 8 | from evall.bleu import corpus_bleu 9 | from evall.rouge import Rouge 10 | from evall.meteor import Meteor 11 | 12 | def eval_accuracies(hypotheses, references, sources=None, 13 | filename=None, print_copy_info=False, mode='dev'): 14 | _, bleu, ind_bleu = corpus_bleu(hypotheses, references) 15 | 16 | # Compute ROUGE scores 17 | rouge_calculator = Rouge() 18 | rouge_l, ind_rouge = rouge_calculator.compute_score(references, hypotheses) 19 | meteor_calculator = Meteor() 20 | meteor, _ = meteor_calculator.compute_score(references, hypotheses) 21 | 22 | return bleu * 100, rouge_l * 100, meteor * 100 23 | 24 | def read_file(base_dir, task_id): 25 | f = csv.reader(open(os.path.join(base_dir, 'test_'+task_id+'.gold'),'r')) 26 | ref = {} 27 | count = 0 28 | for i in f: 29 | ref[count] = [i[0].split('\t')[1].strip()] 30 | count+=1 31 | f = csv.reader(open(os.path.join(base_dir, 'test_'+task_id+'.output'),'r')) 32 | hypo = {} 33 | count = 0 34 | for i in f: 35 | hypo[count] = [i[0].split('\t')[1].strip()] 36 | if len(hypo[count][0].split()) == 0: 37 | hypo[count] = ['Get'] 38 | count+=1 39 | return hypo,ref 40 | 41 | parser = argparse.ArgumentParser() 42 | parser.add_argument("--mode", default=None, type=str, required=True, help="mode") 43 | parser.add_argument("--lang", default=None, type=str, required=True, help="lang") 44 | args = parser.parse_args() 45 | 46 | bleu_sum=0 47 | meteor_sum=0 48 | rouge_sum=0 49 | first_bleu=0 50 | first_meteor=0 51 | first_rouge=0 52 | for i in range(5): 53 | for j in range(i+1): 54 | hypo,ref = read_file('model/'+args.lang+'/'+args.mode+'/task_'+str(i)+'/', str(j)) 55 | bleu,rouge,meteor = eval_accuracies(hypo,ref) 56 | #print('Model:',i,' Task:',j) 57 | #print('bleu:',bleu,' meteor:',meteor,' rouge:',rouge) 58 | print(round(bleu,2), end=' ') 59 | if j==0: 60 | first_bleu+=bleu 61 | first_meteor+=meteor 62 | first_rouge+=rouge 63 | bleu_sum+=bleu/(i+1) 64 | meteor_sum+=meteor/(i+1) 65 | rouge_sum+=rouge/(i+1) 66 | print('') 67 | 68 | #print('first_bleu:',first_bleu/5,' first_meteor:',first_meteor/5,' first_rouge:',first_rouge/5) 69 | #print('avg_bleu:',bleu_sum/5,' avg_meteor:',meteor_sum/5,' avg_rouge:',rouge_sum/5) 70 | print('Avg_BLEU:',round(bleu_sum/5,2)) 71 | print('Avg_METEOR:',round(meteor_sum/5,2)) 72 | print('Avg_ROUGE:',round(rouge_sum/5,2)) -------------------------------------------------------------------------------- /sum/CodeT5/ewc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | from torch.autograd import Variable 4 | import copy 5 | #from model.masked_cross_entropy import * 6 | 7 | def variable(t: torch.Tensor, use_cuda=True, **kwargs): 8 | if torch.cuda.is_available() and use_cuda: 9 | t = t.cuda() 10 | return Variable(t, **kwargs) 11 | 12 | class EWC(object): 13 | def __init__(self, model, dataset, device, length, tokenizer): 14 | 15 | self.model = model 16 | # the data we use to compute fisher information of ewc (old_exemplars) 17 | self.dataset = dataset 18 | self.device = device 19 | self.length = length 20 | 21 | self.params = {n: p for n, p in self.model.named_parameters() if p.requires_grad} 22 | self._means = {} # previous parameters 23 | self._precision_matrices = self._diag_fisher(tokenizer) # approximated diagnal fisher information matrix 24 | 25 | for n, p in copy.deepcopy(self.params).items(): 26 | self._means[n] = variable(p.data) 27 | 28 | def _diag_fisher(self, tokenizer): 29 | 30 | self.model.train() 31 | precision_matrices = {} 32 | for n, p in copy.deepcopy(self.params).items(): 33 | p.data.zero_() 34 | precision_matrices[n] = variable(p.data) 35 | 36 | for batch in self.dataset: 37 | self.model.zero_grad() 38 | 39 | batch = tuple(t.to(self.device) for t in batch) 40 | source_ids, source_mask, target_ids, target_mask = batch 41 | labels = [ 42 | [(label if label != tokenizer.pad_token_id else -100) for label in labels_example] for 43 | labels_example in target_ids 44 | ] 45 | labels = torch.tensor(labels).to(self.device) 46 | out = self.model(input_ids=source_ids, attention_mask=source_mask, labels=labels) 47 | loss = out.loss 48 | 49 | loss.backward() 50 | 51 | for n, p in self.model.named_parameters(): 52 | if p.grad is None: 53 | continue 54 | precision_matrices[n].data += p.grad.data ** 2 / self.length 55 | 56 | precision_matrices = {n: p for n, p in precision_matrices.items()} 57 | self.model.zero_grad() 58 | return precision_matrices 59 | 60 | def penalty(self, model: nn.Module): 61 | loss = 0 62 | for n, p in model.named_parameters(): 63 | _loss = self._precision_matrices[n] * (p - self._means[n]) ** 2 64 | loss += _loss.sum() 65 | 66 | return loss 67 | -------------------------------------------------------------------------------- /sum/CodeT5/my_lib.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | 4 | #from openprompt.data_utils import InputExample 5 | 6 | 7 | def get_elapse_time(t0): 8 | elapse_time = time.time() - t0 9 | if elapse_time > 3600: 10 | hour = int(elapse_time // 3600) 11 | minute = int((elapse_time % 3600) // 60) 12 | return "{}h{}m".format(hour, minute) 13 | else: 14 | minute = int((elapse_time % 3600) // 60) 15 | return "{}m".format(minute) 16 | 17 | 18 | class Example(object): 19 | """A single training/test example.""" 20 | 21 | def __init__(self, 22 | idx, 23 | source, 24 | target, 25 | original_source, 26 | original_target 27 | ): 28 | self.idx = idx 29 | self.source = source 30 | self.target = target 31 | self.origin_source = original_source 32 | self.origin_target = original_target 33 | self.code='' 34 | 35 | 36 | def read_examples(filename, args): 37 | """Read examples from filename.""" 38 | if args.add_task_prefix: 39 | task_prefix = f"Generating comments for {args.lang}: " 40 | else: 41 | task_prefix = "" 42 | 43 | if args.add_lang_ids: 44 | language_prefix = " " 45 | else: 46 | language_prefix = "" 47 | 48 | examples = [] 49 | with open(filename, encoding="utf-8") as f: 50 | for idx, line in enumerate(f): 51 | line = line.strip() 52 | js = json.loads(line) 53 | if 'idx' not in js: 54 | js['idx'] = idx 55 | code = ' '.join(js['code_tokens']).replace('\n', ' ') 56 | code = ' '.join(code.strip().split()) 57 | 58 | nl = ' '.join(js['docstring_tokens']).replace('\n', '') 59 | nl = ' '.join(nl.strip().split()) 60 | examples.append( 61 | Example( 62 | idx=idx, 63 | source=task_prefix + code, 64 | target=language_prefix + nl, 65 | original_source=js['code_tokens'], 66 | original_target=js['docstring_tokens'], 67 | ) 68 | ) 69 | 70 | return examples 71 | 72 | 73 | class InputFeatures(object): 74 | """A single training/test features for an example.""" 75 | 76 | def __init__(self, 77 | example_id, 78 | source_ids, 79 | target_ids, 80 | source_mask, 81 | target_mask, 82 | 83 | ): 84 | self.example_id = example_id 85 | self.source_ids = source_ids 86 | self.target_ids = target_ids 87 | self.source_mask = source_mask 88 | self.target_mask = target_mask 89 | 90 | 91 | def convert_examples_to_features(examples, tokenizer, args, stage=None): 92 | # collect texts 93 | codes = [] 94 | target_nl = [] 95 | for example_id, example in enumerate(examples): 96 | codes.append(example.source) 97 | 98 | if stage == "test": 99 | target_nl.append("None") 100 | else: 101 | target_nl.append(example.target) 102 | 103 | # begin tokenizing 104 | encoded_codes = tokenizer( 105 | codes, padding=True, verbose=False, add_special_tokens=True, 106 | truncation=True, max_length=args.max_source_length, return_tensors='pt') 107 | 108 | encoded_targets = tokenizer( 109 | target_nl, padding=True, verbose=False, add_special_tokens=True, 110 | truncation=True, max_length=args.max_source_length, return_tensors='pt') 111 | 112 | return {'source_ids':encoded_codes['input_ids'], 'target_ids':encoded_targets['input_ids'], 113 | 'source_mask':encoded_codes['attention_mask'], 'target_mask':encoded_targets['attention_mask']} 114 | 115 | 116 | def read_prompt_examples(filename): 117 | """Read examples from filename.""" 118 | examples = [] 119 | with open(filename, encoding="utf-8") as f: 120 | for idx, line in enumerate(f): 121 | line = line.strip() 122 | js = json.loads(line) 123 | if 'idx' not in js: 124 | js['idx'] = idx 125 | code = ' '.join(js['code_tokens']).replace('\n', ' ') 126 | code = ' '.join(code.strip().split()) 127 | nl = ' '.join(js['docstring_tokens']).replace('\n', '') 128 | nl = ' '.join(nl.strip().split()) 129 | # code = js['code'].replace('\n', ' ').strip() 130 | # nl = js['docstring'].replace('\n', ' ').strip() 131 | examples.append( 132 | InputExample( 133 | guid=idx, 134 | text_a= code, 135 | tgt_text= nl, 136 | ) 137 | ) 138 | 139 | return examples -------------------------------------------------------------------------------- /sum/CodeT5/run_finetune.sh: -------------------------------------------------------------------------------- 1 | gpu=0 2 | lang=java 3 | name=finetune 4 | pretrained_model=/Salesforce/codet5-base 5 | data_dir=/CodeSearchNet 6 | num_train_epochs=15 7 | batch_size=32 8 | beam_size=10 9 | 10 | #train 11 | train_file=$data_dir/$lang/cl/train_0.jsonl 12 | dev_file=$data_dir/$lang/cl/dev_0.jsonl 13 | output_dir=model/$lang/$name/task_0 14 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --train_file $train_file --dev_file $dev_file \ 15 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 16 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 17 | --log_name=./log/$lang.log 18 | 19 | train_file=$data_dir/$lang/cl/train_1.jsonl 20 | dev_file=$data_dir/$lang/cl/dev_1.jsonl 21 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 22 | output_dir=model/$lang/$name/task_1 23 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --train_file $train_file --dev_file $dev_file \ 24 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 25 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 26 | --log_name=./log/$lang.log 27 | 28 | train_file=$data_dir/$lang/cl/train_2.jsonl 29 | dev_file=$data_dir/$lang/cl/dev_2.jsonl 30 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 31 | output_dir=model/$lang/$name/task_2 32 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --train_file $train_file --dev_file $dev_file \ 33 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 34 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 35 | --log_name=./log/$lang.log 36 | 37 | train_file=$data_dir/$lang/cl/train_3.jsonl 38 | dev_file=$data_dir/$lang/cl/dev_3.jsonl 39 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 40 | output_dir=model/$lang/$name/task_3 41 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --train_file $train_file --dev_file $dev_file \ 42 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 43 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 44 | --log_name=./log/$lang.log 45 | 46 | train_file=$data_dir/$lang/cl/train_4.jsonl 47 | dev_file=$data_dir/$lang/cl/dev_4.jsonl 48 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 49 | output_dir=model/$lang/$name/task_4 50 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --train_file $train_file --dev_file $dev_file \ 51 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 52 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 53 | --log_name=./log/$lang.log 54 | 55 | 56 | #generate 57 | beam_size=10 58 | batch_size=128 59 | 60 | output_dir=model/$lang/$name/task_0 61 | test_model=$output_dir/checkpoint-best-bleu/pytorch_model.bin 62 | test_file=$data_dir/$lang/cl/test_0.jsonl 63 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 64 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model\ 65 | --eval_batch_size $batch_size --beam_size $beam_size \ 66 | --log_name=./log/$lang.log 67 | 68 | output_dir=model/$lang/$name/task_1 69 | test_model=$output_dir/checkpoint-best-bleu/pytorch_model.bin 70 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl 71 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 72 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 73 | --eval_batch_size $batch_size --beam_size $beam_size \ 74 | --log_name=./log/$lang.log 75 | 76 | output_dir=model/$lang/$name/task_2 77 | test_model=$output_dir/checkpoint-best-bleu/pytorch_model.bin 78 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl,$data_dir/$lang/cl/test_2.jsonl 79 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 80 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 81 | --eval_batch_size $batch_size --beam_size $beam_size \ 82 | --log_name=./log/$lang.log 83 | 84 | output_dir=model/$lang/$name/task_3 85 | test_model=$output_dir/checkpoint-best-bleu/pytorch_model.bin 86 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl,$data_dir/$lang/cl/test_2.jsonl,$data_dir/$lang/cl/test_3.jsonl 87 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 88 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 89 | --eval_batch_size $batch_size --beam_size $beam_size \ 90 | --log_name=./log/$lang.log 91 | 92 | output_dir=model/$lang/$name/task_4 93 | test_model=$output_dir/checkpoint-best-bleu/pytorch_model.bin 94 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl,$data_dir/$lang/cl/test_2.jsonl,$data_dir/$lang/cl/test_3.jsonl,$data_dir/$lang/cl/test_4.jsonl 95 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 96 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 97 | --eval_batch_size $batch_size --beam_size $beam_size \ 98 | --log_name=./log/$lang.log 99 | 100 | 101 | #evaluate 102 | python evaluate.py --mode $name --lang $lang -------------------------------------------------------------------------------- /sum/CodeT5/run_multitask.sh: -------------------------------------------------------------------------------- 1 | gpu=1 2 | lang=java 3 | name=multi_task 4 | pretrained_model=/Salesforce/codet5-base 5 | data_dir=/CodeSearchNet 6 | num_train_epochs=15 7 | batch_size=32 8 | beam_size=10 9 | 10 | #train 11 | train_file=$data_dir/$lang/cl/train.jsonl 12 | dev_file=$data_dir/$lang/cl/dev.jsonl 13 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 14 | output_dir=model/$lang/$name/task_4 15 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --train_file $train_file --dev_file $dev_file \ 16 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 17 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 18 | --log_name=./log/$lang.log 19 | 20 | train_file=$data_dir/$lang/cl/train_01.jsonl 21 | dev_file=$data_dir/$lang/cl/dev_01.jsonl 22 | output_dir=model/$lang/$name/task_1 23 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --train_file $train_file --dev_file $dev_file \ 24 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 25 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 26 | --log_name=./log/$lang.log 27 | 28 | train_file=$data_dir/$lang/cl/train_012.jsonl 29 | dev_file=$data_dir/$lang/cl/dev_012.jsonl 30 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 31 | output_dir=model/$lang/$name/task_2 32 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --train_file $train_file --dev_file $dev_file \ 33 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 34 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 35 | --log_name=./log/$lang.log 36 | 37 | train_file=$data_dir/$lang/cl/train_0123.jsonl 38 | dev_file=$data_dir/$lang/cl/dev_0123.jsonl 39 | load_model_path=$output_dir/checkpoint-best-bleu/pytorch_model.bin 40 | output_dir=model/$lang/$name/task_3 41 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --train_file $train_file --dev_file $dev_file \ 42 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 43 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 44 | --log_name=./log/$lang.log -------------------------------------------------------------------------------- /svd/CodeBERT/ewc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | from torch.autograd import Variable 4 | from torch.nn import CrossEntropyLoss, MSELoss 5 | import copy 6 | #from model.masked_cross_entropy import * 7 | def variable(t: torch.Tensor, use_cuda=True, **kwargs): 8 | if torch.cuda.is_available() and use_cuda: 9 | t = t.cuda() 10 | return Variable(t, **kwargs) 11 | 12 | class EWC(object): 13 | def __init__(self, model, dataset, device, length): 14 | 15 | self.model = model 16 | # the data we use to compute fisher information of ewc (old_exemplars) 17 | self.dataset = dataset 18 | self.device = device 19 | self.length = length 20 | 21 | self.params = {n: p for n, p in self.model.named_parameters() if p.requires_grad} 22 | self._means = {} # previous parameters 23 | self._precision_matrices = self._diag_fisher() # approximated diagnal fisher information matrix 24 | 25 | for n, p in copy.deepcopy(self.params).items(): 26 | self._means[n] = variable(p.data) 27 | 28 | def _diag_fisher(self): 29 | 30 | self.model.train() 31 | precision_matrices = {} 32 | for n, p in copy.deepcopy(self.params).items(): 33 | p.data.zero_() 34 | precision_matrices[n] = variable(p.data) 35 | 36 | for step,batch in enumerate(self.dataset): 37 | self.model.zero_grad() 38 | 39 | inputs = batch[0].to(self.device) 40 | labels=batch[1].to(self.device) 41 | self.model.train() 42 | loss,logits = self.model(inputs,labels) 43 | 44 | loss.backward() 45 | 46 | for n, p in self.model.named_parameters(): 47 | if p.grad is None: 48 | continue 49 | precision_matrices[n].data += p.grad.data ** 2 / self.length 50 | 51 | precision_matrices = {n: p for n, p in precision_matrices.items()} 52 | self.model.zero_grad() 53 | return precision_matrices 54 | 55 | def penalty(self, model: nn.Module): 56 | loss = 0 57 | for n, p in model.named_parameters(): 58 | _loss = self._precision_matrices[n] * (p - self._means[n]) ** 2 59 | loss += _loss.sum() 60 | 61 | return loss 62 | -------------------------------------------------------------------------------- /svd/CodeBERT/model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch 4 | from torch.autograd import Variable 5 | import copy 6 | from torch.nn import CrossEntropyLoss, MSELoss 7 | 8 | 9 | 10 | class Model(nn.Module): 11 | def __init__(self, encoder,config,tokenizer,args): 12 | super(Model, self).__init__() 13 | self.encoder = encoder 14 | self.config=config 15 | self.tokenizer=tokenizer 16 | self.args=args 17 | self.weight=1 18 | 19 | 20 | def forward(self, input_ids=None,labels=None,relay=None): 21 | outputs=self.encoder(input_ids,attention_mask=input_ids.ne(1))[0] 22 | logits=outputs 23 | prob=torch.sigmoid(logits) 24 | if relay is True: 25 | abels=labels.float() 26 | loss=torch.log(prob[:,0]+1e-10)*labels+torch.log((1-prob)[:,0]+1e-10)*(1-labels) 27 | return -loss,prob 28 | if labels is not None: 29 | labels=labels.float() 30 | loss=torch.log(prob[:,0]+1e-10)*labels+torch.log((1-prob)[:,0]+1e-10)*(1-labels) 31 | loss=-loss.mean() 32 | return loss,prob 33 | else: 34 | return prob 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /svd/CodeBERT/run_emr.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=0 2 | name=emr 3 | train_replay_size=1509 4 | train_examplar_path=./saved_models/$name/train_examplar.jsonl 5 | dev_examplar_path=./saved_models/$name/dev_examplar.jsonl 6 | pretrained_model=/microsoft/codebert-base 7 | 8 | #train 9 | train_data_file=/CodeSearchNet/defect/train_0.jsonl 10 | eval_data_file=/CodeSearchNet/defect/dev_0.jsonl 11 | load_model_path=./saved_models/finetune/task_0/checkpoint-best-acc/model.bin 12 | output_dir=./saved_models/$name/task_0 13 | python run_cl.py \ 14 | --output_dir=$output_dir \ 15 | --model_type=roberta \ 16 | --tokenizer_name=$pretrained_model \ 17 | --model_name_or_path=$pretrained_model \ 18 | --load_model_path $load_model_path\ 19 | --do_train \ 20 | --train_data_file=$train_data_file \ 21 | --eval_data_file=$eval_data_file \ 22 | --mode $name \ 23 | --task_id 0 \ 24 | --train_replay_size=$train_replay_size \ 25 | --train_examplar_path=$train_examplar_path \ 26 | --eval_examplar_path=$dev_examplar_path \ 27 | --epoch 10 \ 28 | --block_size 400 \ 29 | --train_batch_size 16 \ 30 | --eval_batch_size 64 \ 31 | --learning_rate 2e-5 \ 32 | --max_grad_norm 1.0 \ 33 | --evaluate_during_training \ 34 | --seed 123456 2>&1 | tee train.log 35 | train_data_file=/CodeSearchNet/defect/train_1.jsonl 36 | eval_data_file=/CodeSearchNet/defect/dev_1.jsonl 37 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 38 | output_dir=./saved_models/$name/task_1 39 | python run_cl.py \ 40 | --output_dir=$output_dir \ 41 | --model_type=roberta \ 42 | --tokenizer_name=$pretrained_model \ 43 | --model_name_or_path=$pretrained_model \ 44 | --load_model_path $load_model_path\ 45 | --do_train \ 46 | --train_data_file=$train_data_file \ 47 | --eval_data_file=$eval_data_file \ 48 | --mode $name \ 49 | --task_id 1 \ 50 | --train_replay_size=$train_replay_size \ 51 | --train_examplar_path=$train_examplar_path \ 52 | --eval_examplar_path=$dev_examplar_path \ 53 | --epoch 10 \ 54 | --block_size 400 \ 55 | --train_batch_size 16 \ 56 | --eval_batch_size 64 \ 57 | --learning_rate 2e-5 \ 58 | --max_grad_norm 1.0 \ 59 | --evaluate_during_training \ 60 | --seed 123456 2>&1 | tee train.log 61 | train_data_file=/CodeSearchNet/defect/train_2.jsonl 62 | eval_data_file=/CodeSearchNet/defect/dev_2.jsonl 63 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 64 | output_dir=./saved_models/$name/task_2 65 | python run_cl.py \ 66 | --output_dir=$output_dir \ 67 | --model_type=roberta \ 68 | --tokenizer_name=$pretrained_model \ 69 | --model_name_or_path=$pretrained_model \ 70 | --load_model_path $load_model_path\ 71 | --do_train \ 72 | --train_data_file=$train_data_file \ 73 | --eval_data_file=$eval_data_file \ 74 | --mode $name \ 75 | --task_id 2 \ 76 | --train_replay_size=$train_replay_size \ 77 | --train_examplar_path=$train_examplar_path \ 78 | --eval_examplar_path=$dev_examplar_path \ 79 | --epoch 10 \ 80 | --block_size 400 \ 81 | --train_batch_size 16 \ 82 | --eval_batch_size 64 \ 83 | --learning_rate 2e-5 \ 84 | --max_grad_norm 1.0 \ 85 | --evaluate_during_training \ 86 | --seed 123456 2>&1 | tee train.log 87 | train_data_file=/CodeSearchNet/defect/train_3.jsonl 88 | eval_data_file=/CodeSearchNet/defect/dev_3.jsonl 89 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 90 | output_dir=./saved_models/$name/task_3 91 | python run_cl.py \ 92 | --output_dir=$output_dir \ 93 | --model_type=roberta \ 94 | --tokenizer_name=$pretrained_model \ 95 | --model_name_or_path=$pretrained_model \ 96 | --load_model_path $load_model_path\ 97 | --do_train \ 98 | --train_data_file=$train_data_file \ 99 | --eval_data_file=$eval_data_file \ 100 | --mode $name \ 101 | --task_id 3 \ 102 | --train_replay_size=$train_replay_size \ 103 | --train_examplar_path=$train_examplar_path \ 104 | --eval_examplar_path=$dev_examplar_path \ 105 | --epoch 10 \ 106 | --block_size 400 \ 107 | --train_batch_size 16 \ 108 | --eval_batch_size 64 \ 109 | --learning_rate 2e-5 \ 110 | --max_grad_norm 1.0 \ 111 | --evaluate_during_training \ 112 | --seed 123456 2>&1 | tee train.log 113 | train_data_file=/CodeSearchNet/defect/train_4.jsonl 114 | eval_data_file=/CodeSearchNet/defect/dev_4.jsonl 115 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 116 | output_dir=./saved_models/$name/task_4 117 | python run_cl.py \ 118 | --output_dir=$output_dir \ 119 | --model_type=roberta \ 120 | --tokenizer_name=$pretrained_model \ 121 | --model_name_or_path=$pretrained_model \ 122 | --load_model_path $load_model_path\ 123 | --do_train \ 124 | --train_data_file=$train_data_file \ 125 | --eval_data_file=$eval_data_file \ 126 | --mode $name \ 127 | --task_id 4 \ 128 | --train_replay_size=$train_replay_size \ 129 | --train_examplar_path=$train_examplar_path \ 130 | --eval_examplar_path=$dev_examplar_path \ 131 | --epoch 10 \ 132 | --block_size 400 \ 133 | --train_batch_size 16 \ 134 | --eval_batch_size 64 \ 135 | --learning_rate 2e-5 \ 136 | --max_grad_norm 1.0 \ 137 | --evaluate_during_training \ 138 | --seed 123456 2>&1 | tee train.log 139 | 140 | 141 | #generate 142 | cp -r ./saved_models/finetune/task_0 ./saved_models/$name 143 | 144 | data_dir=/CodeSearchNet/defect 145 | test_data_file=$data_dir/test_0.jsonl,$data_dir/test_1.jsonl,$data_dir/test_2.jsonl,$data_dir/test_3.jsonl,$data_dir/test_4.jsonl 146 | output_dir=./saved_models/$name/task_0,./saved_models/$name/task_1,./saved_models/$name/task_2,./saved_models/$name/task_3,./saved_models/$name/task_4 147 | python run.py \ 148 | --output_dir=$output_dir \ 149 | --model_type=roberta \ 150 | --tokenizer_name=$pretrained_model \ 151 | --model_name_or_path=$pretrained_model \ 152 | --do_test \ 153 | --train_data_file=$test_data_file \ 154 | --test_data_file=$test_data_file \ 155 | --epoch 5 \ 156 | --block_size 400 \ 157 | --train_batch_size 32 \ 158 | --eval_batch_size 64 \ 159 | --learning_rate 2e-5 \ 160 | --max_grad_norm 1.0 \ 161 | --evaluate_during_training \ 162 | --seed 123456 2>&1 | tee test.log 163 | -------------------------------------------------------------------------------- /svd/CodeBERT/run_ewc.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=0 2 | name=ewc 3 | train_replay_size=1509 4 | weight=2000 5 | train_examplar_path=./saved_models/$name/train_examplar.jsonl 6 | dev_examplar_path=./saved_models/$name/dev_examplar.jsonl 7 | pretrained_model=/microsoft/codebert-base 8 | 9 | #train 10 | train_data_file=/CodeSearchNet/defect/train_0.jsonl 11 | eval_data_file=/CodeSearchNet/defect/dev_0.jsonl 12 | load_model_path=./saved_models/finetune/task_0/checkpoint-best-acc/model.bin 13 | output_dir=./saved_models/$name/task_0 14 | python run_cl.py \ 15 | --output_dir=$output_dir \ 16 | --model_type=roberta \ 17 | --tokenizer_name=$pretrained_model \ 18 | --model_name_or_path=$pretrained_model \ 19 | --load_model_path $load_model_path\ 20 | --do_train \ 21 | --train_data_file=$train_data_file \ 22 | --eval_data_file=$eval_data_file \ 23 | --mode $name \ 24 | --task_id 0 \ 25 | --ewc_weight $weight \ 26 | --train_replay_size=$train_replay_size \ 27 | --train_examplar_path=$train_examplar_path \ 28 | --eval_examplar_path=$dev_examplar_path \ 29 | --epoch 10 \ 30 | --block_size 400 \ 31 | --train_batch_size 16 \ 32 | --eval_batch_size 64 \ 33 | --learning_rate 2e-5 \ 34 | --max_grad_norm 1.0 \ 35 | --evaluate_during_training \ 36 | --seed 123456 2>&1 | tee train.log 37 | train_data_file=/CodeSearchNet/defect/train_1.jsonl 38 | eval_data_file=/CodeSearchNet/defect/dev_1.jsonl 39 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 40 | output_dir=./saved_models/$name/task_1 41 | python run_cl.py \ 42 | --output_dir=$output_dir \ 43 | --model_type=roberta \ 44 | --tokenizer_name=$pretrained_model \ 45 | --model_name_or_path=$pretrained_model \ 46 | --load_model_path $load_model_path\ 47 | --do_train \ 48 | --train_data_file=$train_data_file \ 49 | --eval_data_file=$eval_data_file \ 50 | --mode $name \ 51 | --task_id 1 \ 52 | --ewc_weight $weight \ 53 | --train_replay_size=$train_replay_size \ 54 | --train_examplar_path=$train_examplar_path \ 55 | --eval_examplar_path=$dev_examplar_path \ 56 | --epoch 10 \ 57 | --block_size 400 \ 58 | --train_batch_size 16 \ 59 | --eval_batch_size 64 \ 60 | --learning_rate 2e-5 \ 61 | --max_grad_norm 1.0 \ 62 | --evaluate_during_training \ 63 | --seed 123456 2>&1 | tee train.log 64 | train_data_file=/CodeSearchNet/defect/train_2.jsonl 65 | eval_data_file=/CodeSearchNet/defect/dev_2.jsonl 66 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 67 | output_dir=./saved_models/$name/task_2 68 | python run_cl.py \ 69 | --output_dir=$output_dir \ 70 | --model_type=roberta \ 71 | --tokenizer_name=$pretrained_model \ 72 | --model_name_or_path=$pretrained_model \ 73 | --load_model_path $load_model_path\ 74 | --do_train \ 75 | --train_data_file=$train_data_file \ 76 | --eval_data_file=$eval_data_file \ 77 | --mode $name \ 78 | --task_id 2 \ 79 | --ewc_weight $weight \ 80 | --train_replay_size=$train_replay_size \ 81 | --train_examplar_path=$train_examplar_path \ 82 | --eval_examplar_path=$dev_examplar_path \ 83 | --epoch 10 \ 84 | --block_size 400 \ 85 | --train_batch_size 16 \ 86 | --eval_batch_size 64 \ 87 | --learning_rate 2e-5 \ 88 | --max_grad_norm 1.0 \ 89 | --evaluate_during_training \ 90 | --seed 123456 2>&1 | tee train.log 91 | train_data_file=/CodeSearchNet/defect/train_3.jsonl 92 | eval_data_file=/CodeSearchNet/defect/dev_3.jsonl 93 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 94 | output_dir=./saved_models/$name/task_3 95 | python run_cl.py \ 96 | --output_dir=$output_dir \ 97 | --model_type=roberta \ 98 | --tokenizer_name=$pretrained_model \ 99 | --model_name_or_path=$pretrained_model \ 100 | --load_model_path $load_model_path\ 101 | --do_train \ 102 | --train_data_file=$train_data_file \ 103 | --eval_data_file=$eval_data_file \ 104 | --mode $name \ 105 | --task_id 3 \ 106 | --ewc_weight $weight \ 107 | --train_replay_size=$train_replay_size \ 108 | --train_examplar_path=$train_examplar_path \ 109 | --eval_examplar_path=$dev_examplar_path \ 110 | --epoch 10 \ 111 | --block_size 400 \ 112 | --train_batch_size 16 \ 113 | --eval_batch_size 64 \ 114 | --learning_rate 2e-5 \ 115 | --max_grad_norm 1.0 \ 116 | --evaluate_during_training \ 117 | --seed 123456 2>&1 | tee train.log 118 | train_data_file=/CodeSearchNet/defect/train_4.jsonl 119 | eval_data_file=/CodeSearchNet/defect/dev_4.jsonl 120 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 121 | output_dir=./saved_models/$name/task_4 122 | python run_cl.py \ 123 | --output_dir=$output_dir \ 124 | --model_type=roberta \ 125 | --tokenizer_name=$pretrained_model \ 126 | --model_name_or_path=$pretrained_model \ 127 | --load_model_path $load_model_path\ 128 | --do_train \ 129 | --train_data_file=$train_data_file \ 130 | --eval_data_file=$eval_data_file \ 131 | --mode $name \ 132 | --task_id 4 \ 133 | --ewc_weight $weight \ 134 | --train_replay_size=$train_replay_size \ 135 | --train_examplar_path=$train_examplar_path \ 136 | --eval_examplar_path=$dev_examplar_path \ 137 | --epoch 10 \ 138 | --block_size 400 \ 139 | --train_batch_size 16 \ 140 | --eval_batch_size 64 \ 141 | --learning_rate 2e-5 \ 142 | --max_grad_norm 1.0 \ 143 | --evaluate_during_training \ 144 | --seed 123456 2>&1 | tee train.log 145 | 146 | 147 | #generate 148 | cp -r ./saved_models/finetune/task_0 ./saved_models/$name 149 | 150 | data_dir=/CodeSearchNet/defect 151 | test_data_file=$data_dir/test_0.jsonl,$data_dir/test_1.jsonl,$data_dir/test_2.jsonl,$data_dir/test_3.jsonl,$data_dir/test_4.jsonl 152 | output_dir=./saved_models/$name/task_0,./saved_models/$name/task_1,./saved_models/$name/task_2,./saved_models/$name/task_3,./saved_models/$name/task_4 153 | python run.py \ 154 | --output_dir=$output_dir \ 155 | --model_type=roberta \ 156 | --tokenizer_name=$pretrained_model \ 157 | --model_name_or_path=$pretrained_model \ 158 | --do_test \ 159 | --train_data_file=$test_data_file \ 160 | --test_data_file=$test_data_file \ 161 | --epoch 5 \ 162 | --block_size 400 \ 163 | --train_batch_size 32 \ 164 | --eval_batch_size 64 \ 165 | --learning_rate 2e-5 \ 166 | --max_grad_norm 1.0 \ 167 | --evaluate_during_training \ 168 | --seed 123456 2>&1 | tee test.log -------------------------------------------------------------------------------- /svd/CodeBERT/run_finetune.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=0 2 | name=finetune 3 | pretrained_model=/microsoft/codebert-base 4 | 5 | #train 6 | train_data_file=/CodeSearchNet/defect/train_0.jsonl 7 | eval_data_file=/CodeSearchNet/defect/dev_0.jsonl 8 | output_dir=./saved_models/$name/task_0 9 | python run.py \ 10 | --output_dir=$output_dir \ 11 | --model_type=roberta \ 12 | --tokenizer_name=$pretrained_model \ 13 | --model_name_or_path=$pretrained_model \ 14 | --do_train \ 15 | --train_data_file=$train_data_file \ 16 | --eval_data_file=$eval_data_file \ 17 | --epoch 10 \ 18 | --block_size 400 \ 19 | --train_batch_size 16 \ 20 | --eval_batch_size 64 \ 21 | --learning_rate 2e-5 \ 22 | --max_grad_norm 1.0 \ 23 | --evaluate_during_training \ 24 | --seed 123456 2>&1 | tee train.log 25 | train_data_file=/CodeSearchNet/defect/train_1.jsonl 26 | eval_data_file=/CodeSearchNet/defect/dev_1.jsonl 27 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 28 | output_dir=./saved_models/$name/task_1 29 | python run.py \ 30 | --output_dir=$output_dir \ 31 | --model_type=roberta \ 32 | --tokenizer_name=$pretrained_model \ 33 | --model_name_or_path=$pretrained_model \ 34 | --do_train \ 35 | --train_data_file=$train_data_file \ 36 | --eval_data_file=$eval_data_file \ 37 | --load_model_path=$load_model_path \ 38 | --epoch 10 \ 39 | --block_size 400 \ 40 | --train_batch_size 16 \ 41 | --eval_batch_size 64 \ 42 | --learning_rate 2e-5 \ 43 | --max_grad_norm 1.0 \ 44 | --evaluate_during_training \ 45 | --seed 123456 2>&1 | tee train.log 46 | train_data_file=/CodeSearchNet/defect/train_2.jsonl 47 | eval_data_file=/CodeSearchNet/defect/dev_2.jsonl 48 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 49 | output_dir=./saved_models/$name/task_2 50 | python run.py \ 51 | --output_dir=$output_dir \ 52 | --model_type=roberta \ 53 | --tokenizer_name=$pretrained_model \ 54 | --model_name_or_path=$pretrained_model \ 55 | --do_train \ 56 | --train_data_file=$train_data_file \ 57 | --eval_data_file=$eval_data_file \ 58 | --load_model_path=$load_model_path \ 59 | --epoch 10 \ 60 | --block_size 400 \ 61 | --train_batch_size 16 \ 62 | --eval_batch_size 64 \ 63 | --learning_rate 2e-5 \ 64 | --max_grad_norm 1.0 \ 65 | --evaluate_during_training \ 66 | --seed 123456 2>&1 | tee train.log 67 | train_data_file=/CodeSearchNet/defect/train_3.jsonl 68 | eval_data_file=/CodeSearchNet/defect/dev_3.jsonl 69 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 70 | output_dir=./saved_models/$name/task_3 71 | python run.py \ 72 | --output_dir=$output_dir \ 73 | --model_type=roberta \ 74 | --tokenizer_name=$pretrained_model \ 75 | --model_name_or_path=$pretrained_model \ 76 | --do_train \ 77 | --train_data_file=$train_data_file \ 78 | --eval_data_file=$eval_data_file \ 79 | --load_model_path=$load_model_path \ 80 | --epoch 10 \ 81 | --block_size 400 \ 82 | --train_batch_size 16 \ 83 | --eval_batch_size 64 \ 84 | --learning_rate 2e-5 \ 85 | --max_grad_norm 1.0 \ 86 | --evaluate_during_training \ 87 | --seed 123456 2>&1 | tee train.log 88 | train_data_file=/CodeSearchNet/defect/train_4.jsonl 89 | eval_data_file=/CodeSearchNet/defect/dev_4.jsonl 90 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 91 | output_dir=./saved_models/$name/task_4 92 | python run.py \ 93 | --output_dir=$output_dir \ 94 | --model_type=roberta \ 95 | --tokenizer_name=$pretrained_model \ 96 | --model_name_or_path=$pretrained_model \ 97 | --do_train \ 98 | --train_data_file=$train_data_file \ 99 | --eval_data_file=$eval_data_file \ 100 | --load_model_path=$load_model_path \ 101 | --epoch 10 \ 102 | --block_size 400 \ 103 | --train_batch_size 16 \ 104 | --eval_batch_size 64 \ 105 | --learning_rate 2e-5 \ 106 | --max_grad_norm 1.0 \ 107 | --evaluate_during_training \ 108 | --seed 123456 2>&1 | tee train.log 109 | 110 | #generate 111 | data_dir=/CodeSearchNet/defect 112 | test_data_file=$data_dir/test_0.jsonl,$data_dir/test_1.jsonl,$data_dir/test_2.jsonl,$data_dir/test_3.jsonl,$data_dir/test_4.jsonl 113 | output_dir=./saved_models/$name/task_0,./saved_models/$name/task_1,./saved_models/$name/task_2,./saved_models/$name/task_3,./saved_models/$name/task_4 114 | python run.py \ 115 | --output_dir=$output_dir \ 116 | --model_type=roberta \ 117 | --tokenizer_name=$pretrained_model \ 118 | --model_name_or_path=$pretrained_model \ 119 | --do_test \ 120 | --train_data_file=$test_data_file \ 121 | --test_data_file=$test_data_file \ 122 | --epoch 5 \ 123 | --block_size 400 \ 124 | --train_batch_size 32 \ 125 | --eval_batch_size 64 \ 126 | --learning_rate 2e-5 \ 127 | --max_grad_norm 1.0 \ 128 | --evaluate_during_training \ 129 | --seed 123456 2>&1 | tee test.log 130 | -------------------------------------------------------------------------------- /svd/CodeBERT/run_multitask.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=0 2 | name=multi_task 3 | pretrained_model=/microsoft/codebert-base 4 | 5 | #train 6 | train_data_file=/CodeSearchNet/defect/train.jsonl 7 | eval_data_file=/CodeSearchNet/defect/dev.jsonl 8 | output_dir=./saved_models/$name/task_4 9 | python run.py \ 10 | --output_dir=$output_dir \ 11 | --model_type=roberta \ 12 | --tokenizer_name=$pretrained_model \ 13 | --model_name_or_path=$pretrained_model \ 14 | --do_train \ 15 | --train_data_file=$train_data_file \ 16 | --eval_data_file=$eval_data_file \ 17 | --epoch 10 \ 18 | --block_size 400 \ 19 | --train_batch_size 16 \ 20 | --eval_batch_size 64 \ 21 | --learning_rate 2e-5 \ 22 | --max_grad_norm 1.0 \ 23 | --evaluate_during_training \ 24 | --seed 123456 2>&1 | tee train.log 25 | 26 | 27 | train_data_file=/CodeSearchNet/defect/train_0123.jsonl 28 | eval_data_file=/CodeSearchNet/defect/dev_0123.jsonl 29 | output_dir=./saved_models/$name/task_3 30 | python run.py \ 31 | --output_dir=$output_dir \ 32 | --model_type=roberta \ 33 | --tokenizer_name=$pretrained_model \ 34 | --model_name_or_path=$pretrained_model \ 35 | --do_train \ 36 | --train_data_file=$train_data_file \ 37 | --eval_data_file=$eval_data_file \ 38 | --epoch 10 \ 39 | --block_size 400 \ 40 | --train_batch_size 16 \ 41 | --eval_batch_size 64 \ 42 | --learning_rate 2e-5 \ 43 | --max_grad_norm 1.0 \ 44 | --evaluate_during_training \ 45 | --seed 123456 2>&1 | tee train.log 46 | 47 | 48 | train_data_file=/CodeSearchNet/defect/train_012.jsonl 49 | eval_data_file=/CodeSearchNet/defect/dev_012.jsonl 50 | output_dir=./saved_models/$name/task_2 51 | python run.py \ 52 | --output_dir=$output_dir \ 53 | --model_type=roberta \ 54 | --tokenizer_name=$pretrained_model \ 55 | --model_name_or_path=$pretrained_model \ 56 | --do_train \ 57 | --train_data_file=$train_data_file \ 58 | --eval_data_file=$eval_data_file \ 59 | --epoch 10 \ 60 | --block_size 400 \ 61 | --train_batch_size 16 \ 62 | --eval_batch_size 64 \ 63 | --learning_rate 2e-5 \ 64 | --max_grad_norm 1.0 \ 65 | --evaluate_during_training \ 66 | --seed 123456 2>&1 | tee train.log 67 | 68 | 69 | train_data_file=/CodeSearchNet/defect/train_01.jsonl 70 | eval_data_file=/CodeSearchNet/defect/dev_01.jsonl 71 | output_dir=./saved_models/$name/task_1 72 | python run.py \ 73 | --output_dir=$output_dir \ 74 | --model_type=roberta \ 75 | --tokenizer_name=$pretrained_model \ 76 | --model_name_or_path=$pretrained_model \ 77 | --do_train \ 78 | --train_data_file=$train_data_file \ 79 | --eval_data_file=$eval_data_file \ 80 | --epoch 10 \ 81 | --block_size 400 \ 82 | --train_batch_size 16 \ 83 | --eval_batch_size 64 \ 84 | --learning_rate 2e-5 \ 85 | --max_grad_norm 1.0 \ 86 | --evaluate_during_training \ 87 | --seed 123456 2>&1 | tee train.log 88 | -------------------------------------------------------------------------------- /svd/CodeBERT/run_ours.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=0 2 | name=emr_ours_adaewc 3 | train_replay_size=1509 4 | weight=2000 5 | k=5 6 | mu=5 7 | train_examplar_path=./saved_models/$name/train_examplar.jsonl 8 | dev_examplar_path=./saved_models/$name/dev_examplar.jsonl 9 | pretrained_model=/microsoft/codebert-base 10 | 11 | #train 12 | train_data_file=/CodeSearchNet/defect/train_0.jsonl 13 | eval_data_file=/CodeSearchNet/defect/dev_0.jsonl 14 | load_model_path=./saved_models/finetune/task_0/checkpoint-best-acc/model.bin 15 | output_dir=./saved_models/$name/task_0 16 | python run_cl.py \ 17 | --output_dir=$output_dir \ 18 | --model_type=roberta \ 19 | --tokenizer_name=$pretrained_model \ 20 | --model_name_or_path=$pretrained_model \ 21 | --load_model_path $load_model_path\ 22 | --do_train \ 23 | --train_data_file=$train_data_file \ 24 | --eval_data_file=$eval_data_file \ 25 | --mode $name \ 26 | --task_id 0 \ 27 | --k $k \ 28 | --mu $mu \ 29 | --ewc_weight $weight \ 30 | --train_replay_size=$train_replay_size \ 31 | --train_examplar_path=$train_examplar_path \ 32 | --eval_examplar_path=$dev_examplar_path \ 33 | --epoch 10 \ 34 | --block_size 400 \ 35 | --train_batch_size 16 \ 36 | --eval_batch_size 64 \ 37 | --learning_rate 2e-5 \ 38 | --max_grad_norm 1.0 \ 39 | --evaluate_during_training \ 40 | --seed 123456 2>&1 | tee train.log 41 | train_data_file=/CodeSearchNet/defect/train_1.jsonl 42 | eval_data_file=/CodeSearchNet/defect/dev_1.jsonl 43 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 44 | output_dir=./saved_models/$name/task_1 45 | python run_cl.py \ 46 | --output_dir=$output_dir \ 47 | --model_type=roberta \ 48 | --tokenizer_name=$pretrained_model \ 49 | --model_name_or_path=$pretrained_model \ 50 | --load_model_path $load_model_path\ 51 | --do_train \ 52 | --train_data_file=$train_data_file \ 53 | --eval_data_file=$eval_data_file \ 54 | --mode $name \ 55 | --task_id 1 \ 56 | --k $k \ 57 | --mu $mu \ 58 | --ewc_weight $weight \ 59 | --train_replay_size=$train_replay_size \ 60 | --train_examplar_path=$train_examplar_path \ 61 | --eval_examplar_path=$dev_examplar_path \ 62 | --epoch 10 \ 63 | --block_size 400 \ 64 | --train_batch_size 16 \ 65 | --eval_batch_size 64 \ 66 | --learning_rate 2e-5 \ 67 | --max_grad_norm 1.0 \ 68 | --evaluate_during_training \ 69 | --seed 123456 2>&1 | tee train.log 70 | train_data_file=/CodeSearchNet/defect/train_2.jsonl 71 | eval_data_file=/CodeSearchNet/defect/dev_2.jsonl 72 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 73 | output_dir=./saved_models/$name/task_2 74 | python run_cl.py \ 75 | --output_dir=$output_dir \ 76 | --model_type=roberta \ 77 | --tokenizer_name=$pretrained_model \ 78 | --model_name_or_path=$pretrained_model \ 79 | --load_model_path $load_model_path\ 80 | --do_train \ 81 | --train_data_file=$train_data_file \ 82 | --eval_data_file=$eval_data_file \ 83 | --mode $name \ 84 | --task_id 2 \ 85 | --k $k \ 86 | --mu $mu \ 87 | --ewc_weight $weight \ 88 | --train_replay_size=$train_replay_size \ 89 | --train_examplar_path=$train_examplar_path \ 90 | --eval_examplar_path=$dev_examplar_path \ 91 | --epoch 10 \ 92 | --block_size 400 \ 93 | --train_batch_size 16 \ 94 | --eval_batch_size 64 \ 95 | --learning_rate 2e-5 \ 96 | --max_grad_norm 1.0 \ 97 | --evaluate_during_training \ 98 | --seed 123456 2>&1 | tee train.log 99 | train_data_file=/CodeSearchNet/defect/train_3.jsonl 100 | eval_data_file=/CodeSearchNet/defect/dev_3.jsonl 101 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 102 | output_dir=./saved_models/$name/task_3 103 | python run_cl.py \ 104 | --output_dir=$output_dir \ 105 | --model_type=roberta \ 106 | --tokenizer_name=$pretrained_model \ 107 | --model_name_or_path=$pretrained_model \ 108 | --load_model_path $load_model_path\ 109 | --do_train \ 110 | --train_data_file=$train_data_file \ 111 | --eval_data_file=$eval_data_file \ 112 | --mode $name \ 113 | --task_id 3 \ 114 | --k $k \ 115 | --mu $mu \ 116 | --ewc_weight $weight \ 117 | --train_replay_size=$train_replay_size \ 118 | --train_examplar_path=$train_examplar_path \ 119 | --eval_examplar_path=$dev_examplar_path \ 120 | --epoch 10 \ 121 | --block_size 400 \ 122 | --train_batch_size 16 \ 123 | --eval_batch_size 64 \ 124 | --learning_rate 2e-5 \ 125 | --max_grad_norm 1.0 \ 126 | --evaluate_during_training \ 127 | --seed 123456 2>&1 | tee train.log 128 | train_data_file=/CodeSearchNet/defect/train_4.jsonl 129 | eval_data_file=/CodeSearchNet/defect/dev_4.jsonl 130 | load_model_path=$output_dir/checkpoint-best-acc/model.bin 131 | output_dir=./saved_models/$name/task_4 132 | python run_cl.py \ 133 | --output_dir=$output_dir \ 134 | --model_type=roberta \ 135 | --tokenizer_name=$pretrained_model \ 136 | --model_name_or_path=$pretrained_model \ 137 | --load_model_path $load_model_path\ 138 | --do_train \ 139 | --train_data_file=$train_data_file \ 140 | --eval_data_file=$eval_data_file \ 141 | --mode $name \ 142 | --task_id 4 \ 143 | --k $k \ 144 | --mu $mu \ 145 | --ewc_weight $weight \ 146 | --train_replay_size=$train_replay_size \ 147 | --train_examplar_path=$train_examplar_path \ 148 | --eval_examplar_path=$dev_examplar_path \ 149 | --epoch 10 \ 150 | --block_size 400 \ 151 | --train_batch_size 16 \ 152 | --eval_batch_size 64 \ 153 | --learning_rate 2e-5 \ 154 | --max_grad_norm 1.0 \ 155 | --evaluate_during_training \ 156 | --seed 123456 2>&1 | tee train.log 157 | 158 | 159 | #generate 160 | cp -r ./saved_models/finetune/task_0 ./saved_models/multi_task 161 | data_dir=/CodeSearchNet/defect 162 | test_data_file=$data_dir/test_0.jsonl,$data_dir/test_1.jsonl,$data_dir/test_2.jsonl,$data_dir/test_3.jsonl,$data_dir/test_4.jsonl 163 | output_dir=./saved_models/$name/task_0,./saved_models/$name/task_1,./saved_models/$name/task_2,./saved_models/$name/task_3,./saved_models/$name/task_4 164 | python run.py \ 165 | --output_dir=$output_dir \ 166 | --model_type=roberta \ 167 | --tokenizer_name=$pretrained_model \ 168 | --model_name_or_path=$pretrained_model \ 169 | --do_test \ 170 | --train_data_file=$test_data_file \ 171 | --test_data_file=$test_data_file \ 172 | --epoch 5 \ 173 | --block_size 256 \ 174 | --train_batch_size 32 \ 175 | --eval_batch_size 64 \ 176 | --learning_rate 2e-5 \ 177 | --max_grad_norm 1.0 \ 178 | --evaluate_during_training \ 179 | --seed 123456 2>&1 | tee test.log 180 | -------------------------------------------------------------------------------- /svd/CodeBERT/test.sh: -------------------------------------------------------------------------------- 1 | export CUDA_VISIBLE_DEVICES=3 2 | name=finetune 3 | pretrained_model=/microsoft/codebert-base 4 | 5 | 6 | data_dir=/CodeSearchNet/defect 7 | test_data_file=$data_dir/test_0.jsonl,$data_dir/test_1.jsonl,$data_dir/test_2.jsonl,$data_dir/test_3.jsonl,$data_dir/test_4.jsonl 8 | output_dir=./saved_models/$name/task_0,./saved_models/$name/task_1,./saved_models/$name/task_2,./saved_models/$name/task_3,./saved_models/$name/task_4 9 | python run.py \ 10 | --output_dir=$output_dir \ 11 | --model_type=roberta \ 12 | --tokenizer_name=$pretrained_model \ 13 | --model_name_or_path=$pretrained_model \ 14 | --do_test \ 15 | --train_data_file=$test_data_file \ 16 | --test_data_file=$test_data_file \ 17 | --epoch 5 \ 18 | --block_size 400 \ 19 | --train_batch_size 32 \ 20 | --eval_batch_size 64 \ 21 | --learning_rate 2e-5 \ 22 | --max_grad_norm 1.0 \ 23 | --evaluate_during_training \ 24 | --seed 123456 2>&1 | tee test.log 25 | -------------------------------------------------------------------------------- /svd/CodeT5/evaluate.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import pandas as pd 3 | import json 4 | from collections import OrderedDict, Counter 5 | import os 6 | import argparse 7 | 8 | 9 | def read_file(base_dir, task_id): 10 | f = csv.reader(open(os.path.join(base_dir, 'test_'+task_id+'.gold'),'r')) 11 | ref = {} 12 | count = 0 13 | for i in f: 14 | ref[count] = [i[0].split('\t')[1].strip()] 15 | count+=1 16 | f = csv.reader(open(os.path.join(base_dir, 'test_'+task_id+'.output'),'r')) 17 | hypo = {} 18 | count = 0 19 | for i in f: 20 | hypo[count] = [i[0].split('\t')[1].strip()] 21 | if len(hypo[count][0].split()) == 0: 22 | hypo[count] = ['Get'] 23 | count+=1 24 | return hypo,ref 25 | 26 | parser = argparse.ArgumentParser() 27 | parser.add_argument("--mode", default=None, type=str, required=True, help="mode") 28 | parser.add_argument("--lang", default=None, type=str, required=True, help="lang") 29 | args = parser.parse_args() 30 | 31 | acc_sum=0 32 | p_sum=0 33 | r_sum=0 34 | f1_sum=0 35 | acc_list=[[] for _ in range(5)] 36 | f1_list=[[] for _ in range(5)] 37 | for i in range(5): 38 | for j in range(i+1): 39 | hypo,ref = read_file('model/'+args.mode+'/task_'+str(i)+'/', str(j)) 40 | tp,tn,fp,fn=0,0,0,0 41 | for k in hypo: 42 | prediction=hypo[k][0] 43 | refrence=ref[k][0] 44 | if prediction=='true' and refrence =='true': 45 | tp+=1 46 | elif prediction=='true' and refrence =='false': 47 | fn+=1 48 | elif prediction=='false' and refrence =='false': 49 | tn+=1 50 | elif prediction=='false' and refrence =='true': 51 | fp+=1 52 | p = tp/(tp+fp) 53 | r = tp/(tp+fn) 54 | f1 = 2*p*r/(p+r) 55 | acc = (tp+tn)/(tp+tn+fp+fn) 56 | #print('('+str(round(100*f1,2))+','+str(round(100*acc,2))+')', end=' ') 57 | print(round(f1*100,2), end=' ') 58 | f1_sum+=f1/(i+1) 59 | acc_sum+=acc/(i+1) 60 | p_sum+=p/(i+1) 61 | r_sum+=r/(i+1) 62 | print('') 63 | 64 | print('Avg_F1:',round(100*f1_sum/5,2)) 65 | print('Avg_P:',round(100*p_sum/5,2)) 66 | print('Avg_R:',round(100*r_sum/5,2)) 67 | print('Avg_ACC:',round(100*acc_sum/5,2)) -------------------------------------------------------------------------------- /svd/CodeT5/ewc.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | from torch.autograd import Variable 4 | import copy 5 | #from model.masked_cross_entropy import * 6 | 7 | def variable(t: torch.Tensor, use_cuda=True, **kwargs): 8 | if torch.cuda.is_available() and use_cuda: 9 | t = t.cuda() 10 | return Variable(t, **kwargs) 11 | 12 | class EWC(object): 13 | def __init__(self, model, dataset, device, length, tokenizer): 14 | 15 | self.model = model 16 | # the data we use to compute fisher information of ewc (old_exemplars) 17 | self.dataset = dataset 18 | self.device = device 19 | self.length = length 20 | self.tokenizer = tokenizer 21 | 22 | self.params = {n: p for n, p in self.model.named_parameters() if p.requires_grad} 23 | self._means = {} # previous parameters 24 | self._precision_matrices = self._diag_fisher() # approximated diagnal fisher information matrix 25 | 26 | for n, p in copy.deepcopy(self.params).items(): 27 | self._means[n] = variable(p.data) 28 | 29 | def _diag_fisher(self): 30 | 31 | self.model.train() 32 | precision_matrices = {} 33 | for n, p in copy.deepcopy(self.params).items(): 34 | p.data.zero_() 35 | precision_matrices[n] = variable(p.data) 36 | 37 | for batch in self.dataset: 38 | self.model.zero_grad() 39 | 40 | batch = tuple(t.to(self.device) for t in batch) 41 | source_ids, source_mask, target_ids, target_mask = batch 42 | labels = [ 43 | [(label if label != self.tokenizer.pad_token_id else -100) for label in labels_example] for 44 | labels_example in target_ids 45 | ] 46 | labels = torch.tensor(labels).to(self.device) 47 | out = self.model(input_ids=source_ids, attention_mask=source_mask, labels=labels) 48 | loss = out.loss 49 | 50 | loss.backward() 51 | 52 | for n, p in self.model.named_parameters(): 53 | if p.grad is None: 54 | continue 55 | precision_matrices[n].data += p.grad.data ** 2 / self.length 56 | 57 | precision_matrices = {n: p for n, p in precision_matrices.items()} 58 | self.model.zero_grad() 59 | return precision_matrices 60 | 61 | def penalty(self, model: nn.Module): 62 | loss = 0 63 | for n, p in model.named_parameters(): 64 | _loss = self._precision_matrices[n] * (p - self._means[n]) ** 2 65 | loss += _loss.sum() 66 | 67 | return loss 68 | -------------------------------------------------------------------------------- /svd/CodeT5/my_lib.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | 4 | from openprompt.data_utils import InputExample 5 | 6 | 7 | def get_elapse_time(t0): 8 | elapse_time = time.time() - t0 9 | if elapse_time > 3600: 10 | hour = int(elapse_time // 3600) 11 | minute = int((elapse_time % 3600) // 60) 12 | return "{}h{}m".format(hour, minute) 13 | else: 14 | minute = int((elapse_time % 3600) // 60) 15 | return "{}m".format(minute) 16 | 17 | 18 | class Example(object): 19 | """A single training/test example.""" 20 | 21 | def __init__(self, 22 | idx, 23 | source, 24 | target, 25 | original_source, 26 | original_target 27 | ): 28 | self.idx = idx 29 | self.source = source 30 | self.target = target 31 | self.original_source = original_source 32 | self.original_target = original_target 33 | 34 | 35 | def read_examples(filename, args): 36 | """Read examples from filename.""" 37 | if args.add_task_prefix: 38 | task_prefix = f"Generating comments for {args.lang}: " 39 | else: 40 | task_prefix = "" 41 | 42 | if args.add_lang_ids: 43 | language_prefix = " " 44 | else: 45 | language_prefix = "" 46 | 47 | examples = [] 48 | with open(filename, encoding="utf-8") as f: 49 | for idx, line in enumerate(f): 50 | line = line.strip() 51 | js = json.loads(line) 52 | if 'idx' not in js: 53 | js['idx'] = idx 54 | code = ' '.join(js['code_tokens']).replace('\n', ' ') 55 | code = ' '.join(code.strip().split()) 56 | 57 | nl = ' '.join(js['docstring_tokens']).replace('\n', '') 58 | nl = ' '.join(nl.strip().split()) 59 | examples.append( 60 | Example( 61 | idx=idx, 62 | source=task_prefix + code, 63 | target=language_prefix + nl, 64 | original_source=js['code_tokens'], 65 | original_target=js['docstring_tokens'], 66 | ) 67 | ) 68 | 69 | return examples 70 | 71 | 72 | class InputFeatures(object): 73 | """A single training/test features for an example.""" 74 | 75 | def __init__(self, 76 | example_id, 77 | source_ids, 78 | target_ids, 79 | source_mask, 80 | target_mask, 81 | 82 | ): 83 | self.example_id = example_id 84 | self.source_ids = source_ids 85 | self.target_ids = target_ids 86 | self.source_mask = source_mask 87 | self.target_mask = target_mask 88 | 89 | 90 | def convert_examples_to_features(examples, tokenizer, args, stage=None): 91 | # collect texts 92 | codes = [] 93 | target_nl = [] 94 | for example_id, example in enumerate(examples): 95 | codes.append(example.source) 96 | 97 | if stage == "test": 98 | target_nl.append("None") 99 | else: 100 | target_nl.append(example.target) 101 | 102 | # begin tokenizing 103 | encoded_codes = tokenizer( 104 | codes, padding=True, verbose=False, add_special_tokens=True, 105 | truncation=True, max_length=args.max_source_length, return_tensors='pt') 106 | 107 | encoded_targets = tokenizer( 108 | target_nl, padding=True, verbose=False, add_special_tokens=True, 109 | truncation=True, max_length=args.max_source_length, return_tensors='pt') 110 | 111 | return {'source_ids':encoded_codes['input_ids'], 'target_ids':encoded_targets['input_ids'], 112 | 'source_mask':encoded_codes['attention_mask'], 'target_mask':encoded_targets['attention_mask']} 113 | 114 | 115 | def read_prompt_examples(filename): 116 | """Read examples from filename.""" 117 | examples = [] 118 | with open(filename, encoding="utf-8") as f: 119 | for idx, line in enumerate(f): 120 | line = line.strip() 121 | js = json.loads(line) 122 | if 'idx' not in js: 123 | js['idx'] = idx 124 | code = ' '.join(js['code_tokens']).replace('\n', ' ') 125 | code = ' '.join(code.strip().split()) 126 | nl = ' '.join(js['docstring_tokens']).replace('\n', '') 127 | nl = ' '.join(nl.strip().split()) 128 | # code = js['code'].replace('\n', ' ').strip() 129 | # nl = js['docstring'].replace('\n', ' ').strip() 130 | examples.append( 131 | InputExample( 132 | guid=idx, 133 | text_a= code, 134 | tgt_text= nl, 135 | ) 136 | ) 137 | 138 | return examples -------------------------------------------------------------------------------- /svd/CodeT5/run_emr.sh: -------------------------------------------------------------------------------- 1 | gpu=1 2 | lang=defect 3 | name=emr 4 | train_replay_size=1509 5 | pretrained_model=/Salesforce/codet5-base 6 | num_train_epochs=10 7 | batch_size=32 8 | beam_size=1 9 | train_examplar_path=model/$name/train_examplar.jsonl 10 | dev_examplar_path=model/$name/dev_examplar.jsonl 11 | 12 | #train 13 | train_file=/CodeSearchNet/defect/train_0.jsonl 14 | dev_file=/CodeSearchNet/defect/dev_0.jsonl 15 | load_model_path=./model/defect/finetune1/task_0/checkpoint-best-f1/pytorch_model.bin 16 | output_dir=model/$name/task_0 17 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 18 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 19 | --mode_name $name --task_id 0 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 20 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 21 | 22 | train_file=/CodeSearchNet/defect/train_1.jsonl 23 | dev_file=/CodeSearchNet/defect/dev_1.jsonl 24 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 25 | output_dir=model/$name/task_1 26 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 27 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 28 | --mode_name $name --task_id 1 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 29 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 30 | 31 | train_file=/CodeSearchNet/defect/train_2.jsonl 32 | dev_file=/CodeSearchNet/defect/dev_2.jsonl 33 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 34 | output_dir=model/$name/task_2 35 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 36 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 37 | --mode_name $name --task_id 2 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 38 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 39 | 40 | train_file=/CodeSearchNet/defect/train_3.jsonl 41 | dev_file=/CodeSearchNet/defect/dev_3.jsonl 42 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 43 | output_dir=model/$name/task_3 44 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 45 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 46 | --mode_name $name --task_id 3 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 47 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 48 | 49 | train_file=/CodeSearchNet/defect/train_4.jsonl 50 | dev_file=/CodeSearchNet/defect/dev_4.jsonl 51 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 52 | output_dir=model/$name/task_4 53 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 54 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 55 | --mode_name $name --task_id 4 --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 56 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 57 | 58 | 59 | #generate 60 | beam_size=10 61 | batch_size=32 62 | 63 | mkdir -p model/$name/task_0 64 | cp ./model/$lang/finetune1/task_0/test_0.gold ./model/$name/task_0 65 | cp ./model/$lang/finetune1/task_0/test_0.output ./model/$name/task_0 66 | 67 | output_dir=model/$name/task_1 68 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 69 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl 70 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 71 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir \ 72 | --eval_batch_size $batch_size --beam_size $beam_size 73 | 74 | output_dir=model/$name/task_2 75 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 76 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl,/CodeSearchNet/defect/test_2.jsonl 77 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 78 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir \ 79 | --eval_batch_size $batch_size --beam_size $beam_size 80 | 81 | output_dir=model/$name/task_3 82 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 83 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl,/CodeSearchNet/defect/test_2.jsonl,/CodeSearchNet/defect/test_3.jsonl 84 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 85 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir \ 86 | --eval_batch_size $batch_size --beam_size $beam_size 87 | 88 | output_dir=model/$name/task_4 89 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 90 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl,/CodeSearchNet/defect/test_2.jsonl,/CodeSearchNet/defect/test_3.jsonl,/CodeSearchNet/defect/test_4.jsonl 91 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 92 | --do_test --model_name_or_path $pretrained_model --output_dir $output_dir \ 93 | --eval_batch_size $batch_size --beam_size $beam_size 94 | 95 | 96 | #evaluate 97 | python evaluate.py --mode $name --lang $lang -------------------------------------------------------------------------------- /svd/CodeT5/run_ewc.sh: -------------------------------------------------------------------------------- 1 | gpu=0 2 | lang=defect 3 | name=ewc 4 | train_replay_size=1509 5 | weight=2000 6 | pretrained_model=/Salesforce/codet5-base 7 | num_train_epochs=10 8 | batch_size=32 9 | beam_size=1 10 | train_examplar_path=model/$name/train_examplar.jsonl 11 | dev_examplar_path=model/$name/dev_examplar.jsonl 12 | 13 | #train 14 | train_file=/CodeSearchNet/defect/train_0.jsonl 15 | dev_file=/CodeSearchNet/defect/dev_0.jsonl 16 | load_model_path=./model/defect/finetune1/task_0/checkpoint-best-f1/pytorch_model.bin 17 | output_dir=model/$name/task_0 18 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 19 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 20 | --mode_name $name --task_id 0 --ewc_weight=$weight --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 21 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 22 | 23 | train_file=/CodeSearchNet/defect/train_1.jsonl 24 | dev_file=/CodeSearchNet/defect/dev_1.jsonl 25 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 26 | output_dir=model/$name/task_1 27 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 28 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 29 | --mode_name $name --task_id 1 --ewc_weight=$weight --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 30 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 31 | 32 | train_file=/CodeSearchNet/defect/train_2.jsonl 33 | dev_file=/CodeSearchNet/defect/dev_2.jsonl 34 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 35 | output_dir=model/$name/task_2 36 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 37 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 38 | --mode_name $name --task_id 2 --ewc_weight=$weight --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 39 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 40 | 41 | train_file=/CodeSearchNet/defect/train_3.jsonl 42 | dev_file=/CodeSearchNet/defect/dev_3.jsonl 43 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 44 | output_dir=model/$name/task_3 45 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 46 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 47 | --mode_name $name --task_id 3 --ewc_weight=$weight --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 48 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 49 | 50 | train_file=/CodeSearchNet/defect/train_4.jsonl 51 | dev_file=/CodeSearchNet/defect/dev_4.jsonl 52 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 53 | output_dir=model/$name/task_4 54 | python run_cl.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 55 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path\ 56 | --mode_name $name --task_id 4 --ewc_weight=$weight --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size \ 57 | --train_examplar_path $train_examplar_path --eval_examplar_path $dev_examplar_path --train_replay_size $train_replay_size 58 | 59 | 60 | #generate 61 | beam_size=10 62 | batch_size=128 63 | 64 | mkdir -p model/$name/task_0 65 | cp ./model/$lang/finetune1/task_0/test_0.gold ./model/$name/task_0 66 | cp ./model/$lang/finetune1/task_0/test_0.output ./model/$name/task_0 67 | 68 | output_dir=model/$name/task_1 69 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 70 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl 71 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 72 | --do_test --model_name $pretrained_model --output_dir $output_dir \ 73 | --eval_batch_size $batch_size --beam_size $beam_size 74 | 75 | output_dir=model/$name/task_2 76 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 77 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl,/CodeSearchNet/defect/test_2.jsonl 78 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 79 | --do_test --model_name $pretrained_model --output_dir $output_dir \ 80 | --eval_batch_size $batch_size --beam_size $beam_size 81 | 82 | output_dir=model/$name/task_3 83 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 84 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl,/CodeSearchNet/defect/test_2.jsonl,/CodeSearchNet/defect/test_3.jsonl 85 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 86 | --do_test --model_name $pretrained_model --output_dir $output_dir \ 87 | --eval_batch_size $batch_size --beam_size $beam_size 88 | 89 | output_dir=model/$name/task_4 90 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 91 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl,/CodeSearchNet/defect/test_2.jsonl,/CodeSearchNet/defect/test_3.jsonl,/CodeSearchNet/defect/test_4.jsonl 92 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 93 | --do_test --model_name $pretrained_model --output_dir $output_dir \ 94 | --eval_batch_size $batch_size --beam_size $beam_size 95 | 96 | 97 | #evaluate 98 | python evaluate.py --mode $name --lang $lang -------------------------------------------------------------------------------- /svd/CodeT5/run_finetune.sh: -------------------------------------------------------------------------------- 1 | gpu=1 2 | lang=defect 3 | name=finetune 4 | pretrained_model=/Salesforce/codet5-base 5 | num_train_epochs=10 6 | batch_size=32 7 | beam_size=1 8 | 9 | #train 10 | train_file=/CodeSearchNet/defect/train_0.jsonl 11 | dev_file=/CodeSearchNet/defect/dev_0.jsonl 12 | output_dir=model/$lang/$name/task_0 13 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 14 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir \ 15 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 16 | 17 | train_file=/CodeSearchNet/defect/train_1.jsonl 18 | dev_file=/CodeSearchNet/defect/dev_1.jsonl 19 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 20 | output_dir=model/$lang/$name/task_1 21 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 22 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 23 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 24 | 25 | train_file=/CodeSearchNet/defect/train_2.jsonl 26 | dev_file=/CodeSearchNet/defect/dev_2.jsonl 27 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 28 | output_dir=model/$lang/$name/task_2 29 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 30 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 31 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 32 | 33 | train_file=/CodeSearchNet/defect/train_3.jsonl 34 | dev_file=/CodeSearchNet/defect/dev_3.jsonl 35 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 36 | output_dir=model/$lang/$name/task_3 37 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 38 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 39 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 40 | 41 | train_file=/CodeSearchNet/defect/train_4.jsonl 42 | dev_file=/CodeSearchNet/defect/dev_4.jsonl 43 | load_model_path=$output_dir/checkpoint-best-f1/pytorch_model.bin 44 | output_dir=model/$lang/$name/task_4 45 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 46 | --do_train --do_eval --model_name_or_path $pretrained_model --output_dir $output_dir --load_model_path $load_model_path \ 47 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 48 | 49 | 50 | #generate 51 | beam_size=10 52 | batch_size=32 53 | 54 | output_dir=model/$lang/$name/task_0 55 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 56 | test_file=/CodeSearchNet/defect/test_0.jsonl 57 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 58 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model\ 59 | --eval_batch_size $batch_size --beam_size $beam_size 60 | 61 | output_dir=model/$lang/$name/task_1 62 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 63 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl 64 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 65 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 66 | --eval_batch_size $batch_size --beam_size $beam_size 67 | 68 | output_dir=model/$lang/$name/task_2 69 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 70 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl,/CodeSearchNet/defect/test_2.jsonl 71 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 72 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 73 | --eval_batch_size $batch_size --beam_size $beam_size 74 | 75 | output_dir=model/$lang/$name/task_3 76 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 77 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl,/CodeSearchNet/defect/test_2.jsonl,/CodeSearchNet/defect/test_3.jsonl 78 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 79 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 80 | --eval_batch_size $batch_size --beam_size $beam_size 81 | 82 | output_dir=model/$lang/$name/task_4 83 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 84 | test_file=/CodeSearchNet/defect/test_0.jsonl,/CodeSearchNet/defect/test_1.jsonl,/CodeSearchNet/defect/test_2.jsonl,/CodeSearchNet/defect/test_3.jsonl,/CodeSearchNet/defect/test_4.jsonl 85 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 86 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 87 | --eval_batch_size $batch_size --beam_size $beam_size 88 | 89 | 90 | #evaluate 91 | python evaluate.py --mode $name --lang $lang -------------------------------------------------------------------------------- /svd/CodeT5/run_multitask.sh: -------------------------------------------------------------------------------- 1 | gpu=0 2 | lang=defect 3 | name=multi_task 4 | pretrained_model=/Salesforce/codet5-base 5 | num_train_epochs=10 6 | batch_size=32 7 | beam_size=1 8 | 9 | #train 10 | 11 | train_file=/CodeSearchNet/defect/train_01.jsonl 12 | dev_file=/CodeSearchNet/defect/dev_01.jsonl 13 | output_dir=model/$name/task_1 14 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 15 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 16 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 17 | 18 | train_file=/CodeSearchNet/defect/train_012.jsonl 19 | dev_file=/CodeSearchNet/defect/dev_012.jsonl 20 | output_dir=model/$name/task_2 21 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 22 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 23 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 24 | 25 | train_file=/CodeSearchNet/defect/train_0123.jsonl 26 | dev_file=/CodeSearchNet/defect/dev_0123.jsonl 27 | output_dir=model/$name/task_3 28 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 29 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 30 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 31 | 32 | train_file=/CodeSearchNet/defect/train.jsonl 33 | dev_file=/CodeSearchNet/defect/dev.jsonl 34 | output_dir=model/$name/task_4 35 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 3 --train_file $train_file --dev_file $dev_file \ 36 | --do_train --do_eval --model_name $pretrained_model --output_dir $output_dir \ 37 | --num_train_epochs $num_train_epochs --train_batch_size $batch_size --beam_size $beam_size 38 | 39 | 40 | #generate 41 | beam_size=10 42 | batch_size=32 43 | 44 | mkdir -p model/$lang/$name/task_0 45 | cp ./model/$lang/finetune/task_0/test_0.gold ./model/$lang/$name/task_0 46 | cp ./model/$lang/finetune/task_0/test_0.output ./model/$lang/$name/task_0 47 | 48 | output_dir=model/$lang/$name/task_1 49 | test_model=$output_dir/checkpoint-last/pytorch_model.bin 50 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl 51 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 52 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 53 | --eval_batch_size $batch_size --beam_size $beam_size 54 | 55 | output_dir=model/$lang/$name/task_2 56 | test_model=$output_dir/checkpoint-last/pytorch_model.bin 57 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl,$data_dir/$lang/cl/test_2.jsonl 58 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 59 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 60 | --eval_batch_size $batch_size --beam_size $beam_size 61 | 62 | output_dir=model/$lang/$name/task_3 63 | test_model=$output_dir/checkpoint-last/pytorch_model.bin 64 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl,$data_dir/$lang/cl/test_2.jsonl,$data_dir/$lang/cl/test_3.jsonl 65 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 66 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 67 | --eval_batch_size $batch_size --beam_size $beam_size 68 | 69 | output_dir=model/$lang/$name/task_4 70 | test_model=$output_dir/checkpoint-last/pytorch_model.bin 71 | test_file=$data_dir/$lang/cl/test_0.jsonl,$data_dir/$lang/cl/test_1.jsonl,$data_dir/$lang/cl/test_2.jsonl,$data_dir/$lang/cl/test_3.jsonl,$data_dir/$lang/cl/test_4.jsonl 72 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 73 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 74 | --eval_batch_size $batch_size --beam_size $beam_size 75 | 76 | 77 | #evaluate 78 | python evaluate.py --mode $name --lang $lang -------------------------------------------------------------------------------- /svd/CodeT5/test.sh: -------------------------------------------------------------------------------- 1 | gpu=0 2 | lang=defect 3 | name=multi_task 4 | pretrained_model=/Salesforce/codet5-base 5 | data_dir=/CodeSearchNet 6 | num_train_epochs=10 7 | batch_size=32 8 | beam_size=1 9 | 10 | 11 | #generate 12 | beam_size=10 13 | batch_size=32 14 | 15 | mkdir -p model/$name/task_0 16 | cp ./model/$lang/finetune/task_0/test_0.gold ./model/$name/task_0 17 | cp ./model/$lang/finetune/task_0/test_0.output ./model/$name/task_0 18 | 19 | output_dir=model/$name/task_1 20 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 21 | test_file=$data_dir/$lang/test_0.jsonl,$data_dir/$lang/test_1.jsonl 22 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 23 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 24 | --eval_batch_size $batch_size --beam_size $beam_size 25 | 26 | output_dir=model/$name/task_2 27 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 28 | test_file=$data_dir/$lang/test_0.jsonl,$data_dir/$lang/test_1.jsonl,$data_dir/$lang/test_2.jsonl 29 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 30 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 31 | --eval_batch_size $batch_size --beam_size $beam_size 32 | 33 | output_dir=model/$name/task_3 34 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 35 | test_file=$data_dir/$lang/test_0.jsonl,$data_dir/$lang/test_1.jsonl,$data_dir/$lang/test_2.jsonl,$data_dir/$lang/test_3.jsonl 36 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 37 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 38 | --eval_batch_size $batch_size --beam_size $beam_size 39 | 40 | output_dir=model/$name/task_4 41 | test_model=$output_dir/checkpoint-best-f1/pytorch_model.bin 42 | test_file=$data_dir/$lang/test_0.jsonl,$data_dir/$lang/test_1.jsonl,$data_dir/$lang/test_2.jsonl,$data_dir/$lang/test_3.jsonl,$data_dir/$lang/test_4.jsonl 43 | python run.py --visible_gpu $gpu --lang $lang --max_source_length 256 --max_target_length 128 --test_file $test_file \ 44 | --do_test --model_name $pretrained_model --output_dir $output_dir --load_model_path $test_model \ 45 | --eval_batch_size $batch_size --beam_size $beam_size 46 | 47 | 48 | #evaluate 49 | python evaluate.py --mode $name --lang $lang --------------------------------------------------------------------------------