├── .gitignore ├── ._utils.py ├── experiments ├── ._rnnTest.csv ├── ._rnnValidation.csv ├── transformerTest.csv ├── transformerValidation.csv ├── rnnTest.csv └── rnnValidation.csv ├── Hyperparameter_Analysis_for_Image_Captioning.pdf ├── requirements.txt ├── vocab_builder.py ├── utils.py ├── bleu.py ├── README.md ├── eval.py ├── train.py ├── LICENSE └── models.py /.gitignore: -------------------------------------------------------------------------------- 1 | /dataset 2 | /proj_env 3 | /__pycache__ 4 | /model_saves 5 | *.sh 6 | -------------------------------------------------------------------------------- /._utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aravindvarier/Image-Captioning-Pytorch/HEAD/._utils.py -------------------------------------------------------------------------------- /experiments/._rnnTest.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aravindvarier/Image-Captioning-Pytorch/HEAD/experiments/._rnnTest.csv -------------------------------------------------------------------------------- /experiments/._rnnValidation.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aravindvarier/Image-Captioning-Pytorch/HEAD/experiments/._rnnValidation.csv -------------------------------------------------------------------------------- /Hyperparameter_Analysis_for_Image_Captioning.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aravindvarier/Image-Captioning-Pytorch/HEAD/Hyperparameter_Analysis_for_Image_Captioning.pdf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cycler==0.10.0 2 | Cython==0.29.16 3 | kiwisolver==1.2.0 4 | matplotlib==3.2.1 5 | nltk==3.4.5 6 | numpy==1.18.2 7 | Pillow==8.1.1 8 | pkg-resources==0.0.0 9 | pycocoevalcap==1.1 10 | pycocotools==2.0.0 11 | pyparsing==2.4.6 12 | python-dateutil==2.8.1 13 | six==1.14.0 14 | torch==1.4.0 15 | torchvision==0.5.0 16 | tqdm==4.45.0 17 | -------------------------------------------------------------------------------- /vocab_builder.py: -------------------------------------------------------------------------------- 1 | import utils 2 | ann_file = './dataset/Flickr8k_text/Flickr8k.token.txt' 3 | train_file = './dataset/Flickr8k_text/Flickr_8k.trainImages.txt' 4 | val_file = './dataset/Flickr8k_text/Flickr_8k.devImages.txt' 5 | test_file = './dataset/Flickr8k_text/Flickr_8k.testImages.txt' 6 | output_file = './vocab.txt' 7 | 8 | 9 | captions = [] 10 | train_lines = open(train_file, "r").readlines() 11 | val_lines = open(val_file, "r").readlines() 12 | test_lines = open(test_file, "r").readlines() 13 | 14 | with open(ann_file, "r") as ann_f: 15 | for line in ann_f: 16 | img = line.split('#')[0] + "\n" 17 | if (img in train_lines) or (img in val_lines) or (img in test_lines): 18 | caption = utils.clean_description(line.replace("-", " ").split()[1:]) 19 | captions.append(caption) 20 | 21 | vocab = [] 22 | for caption in captions: 23 | for word in caption: 24 | if word not in vocab: 25 | vocab.append(word) 26 | 27 | print("Vocabulary length: ",len(vocab)) 28 | with open(output_file, "w") as out_f: 29 | for word in vocab: 30 | out_f.write(word + "\n") 31 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import string 2 | import torch 3 | import eval 4 | from csv import writer 5 | import datetime 6 | 7 | def clean_description(desc): 8 | # prepare translation table for removing punctuation 9 | table = str.maketrans('', '', string.punctuation) 10 | # # tokenize 11 | # desc = desc.split() 12 | # convert to lower case 13 | desc = [word.lower() for word in desc] 14 | # remove punctuation from each token 15 | desc = [w.translate(table) for w in desc] 16 | #remove numbers 17 | table = str.maketrans('', '', string.digits) 18 | desc = [w.translate(table) for w in desc] 19 | # remove one letter words except 'a' 20 | desc = [word for word in desc if len(word)>1 or word == 'a'] 21 | 22 | 23 | return desc 24 | 25 | def adjust_learning_rate(optimizer, shrink_factor): 26 | """ 27 | Shrinks learning rate by a specified factor. 28 | :param optimizer: optimizer whose learning rate must be shrunk. 29 | :param shrink_factor: factor in interval (0, 1) to multiply learning rate with. 30 | """ 31 | 32 | print("\nDECAYING learning rate.") 33 | for param_group in optimizer.param_groups: 34 | param_group['lr'] = param_group['lr'] * shrink_factor 35 | print("The new learning rate is %f\n" % (optimizer.param_groups[0]['lr'],)) 36 | 37 | def append_new_metric(file_name, metric): 38 | with open('./experiments/'+file_name, 'a+', newline='') as write_obj: 39 | csv_writer = writer(write_obj) 40 | csv_writer.writerow(metric) 41 | 42 | def save_model_and_result(save_path, experiment, model, decoder_type, optimizer, best_epoch, bleu4, loss, val_metrics, test_metrics): 43 | print(f"Saving Best Model with Bleu_4 score of {bleu4}") 44 | torch.save({ 45 | "model_state_dict": model.state_dict(), 46 | # "optimizer_state_dict": optimizer.state_dict(), 47 | "epoch": best_epoch, 48 | "loss": loss, 49 | "best_bleu4": bleu4 50 | }, save_path + f'{experiment}.pt') 51 | now = datetime.datetime.now() 52 | val_row = [experiment, best_epoch, loss, now] + list(val_metrics.values()) 53 | test_row = [experiment, best_epoch, loss, now] + list(test_metrics.values()) 54 | append_new_metric(f"{decoder_type}Validation.csv", val_row) 55 | append_new_metric(f"{decoder_type}Test.csv", test_row) 56 | -------------------------------------------------------------------------------- /bleu.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from tqdm import tqdm 4 | from torch.nn.utils.rnn import pad_sequence 5 | 6 | 7 | def compute_average_bleu_over_dataset(model, dataloader, target_sos, target_eos, device): 8 | '''Determine the average BLEU score across sequences 9 | ''' 10 | with torch.no_grad(): 11 | total_score = [0,0,0,0] 12 | total_num = 0 13 | for data in tqdm(dataloader): 14 | torch.cuda.empty_cache() 15 | images, captions_ref, cap_lens = data 16 | captions_ref = pad_sequence(captions_ref, padding_value=target_eos) 17 | images = images.to(device) 18 | total_num += len(cap_lens) 19 | b_1 = model(images, on_max='halt') 20 | captions_cand = b_1[..., 0] 21 | batch_score = compute_batch_total_bleu(captions_ref, captions_cand, target_sos, target_eos) 22 | total_score = [total_score[i]+batch_score[i] for i in range(len(total_score))] 23 | 24 | total_score = [total_score[i]/total_num for i in range(len(total_score))] 25 | return total_score 26 | 27 | def compute_batch_total_bleu(captions_ref, captions_cand, target_sos, target_eos): 28 | '''Compute the total BLEU score over elements in a batch 29 | ''' 30 | with torch.no_grad(): 31 | refs = captions_ref.T 32 | cands = captions_cand.T 33 | refs_list = refs.tolist() 34 | cands_list = cands.tolist() 35 | for i in range(len(refs_list)): #Removes sos tags 36 | refs_list[i] = list(filter((target_sos).__ne__, refs_list[i])) 37 | cands_list[i] = list(filter((target_sos).__ne__, cands_list[i])) 38 | 39 | for i in range(len(refs_list)): #Removes eos tags 40 | refs_list[i] = list(filter((target_eos).__ne__, refs_list[i])) 41 | cands_list[i] = list(filter((target_eos).__ne__, cands_list[i])) 42 | 43 | total_bleu_scores = [0, 0, 0, 0] 44 | for i in range(refs.shape[0]): 45 | ref = refs_list[i] 46 | cand = cands_list[i] 47 | for n in range(len(total_bleu_scores)): 48 | score = BLEU_score(ref, cand, n+1) 49 | total_bleu_scores[n] += score 50 | return total_bleu_scores 51 | 52 | 53 | def grouper(seq, n): 54 | '''Extract all n-grams from a sequence 55 | ''' 56 | ngrams = [] 57 | for i in range(len(seq) - n + 1): 58 | ngrams.append(seq[i:i+n]) 59 | 60 | return ngrams 61 | 62 | 63 | def n_gram_precision(reference, candidate, n): 64 | '''Calculate the precision for a given order of n-gram 65 | ''' 66 | total_matches = 0 67 | ngrams_r = grouper(reference, n) 68 | ngrams_c = grouper(candidate, n) 69 | total_num = len(ngrams_c) 70 | assert total_num > 0 71 | for ngram_c in ngrams_c: 72 | if ngram_c in ngrams_r: 73 | total_matches += 1 74 | return total_matches/total_num 75 | 76 | 77 | 78 | def brevity_penalty(reference, candidate): 79 | '''Calculate the brevity penalty between a reference and candidate 80 | ''' 81 | if len(candidate) == 0: 82 | return 0 83 | if len(reference) <= len(candidate): 84 | return 1 85 | return np.exp(1 - (len(reference)/len(candidate))) 86 | 87 | 88 | 89 | def BLEU_score(reference, hypothesis, n): 90 | '''Calculate the BLEU score 91 | ''' 92 | bp = brevity_penalty(reference, hypothesis) 93 | prec = 1 94 | cand_len = min(n, len(hypothesis)) 95 | if(cand_len == 0): 96 | return 0 97 | for i in range(1, cand_len + 1): 98 | prec = prec * n_gram_precision(reference, hypothesis, i) 99 | prec = prec ** (1/n) 100 | return bp * prec -------------------------------------------------------------------------------- /experiments/transformerTest.csv: -------------------------------------------------------------------------------- 1 | experiment,epoch,loss,date,Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr 2 | resnet18_bs64_ft0_l3_h1,14,41.9603133544922,2020-04-17 05:43:16.706420,0.590760090145401,0.403517632815797,0.268718588462215,0.17595074387556,0.185485228567412,0.426227261029636,0.43356860562108 3 | resnet18_bs64_ft0_l5_h1,16,39.0515314941406,2020-04-17 08:04:03.627139,0.588656054805502,0.403763075390153,0.270567195128271,0.178123573238505,0.186498551752534,0.430254704077061,0.443502209540603 4 | resnet18_bs64_ft0_l7_h1,11,41.0975938557943,2020-04-17 10:16:25.287667,0.601001980963498,0.414493266364345,0.274843174714057,0.181628545399278,0.183420156509706,0.433460721440532,0.452379097161447 5 | resnet18_bs64_ft0_l3_h2,7,45.1613604410807,2020-04-17 11:51:25.789480,0.601001204926794,0.413147082136466,0.272151730307487,0.177298545486197,0.179166011448366,0.429179282961169,0.415719890196204 6 | resnet18_bs64_ft0_l5_h2,5,46.4478171061198,2020-04-17 13:35:40.651024,0.57245043940317,0.390380807413359,0.25618393577228,0.164004652802936,0.179101552846574,0.418983693295816,0.404707129010406 7 | resnet18_bs64_ft0_l7_h2,8,41.8883940755208,2020-04-17 17:48:32.246026,0.59677709093225,0.411418394520492,0.274774755299893,0.179147640897669,0.179012340003895,0.429063748006963,0.421230911214473 8 | resnet18_bs64_ft0_l3_h3,4,48.1328853759766,2020-04-17 19:22:37.715955,0.588561453947984,0.402376186151046,0.260502643086954,0.166788360798706,0.177312002773512,0.421704901801038,0.3870989940408 9 | resnet18_bs64_ft0_l5_h3,8,41.5873956298828,2020-04-17 21:51:52.000388,0.578904991948412,0.393981177706784,0.260806768860867,0.169273171415641,0.183595139172818,0.425692201652758,0.410287278330553 10 | resnet18_bs64_ft0_l7_h3,6,43.299091813151,2020-04-18 00:36:05.077389,0.598422633995037,0.411753559960007,0.272692890827601,0.177979528517566,0.182364480704372,0.429982741543483,0.442620412664826 11 | resnet18_bs64_ft1_l3_h1,14,41.4246530843099,2020-04-18 01:33:30.143870,0.60257466529345,0.417124165961719,0.278092797999739,0.182354485330053,0.187001505493405,0.435556680641723,0.45278107744311 12 | resnet18_bs64_ft1_l5_h1,19,35.2099123128255,2020-04-18 02:55:16.650848,0.595140873825991,0.402970171044902,0.264956844304415,0.170394547487963,0.186633902523106,0.431528266984938,0.44152153311907 13 | resnet18_bs64_ft1_l7_h1,12,38.9029117757161,2020-04-18 04:16:38.237405,0.602776939329852,0.415402387989405,0.280004181153279,0.186083972167834,0.192127590672247,0.441178029909471,0.465677264242045 14 | resnet18_bs64_ft1_l3_h2,8,42.2260498982747,2020-04-18 05:27:48.936433,0.600946372239684,0.417893039034945,0.280786201561938,0.186135945882614,0.187106731361568,0.435308143628296,0.462584754079035 15 | resnet18_bs64_ft1_l5_h2,8,39.5560412516276,2020-04-18 07:09:22.653572,0.597047510136127,0.409518381512124,0.269170534062667,0.172569549664265,0.184850487142842,0.430832100166568,0.444600593318906 16 | resnet18_bs64_ft1_l7_h2,9,39.0537100260417,2020-04-18 08:53:51.097100,0.587158686189099,0.405290053804633,0.267893054905449,0.174397703903901,0.186022030098844,0.433646091988821,0.441827658380919 17 | resnet18_bs64_ft1_l3_h3,5,43.9431692667643,2020-04-18 10:16:00.278314,0.602084741608549,0.418889356877793,0.279214451154056,0.183468041219053,0.18576437672978,0.432140558018395,0.438098470500037 18 | resnet18_bs64_ft1_l5_h3,11,35.8741467081706,2020-04-18 12:19:29.700306,0.582002182756168,0.390353330871233,0.253135005748273,0.163384658504711,0.184446781133837,0.423522752128783,0.426106675584052 19 | resnet18_bs64_ft1_l7_h3,6,41.7872472086589,2020-04-18 14:20:53.583437,0.589363384736992,0.40363580849473,0.264231312961828,0.1678261314713,0.188835436454669,0.434614816853617,0.439708162442991 20 | resnet50_bs32_ft0_l3_h1,3,42.2063491027832,2020-04-18 16:26:30.588523,0.60147700539023,0.416083565025119,0.280249125838435,0.187324435725625,0.186576307529272,0.439587128405537,0.455342987653431 21 | resnet50_bs32_ft1_l3_h1,4,36.0270101267497,2020-04-18 18:37:03.686163,0.603459732136358,0.412668195069197,0.269746575998155,0.172127641215543,0.189021548230023,0.434565902384892,0.462085794649507 22 | resnet101_bs32_ft0_l3_h1,1,52.9790090393066,2020-04-18 20:35:11.439308,0.594600505403599,0.408885056804159,0.269008319775991,0.17365102819736,0.180138326795706,0.424409692117878,0.421044405189417 23 | resnet101_bs32_ft1_l3_h1,3,37.5786676106771,2020-04-18 23:07:05.628863,0.615548347861279,0.434561926450564,0.295885454809378,0.194329938221477,0.199317194897066,0.451326752120744,0.504518899792966 24 | -------------------------------------------------------------------------------- /experiments/transformerValidation.csv: -------------------------------------------------------------------------------- 1 | experiment,epoch,loss,date,Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr 2 | resnet18_bs64_ft0_l3_h1,14,41.9603133544922,2020-04-17 05:43:16.706420,0.589013016295932,0.400188375165087,0.263286830936659,0.171329064638943,0.178018643236048,0.425673729068694,0.406473811492192 3 | resnet18_bs64_ft0_l5_h1,16,39.0515314941406,2020-04-17 08:04:03.627139,0.590286298568447,0.402942114328052,0.266602279140322,0.174268524796856,0.182132164877874,0.423604562543404,0.419139249052062 4 | resnet18_bs64_ft0_l7_h1,11,41.0975938557943,2020-04-17 10:16:25.287667,0.595721987825533,0.409956330978832,0.271270465104851,0.175049602194183,0.174456903000819,0.418732750145683,0.395834106098999 5 | resnet18_bs64_ft0_l3_h2,7,45.1613604410807,2020-04-17 11:51:25.789480,0.603541667756108,0.410285017723981,0.269332749198793,0.17153371938553,0.172082607297784,0.417483772518542,0.383447383970859 6 | resnet18_bs64_ft0_l5_h2,5,46.4478171061198,2020-04-17 13:35:40.651024,0.575463010334536,0.394366865796196,0.260744584606148,0.168227333331904,0.175673019543892,0.415767964591062,0.393451093081757 7 | resnet18_bs64_ft0_l7_h2,8,41.8883940755208,2020-04-17 17:48:32.246026,0.587544329720275,0.396155055633566,0.260094515763228,0.164701557164773,0.170705656417971,0.415658062650327,0.387475515995044 8 | resnet18_bs64_ft0_l3_h3,4,48.1328853759766,2020-04-17 19:22:37.715955,0.586564885496123,0.401638708039841,0.263255032697311,0.16671754581427,0.171966494316076,0.417716391509688,0.367661099614092 9 | resnet18_bs64_ft0_l5_h3,8,41.5873956298828,2020-04-17 21:51:52.000388,0.575875875875818,0.39233900146715,0.256014627839006,0.164764591167253,0.175336877044022,0.417032333671637,0.388809023552958 10 | resnet18_bs64_ft0_l7_h3,6,43.299091813151,2020-04-18 00:36:05.077389,0.59047015926365,0.404126352220292,0.266633804158014,0.172393709376665,0.172913498722021,0.419797880982603,0.393476949024959 11 | resnet18_bs64_ft1_l3_h1,14,41.4246530843099,2020-04-18 01:33:30.143870,0.591156325543857,0.407428351461491,0.272930891599101,0.177859122555463,0.181343720439741,0.431940840183773,0.437379154907403 12 | resnet18_bs64_ft1_l5_h1,19,35.2099123128255,2020-04-18 02:55:16.650848,0.580202681953058,0.394823027183161,0.262455013075478,0.172431666007569,0.178271143225984,0.41991643613818,0.399942715077044 13 | resnet18_bs64_ft1_l7_h1,12,38.9029117757161,2020-04-18 04:16:38.237405,0.577727585190948,0.394998185453191,0.261606028302783,0.168964816932327,0.181185334114389,0.426205621748728,0.417851907623819 14 | resnet18_bs64_ft1_l3_h2,8,42.2260498982747,2020-04-18 05:27:48.936433,0.605383747631704,0.418895649643492,0.278079246010572,0.178755271159084,0.180698644398809,0.42846892653866,0.411384989694934 15 | resnet18_bs64_ft1_l5_h2,8,39.5560412516276,2020-04-18 07:09:22.653572,0.591422279355107,0.406246531838327,0.26861360113065,0.173151818558772,0.178683107200875,0.423796812868835,0.422884096222352 16 | resnet18_bs64_ft1_l7_h2,9,39.0537100260417,2020-04-18 08:53:51.097100,0.585126425384176,0.401827409159162,0.264920921513326,0.169831030449903,0.179420122353495,0.42518069507969,0.423444742929435 17 | resnet18_bs64_ft1_l3_h3,5,43.9431692667643,2020-04-18 10:16:00.278314,0.597719188114606,0.413155043470736,0.277971272724479,0.182539563688212,0.180615626777468,0.427873269463185,0.430348533019964 18 | resnet18_bs64_ft1_l5_h3,11,35.8741467081706,2020-04-18 12:19:29.700306,0.577513620604202,0.385714449304047,0.251348463868765,0.161638211501512,0.181368067978516,0.419807734834959,0.407615236330005 19 | resnet18_bs64_ft1_l7_h3,6,41.7872472086589,2020-04-18 14:20:53.583437,0.574134039839017,0.392802046537895,0.258968630189222,0.165549603437711,0.183229068065627,0.421299406621612,0.414268190834623 20 | resnet50_bs32_ft0_l3_h1,3,42.2063491027832,2020-04-18 16:26:30.588523,0.60237480640159,0.41314670678871,0.277873596237883,0.184878093652362,0.182540162977088,0.431969296333947,0.453218791380881 21 | resnet50_bs32_ft1_l3_h1,4,36.0270101267497,2020-04-18 18:37:03.686163,0.594914080167896,0.405624611924308,0.267524812997259,0.173873688799283,0.181904376571545,0.425074022969164,0.437307259421733 22 | resnet101_bs32_ft0_l3_h1,1,52.9790090393066,2020-04-18 20:35:11.439308,0.595701069843399,0.41060048083779,0.272004032093772,0.179041274937953,0.177687748908928,0.423993168534624,0.404554644964696 23 | resnet101_bs32_ft1_l3_h1,3,37.5786676106771,2020-04-18 23:07:05.628863,0.613296736809941,0.427229048292185,0.289890819332909,0.192202549799963,0.194992692478327,0.442398104274479,0.473963516007388 24 | -------------------------------------------------------------------------------- /experiments/rnnTest.csv: -------------------------------------------------------------------------------- 1 | experiment,epoch,loss,date,Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr 2 | resnet18_h512_bs64_ft1_aap1,5,188.24726028645833,2020-04-13 18:39:53.063549,0.6052228725798644,0.4237457169772124,0.29227262013869176,0.20072686359100655,0.20140325435562006,0.45747653990692266,0.49566223505300944 3 | resnet18_h1024_bs64_ft0_aap1,10,183.98917141927083,2020-04-13 19:15:36.257830,0.5974674513999824,0.4146283483874521,0.2837159365371284,0.19289965877335385,0.2034351216907056,0.4536161729360475,0.4953083547144631 4 | resnet18_h512_bs64_ft0_aap1,14,184.31729609375,2020-04-13 19:16:50.046402,0.5856969906606204,0.4036659274072851,0.27389491883040396,0.18324970385198092,0.19989131339589852,0.44936794572476924,0.4771058606264396 5 | resnet101_h512_bs64_ft0_aap1,7,189.10871438802084,2020-04-13 19:56:08.234227,0.6072661217074834,0.43064579135102526,0.295453336113155,0.20169694692140946,0.2045549650045583,0.4609968590184625,0.5101445051024974 6 | resnet50_h512_bs32_ft1_aap1,5,190.13573194986978,2020-04-13 19:59:19.375481,0.626872365234494,0.45086017721420796,0.31863281523834086,0.22119287023190892,0.21422123714979732,0.47806040765956187,0.5513217999735498 7 | resnet18_h256_bs64_ft0_aap1,25,183.74132005208332,2020-04-13 20:01:58.462543,0.589038681822135,0.41038851372505725,0.280720761697694,0.1904003332008969,0.1992879984960454,0.4483956375636836,0.4830933401805442 8 | resnet101_h512_bs32_ft1_aap1,6,188.3065055501302,2020-04-13 20:19:54.750396,0.6287265366212156,0.4513231838636786,0.31470947360692053,0.21466313717258104,0.21583533232445778,0.4815374029819684,0.5747499706538368 9 | resnet50_h512_bs64_ft0_aap1,20,182.96988059895833,2020-04-13 21:38:25.208311,0.5995115995115472,0.42342899989325067,0.29255580334580705,0.2006467417429637,0.20790367062067994,0.4562985328064483,0.5265168796463033 10 | resnet18_h512_bs64_ft0_aap1_smoothing0,14,184.31729609375,2020-04-13 22:28:00.906454,0.5856969906606204,0.4036659274072851,0.27389491883040396,0.18324970385198092,0.19989131339589852,0.44936794572476924,0.4771058606264396 11 | resnet50_h512_bs64_ft1_aap1_smoothing0,5,190.13573194986978,2020-04-14 01:07:00.637244,0.626872365234494,0.45086017721420796,0.31863281523834086,0.22119287023190892,0.21422123714979732,0.47806040765956187,0.5513217999735498 12 | resnet50_h512_bs64_ft0_aap1_smoothing0,20,182.96988059895833,2020-04-14 02:27:28.392280,0.5995115995115472,0.42342899989325067,0.29255580334580705,0.2006467417429637,0.20790367062067994,0.4562985328064483,0.5265168796463033 13 | resnet101_h512_bs64_ft1_aap1_smoothing0,6,188.3065055501302,2020-04-14 04:08:49.039776,0.6287265366212156,0.4513231838636786,0.31470947360692053,0.21466313717258104,0.21583533232445778,0.4815374029819684,0.5747499706538368 14 | resnet101_h512_bs64_ft0_aap1_smoothing0,7,189.10871438802084,2020-04-14 05:13:59.726771,0.6072661217074834,0.43064579135102526,0.295453336113155,0.20169694692140946,0.2045549650045583,0.4609968590184625,0.5101445051024974 15 | resnet18_h1024_bs64_ft0_aap1_smoothing0,10,183.98917141927083,2020-04-14 06:07:46.182246,0.5974674513999824,0.4146283483874521,0.2837159365371284,0.19289965877335385,0.2034351216907056,0.4536161729360475,0.4953083547144631 16 | resnet18_h512_bs64_ft1_aap1_smoothing0,5,188.24726028645833,2020-04-14 06:40:54.209452,0.6052228725798644,0.4237457169772124,0.29227262013869176,0.20072686359100655,0.20140325435562006,0.45747653990692266,0.49566223505300944 17 | resnet18_h256_bs64_ft0_aap1_smoothing0,25,183.74132005208332,2020-04-14 09:27:42.048072,0.589038681822135,0.41038851372505725,0.280720761697694,0.1904003332008969,0.1992879984960454,0.4483956375636836,0.4830933401805442 18 | resnet50_h256_bs32_ft1_aap1_smoothing0,7,188.03681915690103,2020-04-15 01:38:43.019344,0.6232506064563703,0.4445173909944001,0.3093389510567379,0.2127331040724113,0.2090553242028222,0.4733830171877116,0.541695333254249 19 | resnet50_h256_bs64_ft0_aap1_smoothing0,29,181.13862532552082,2020-04-15 04:01:17.812167,0.6000704411375715,0.42332516767831607,0.29079851927703954,0.19787502073653598,0.207608432291816,0.4583206913930424,0.5275711654104777 20 | resnet101_h256_bs32_ft1_aap1_smoothing0,12,182.92494092610676,2020-04-15 05:35:41.585121,0.62515533463513,0.44623868499234925,0.3105762944847402,0.21356513319272058,0.21751299931842322,0.47445252473668253,0.5675194669000582 21 | resnet50_h1024_bs32_ft1_aap1_smoothing0,6,187.2109516357422,2020-04-15 08:37:29.503812,0.6298507462685979,0.45265427305588446,0.3190374036647675,0.22040304587782789,0.21361684851895416,0.47562354043899013,0.55452665680076 22 | resnet101_h256_bs64_ft0_aap1_smoothing0,21,182.71220348307293,2020-04-15 08:39:20.754660,0.6050655811849294,0.42424662255611173,0.2928086446540554,0.19917938134098265,0.20602623357529448,0.4602990228916631,0.535179593425173 23 | resnet101_h1024_bs32_ft1_aap1_smoothing0,6,186.54643675130208,2020-04-15 11:53:01.354201,0.6299926713081138,0.45412326054391466,0.3185871113207865,0.21985872870069223,0.21920573170030355,0.4805201468023562,0.5864158826422582 24 | resnet50_h1024_bs32_ft0_aap1_smoothing0,6,190.4266637532552,2020-04-15 12:14:54.934210,0.6068169618893714,0.42977221959111583,0.2995792260670942,0.20796969787683683,0.20622701715264827,0.45900609334004683,0.5187667816420519 25 | resnet101_h1024_bs32_ft0_aap1_smoothing0,12,185.05426982421875,2020-04-15 16:00:34.066662,0.6072846538925766,0.4277179415982155,0.2962950907425172,0.20249712732873013,0.20993342824920733,0.4625954551185498,0.5239224290042607 26 | -------------------------------------------------------------------------------- /experiments/rnnValidation.csv: -------------------------------------------------------------------------------- 1 | experiment,epoch,loss,date,Bleu_1,Bleu_2,Bleu_3,Bleu_4,METEOR,ROUGE_L,CIDEr 2 | resnet18_h512_bs64_ft1_aap1,5,188.24726028645833,2020-04-13 18:39:53.063549,0.6031289336449737,0.42039494809993677,0.28870136346409586,0.19388779636250378,0.19573772630790165,0.45281774669635216,0.4655339763183884 3 | resnet18_h1024_bs64_ft0_aap1,10,183.98917141927083,2020-04-13 19:15:36.257830,0.5839944207130516,0.4040374583284246,0.2758660379683029,0.18771519942985737,0.19409318413292842,0.441779323038474,0.45690819408864775 4 | resnet18_h512_bs64_ft0_aap1,14,184.31729609375,2020-04-13 19:16:50.046402,0.5878845489111141,0.4109508108761784,0.2799084702226398,0.18729567939907385,0.19588050544456845,0.4491274433618033,0.4532399034070124 5 | resnet101_h512_bs64_ft0_aap1,7,189.10871438802084,2020-04-13 19:56:08.234227,0.5999635668093086,0.41978492366243064,0.2883803311328979,0.19519733928596,0.19454199108389594,0.44856289388373805,0.4732483794071278 6 | resnet50_h512_bs32_ft1_aap1,5,190.13573194986978,2020-04-13 19:59:19.375481,0.6167712291242717,0.44261238171324374,0.3121858283431722,0.21499156678754866,0.20782968969787521,0.4711479901546088,0.5291669836114353 7 | resnet18_h256_bs64_ft0_aap1,25,183.74132005208332,2020-04-13 20:01:58.462543,0.5818827401056047,0.4059484391770034,0.2792831460665825,0.1901557495786352,0.19284608483582932,0.44170743559300535,0.4528758575421204 8 | resnet101_h512_bs32_ft1_aap1,6,188.3065055501302,2020-04-13 20:19:54.750396,0.6291451731760658,0.4499784096230112,0.31220601560459715,0.21144285140547492,0.209406092061908,0.4693409602276907,0.5411281699394719 9 | resnet50_h512_bs64_ft0_aap1,20,182.96988059895833,2020-04-13 21:38:25.208311,0.5996471107189589,0.4188048533002812,0.28767043810821685,0.19659408515040436,0.20350155685748064,0.45635364522348476,0.5081529496198199 10 | resnet18_h512_bs64_ft0_aap1_smoothing0,14,184.31729609375,2020-04-13 22:28:00.906454,0.5878845489111141,0.4109508108761784,0.2799084702226398,0.18729567939907385,0.19588050544456845,0.4491274433618033,0.4532399034070124 11 | resnet50_h512_bs64_ft1_aap1_smoothing0,5,190.13573194986978,2020-04-14 01:07:00.637244,0.6167712291242717,0.44261238171324374,0.3121858283431722,0.21499156678754866,0.20782968969787521,0.4711479901546088,0.5291669836114353 12 | resnet50_h512_bs64_ft0_aap1_smoothing0,20,182.96988059895833,2020-04-14 02:27:28.392280,0.5996471107189589,0.4188048533002812,0.28767043810821685,0.19659408515040436,0.20350155685748064,0.45635364522348476,0.5081529496198199 13 | resnet101_h512_bs64_ft1_aap1_smoothing0,6,188.3065055501302,2020-04-14 04:08:49.039776,0.6291451731760658,0.4499784096230112,0.31220601560459715,0.21144285140547492,0.209406092061908,0.4693409602276907,0.5411281699394719 14 | resnet101_h512_bs64_ft0_aap1_smoothing0,7,189.10871438802084,2020-04-14 05:13:59.726771,0.5999635668093086,0.41978492366243064,0.2883803311328979,0.19519733928596,0.19454199108389594,0.44856289388373805,0.4732483794071278 15 | resnet18_h1024_bs64_ft0_aap1_smoothing0,10,183.98917141927083,2020-04-14 06:07:46.182246,0.5839944207130516,0.4040374583284246,0.2758660379683029,0.18771519942985737,0.19409318413292842,0.441779323038474,0.45690819408864775 16 | resnet18_h512_bs64_ft1_aap1_smoothing0,5,188.24726028645833,2020-04-14 06:40:54.209452,0.6031289336449737,0.42039494809993677,0.28870136346409586,0.19388779636250378,0.19573772630790165,0.45281774669635216,0.4655339763183884 17 | resnet18_h256_bs64_ft0_aap1_smoothing0,25,183.74132005208332,2020-04-14 09:27:42.048072,0.5818827401056047,0.4059484391770034,0.2792831460665825,0.1901557495786352,0.19284608483582932,0.44170743559300535,0.4528758575421204 18 | resnet50_h256_bs32_ft1_aap1_smoothing0,7,188.03681915690103,2020-04-15 01:38:43.019344,0.621570660884263,0.44261822630813513,0.3095402055060532,0.2123988062926863,0.20534168672694564,0.4624641558333573,0.5186199913332841 19 | resnet50_h256_bs64_ft0_aap1_smoothing0,29,181.13862532552082,2020-04-15 04:01:17.812167,0.5934472432622341,0.41824354089143817,0.2878970830972872,0.19715974547536996,0.20220779817325435,0.453599722155344,0.517522374527812 20 | resnet101_h256_bs32_ft1_aap1_smoothing0,12,182.92494092610676,2020-04-15 05:35:41.585121,0.6236107464775861,0.45219594019943765,0.3194573836425027,0.22219106301491584,0.2130224916345935,0.47495854287982286,0.5537375312682122 21 | resnet50_h1024_bs32_ft1_aap1_smoothing0,6,187.2109516357422,2020-04-15 08:37:29.503812,0.6252759381897879,0.4478755109209526,0.3135183647630413,0.21570352902230294,0.20778225899706884,0.4655427257801623,0.5313040936089171 22 | resnet101_h256_bs64_ft0_aap1_smoothing0,21,182.71220348307293,2020-04-15 08:39:20.754660,0.5998907302858678,0.4219100008277755,0.2907341793620462,0.19954734092479318,0.19952477672126842,0.45431576136559876,0.501783775881331 23 | resnet101_h1024_bs32_ft1_aap1_smoothing0,6,186.54643675130208,2020-04-15 11:53:01.354201,0.6215877561979216,0.4474710304769794,0.31551965341484706,0.21931763812829472,0.21556081550199505,0.47026928419915703,0.5522572820316625 24 | resnet50_h1024_bs32_ft0_aap1_smoothing0,6,190.4266637532552,2020-04-15 12:14:54.934210,0.6046972014756947,0.4231115431549454,0.2901947731395462,0.19768504241729207,0.1988094239959793,0.4510896788205646,0.49466485789718084 25 | resnet101_h1024_bs32_ft0_aap1_smoothing0,12,185.05426982421875,2020-04-15 16:00:34.066662,0.5937823834196378,0.415695827086098,0.28534300326433887,0.1958184264586681,0.20363652705133783,0.45437055400947635,0.49981683373720265 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hyperparameter Analysis for Image Captioning 2 | 3 | We perform a thorough sensitivity analysis on state-of-the-art image captioning approaches using two different architectures: CNN+LSTM and CNN+Transformer. Experiments were carried out using the Flickr8k dataset. The biggest takeaway from the experiments is that fine-tuning the CNN encoder outperforms the baseline and all other experiments carried out for both architectures. A detailed paper for this project is available here: https://github.com/aravindvarier/Image-Captioning-Pytorch/blob/master/Hyperparameter_Analysis_for_Image_Captioning.pdf 4 | 5 | If you have any questions related to this, please reach out to us by creating an issue on this repository or through our emails listed in the paper. 6 | 7 | ## Getting Started 8 | 9 | These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. 10 | 11 | ### Prerequisites 12 | 1. Download the [Flickr8k Dataset](https://www.kaggle.com/shadabhussain/flickr8k) and place it under `dataset` folder of this directory. 13 | 2. Execute the following commands in this folder to set up the require virtual environment for running these experiments. 14 | 15 | ``` 16 | python3 -m venv proj_env 17 | source proj_env/bin/activate 18 | pip install -r requirements.txt 19 | ``` 20 | 21 | 3. Generate the Vocab file: 22 | ``` 23 | python vocab_builder.py 24 | ``` 25 | 26 | ## Running the experiments 27 | Please execute the following commands in order to reproduce the results discussed in this paper. Please note that the results of the experiment is stored as csv files under `/experiments` folder and gets updated automatically once an experiment has been executed successfully. 28 | 29 | ### CNN + LSTM 30 | There were a total of 3 experiments performed for this architecture. 31 | 32 | 1. Effect of larger CNN models on caption quality (ResNet18, ResNet50, and ResNet101): 33 | ``` 34 | python train.py --encoder-type resnet18 --experiment-name resnet18_h512_bs64_ft0 35 | python train.py --encoder-type resnet50 --experiment-name resnet50_h512_bs64_ft0 36 | python train.py --encoder-type resnet101 --experiment-name resnet101_h512_bs64_ft0 37 | ``` 38 | 39 | 2. Effect of finetuning on caption quality (ResNet18, ResNet50, and ResNet101): 40 | ``` 41 | python train.py --encoder-type resnet18 --experiment-name resnet18_h512_bs64_ft1 --fine-tune 1 42 | python train.py --encoder-type resnet50 --experiment-name resnet50_h512_bs32_ft1 --fine-tune 1 --batch-size 32 43 | python train.py --encoder-type resnet101 --experiment-name resnet101_h512_bs32_ft1 --fine-tune 1 --batch-size 32 44 | ``` 45 | 46 | 3. Effect of varying LSTM units (keeping encoder fixed and varying decoder): 47 | 48 | * Using ResNet18: 49 | ``` 50 | python train.py --decoder-hidden-size 256 --encoder-type resnet18 --experiment-name resnet18_h256_bs64_ft0 51 | python train.py --decoder-hidden-size 512 --encoder-type resnet18 --experiment-name resnet18_h512_bs64_ft0 52 | python train.py --decoder-hidden-size 1024 --encoder-type resnet18 --experiment-name resnet18_h1024_bs64_ft0 53 | ``` 54 | 55 | * Using ResNet50: 56 | ``` 57 | python train.py --decoder-hidden-size 256 --encoder-type resnet50 --experiment-name resnet50_h256_bs64_ft0 58 | python train.py --decoder-hidden-size 512 --encoder-type resnet50 --experiment-name resnet50_h512_bs64_ft0 59 | python train.py --decoder-hidden-size 1024 --encoder-type resnet50 --experiment-name resnet50_h1024_bs32_ft0 --batch-size 32 60 | ``` 61 | 62 | * Using ResNet101: 63 | ``` 64 | python train.py --decoder-hidden-size 256 --encoder-type resnet101 --experiment-name resnet101_h256_bs64_ft0 65 | python train.py --decoder-hidden-size 512 --encoder-type resnet101 --experiment-name resnet101_h512_bs64_ft0 66 | python train.py --decoder-hidden-size 1024 --encoder-type resnet101 --experiment-name resnet101_h1024_bs32_ft0 --batch-size 32 67 | ``` 68 | 69 | ### CNN + Transformer 70 | There were a total of 3 experiments performed for this architecture. 71 | 72 | 1. Effect of larger CNN models on caption quality (ResNet18, ResNet50, and ResNet101): 73 | ``` 74 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 1 --num-tf-layers 3 --experiment-name resnet18_bs64_ft0_l3_h1 75 | python train.py --encoder-type resnet50 --decoder-type transformer --num-heads 1 --num-tf-layers 3 --experiment-name resnet18_bs64_ft0_l3_h1 76 | python train.py --encoder-type resnet101 --decoder-type transformer --num-heads 1 --num-tf-layers 3 --experiment-name resnet18_bs64_ft0_l3_h1 77 | ``` 78 | 79 | 2. Effect of finetuning on caption quality (ResNet18, ResNet50, and ResNet101): 80 | ``` 81 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 1 --num-tf-layers 3 --fine-tune 1 --experiment-name resnet18_bs64_ft1_l3_h1 82 | python train.py --encoder-type resnet50 --decoder-type transformer --num-heads 1 --num-tf-layers 3 --fine-tune 1 --experiment-name resnet18_bs64_ft1_l3_h1 83 | python train.py --encoder-type resnet101 --decoder-type transformer --num-heads 1 --num-tf-layers 3 --fine-tune 1 --experiment-name resnet18_bs64_ft1_l3_h1 84 | ``` 85 | 86 | 3. Effect of varying number of transformer layers and heads (keeping encoder fixed as ResNet18 and varying decoder): 87 | 88 | * Using 1 Head: 89 | ``` 90 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 1 --num-tf-layers 3 --experiment-name resnet18_bs64_ft0_l3_h1 91 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 1 --num-tf-layers 5 --experiment-name resnet18_bs64_ft0_l5_h1 92 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 1 --num-tf-layers 7 --experiment-name resnet18_bs64_ft0_l7_h1 93 | ``` 94 | 95 | * Using 2 Head: 96 | ``` 97 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 2 --num-tf-layers 3 --experiment-name resnet18_bs64_ft0_l3_h2 98 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 2 --num-tf-layers 5 --experiment-name resnet18_bs64_ft0_l5_h2 99 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 2 --num-tf-layers 7 --experiment-name resnet18_bs64_ft0_l7_h2 100 | ``` 101 | * Using 3 Head: 102 | ``` 103 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 3 --num-tf-layers 3 --experiment-name resnet18_bs64_ft0_l3_h3 104 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 3 --num-tf-layers 5 --experiment-name resnet18_bs64_ft0_l5_h3 105 | python train.py --encoder-type resnet18 --decoder-type transformer --num-heads 3 --num-tf-layers 7 --experiment-name resnet18_bs64_ft0_l7_h3 106 | ``` 107 | 108 | 109 | ## Results Visualization Notebooks 110 | 1. CNN+LSTM: https://github.com/aravindvarier/Image-Captioning-Pytorch/blob/master/experiments/CNN%2BLSTM_Results.ipynb 111 | 2. CNN+Transformer: https://github.com/aravindvarier/Image-Captioning-Pytorch/blob/master/experiments/CNN%2BTransformer_Results.ipynb 112 | 113 | ## Built With 114 | 115 | * [PyTorch](https://pytorch.org/) 116 | -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.utils.data import Dataset 3 | from torchvision import transforms 4 | import os 5 | from PIL import Image 6 | from tqdm import tqdm 7 | import warnings 8 | from nltk.translate.bleu_score import corpus_bleu 9 | from nltk.translate.meteor_score import meteor_score 10 | import nltk 11 | nltk.download('wordnet') 12 | import utils 13 | 14 | from pycocoevalcap.bleu.bleu import Bleu 15 | from pycocoevalcap.rouge.rouge import Rouge 16 | from pycocoevalcap.cider.cider import Cider 17 | from pycocoevalcap.meteor.meteor import Meteor 18 | 19 | 20 | 21 | class TestDataset(Dataset): 22 | """Flickr8k dataset.""" 23 | 24 | def __init__(self, img_dir, split_dir, ann_dir, vocab_file, transform=None): 25 | """ 26 | Args: 27 | img_dir (string): Directory with all the images. 28 | ann_dir (string): Directory with all the tokens 29 | split_dir (string): Directory with all the file names which belong to a certain split(train/dev/test) 30 | vocab_file (string): File which has the entire vocabulary of the dataset. 31 | transform (callable, optional): Optional transform to be applied 32 | on a sample. 33 | """ 34 | 35 | self.img_dir = img_dir 36 | self.ann_dir = ann_dir 37 | self.split_dir = split_dir 38 | self.SOS = self.EOS = None 39 | self.vocab = None 40 | self.vocab_size = None 41 | self.images = self.captions = [] 42 | self.all_captions = {} 43 | self.preprocess_files(self.split_dir, self.ann_dir, vocab_file) 44 | 45 | if(transform == None): 46 | self.transform = transforms.Compose([ 47 | transforms.Resize((224,224)), 48 | # transforms.CenterCrop(224), 49 | transforms.ToTensor(), 50 | transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) 51 | ]) 52 | 53 | 54 | def preprocess_files(self, split_dir, ann_dir, vocab_file): 55 | # all_captions = {} 56 | 57 | with open(split_dir, "r") as split_f: 58 | sub_lines = split_f.readlines() 59 | 60 | with open(ann_dir, "r") as ann_f: 61 | for line in ann_f: 62 | if line.split("#")[0] + "\n" in sub_lines: 63 | image_file = line.split('#')[0] 64 | caption = utils.clean_description(line.replace("-", " ").split()[1:]) 65 | if image_file in self.all_captions: 66 | self.all_captions[image_file].append(caption) 67 | else: 68 | self.all_captions[image_file] = [caption] 69 | 70 | self.images = list(self.all_captions.keys()) 71 | self.captions = list(self.all_captions.values()) 72 | assert(len(self.images) == len(self.captions)) 73 | assert(len(self.captions[-1]) == 5) 74 | vocab = [] 75 | with open(vocab_file, "r") as vocab_f: 76 | for line in vocab_f: 77 | vocab.append(line.strip()) 78 | 79 | self.vocab_size = len(vocab) + 2 #The +2 is to accomodate for the SOS and EOS 80 | self.SOS = 0 81 | self.EOS = self.vocab_size - 1 82 | self.vocab = vocab 83 | 84 | def __len__(self): 85 | return len(self.images) 86 | 87 | def __getitem__(self, idx): 88 | if torch.is_tensor(idx): 89 | idx = idx.tolist() 90 | 91 | img_name, caps = self.images[idx], self.captions[idx] 92 | img_name = os.path.join(self.img_dir, 93 | img_name) 94 | image = Image.open(img_name) 95 | if self.transform: 96 | image = self.transform(image) 97 | 98 | return {'image': image, 'captions': caps} 99 | 100 | def collater(batch): 101 | images = torch.stack([item['image'] for item in batch]) 102 | all_caps = [item['captions'] for item in batch] 103 | 104 | return images, all_caps 105 | 106 | def get_output_sentence(model, device, images, vocab): 107 | # hypotheses = [] 108 | with torch.no_grad(): 109 | torch.cuda.empty_cache() 110 | 111 | images = images.to(device) 112 | target_eos = len(vocab) + 1 113 | target_sos = 0 114 | 115 | b_1 = model(images, on_max='halt') 116 | captions_cand = b_1[..., 0] 117 | 118 | cands = captions_cand.T 119 | cands_list = cands.tolist() 120 | for i in range(len(cands_list)): #Removes sos tags 121 | cands_list[i] = list(filter((target_sos).__ne__, cands_list[i])) 122 | cands_list[i] = list(filter((target_eos).__ne__, cands_list[i])) 123 | 124 | # hypotheses += cands_list 125 | 126 | return cands_list 127 | 128 | 129 | def score(ref, hypo): 130 | """ 131 | ref, dictionary of reference sentences (id, sentence) 132 | hypo, dictionary of hypothesis sentences (id, sentence) 133 | score, dictionary of scores 134 | """ 135 | scorers = [ 136 | (Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]), 137 | (Meteor(),"METEOR"), 138 | (Rouge(), "ROUGE_L"), 139 | (Cider(), "CIDEr") 140 | ] 141 | final_scores = {} 142 | for scorer, method in scorers: 143 | score, scores = scorer.compute_score(ref, hypo) 144 | if type(score) == list: 145 | for m, s in zip(method, score): 146 | final_scores[m] = s 147 | else: 148 | final_scores[method] = score 149 | return final_scores 150 | 151 | def get_references_and_hypotheses(model, device, dataset, dataloader): 152 | references = [] 153 | hypotheses = [] 154 | assert(len(dataset.captions) == len(dataset.images)) 155 | with torch.no_grad(): 156 | for data in tqdm(dataloader): 157 | torch.cuda.empty_cache() 158 | images, captions = data 159 | 160 | references += captions 161 | hypotheses += get_output_sentence(model, device, images, dataset.vocab) 162 | 163 | for i in range(len(references)): 164 | hypotheses[i] = " ".join([dataset.vocab[j - 1] for j in hypotheses[i]]) 165 | 166 | assert(len(references) == len(hypotheses)) 167 | 168 | return references, hypotheses 169 | 170 | def get_pycoco_metrics(model, device, dataset, dataloader): 171 | references, hypotheses = get_references_and_hypotheses(model, device, dataset, dataloader) 172 | 173 | hypo = {idx: [h] for idx, h in enumerate(hypotheses)} 174 | ref = {idx: [" ".join(l) for l in r] for idx, r in enumerate(references)} 175 | 176 | metrics = score(ref, hypo) 177 | 178 | return metrics 179 | 180 | 181 | def print_metrics(model, device, dataset, dataloader): 182 | references, hypotheses = get_references_and_hypotheses(model, device, dataset, dataloader) 183 | 184 | # bleu scores 185 | bleu_1 = corpus_bleu(references, hypotheses, weights=(1, 0, 0, 0)) 186 | bleu_2 = corpus_bleu(references, hypotheses, weights=(0.5, 0.5, 0, 0)) 187 | bleu_3 = corpus_bleu(references, hypotheses, weights=(0.33, 0.33, 0.33, 0)) 188 | bleu_4 = corpus_bleu(references, hypotheses) 189 | 190 | print('BLEU-1 ({})\t' 191 | 'BLEU-2 ({})\t' 192 | 'BLEU-3 ({})\t' 193 | 'BLEU-4 ({})\t'.format(bleu_1, bleu_2, bleu_3, bleu_4)) 194 | 195 | # meteor score 196 | total_m_score = 0.0 197 | 198 | for i in range(len(references)): 199 | actual = [" ".join(ref) for ref in references[i]] 200 | total_m_score += meteor_score(actual, " ".join(hypotheses[i])) 201 | 202 | m_score = total_m_score/len(references) 203 | 204 | print('Meteor Score: {}'.format(m_score)) 205 | 206 | metrics = { 207 | 'bleu_1': bleu_1, 208 | 'bleu_2': bleu_2, 209 | 'bleu_3': bleu_3, 210 | 'bleu_4': bleu_4, 211 | 'meteor': m_score 212 | } 213 | 214 | return metrics 215 | 216 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.utils.data import Dataset, DataLoader 3 | from torch.nn.utils.rnn import pad_sequence 4 | from torchvision import transforms 5 | import torchvision.models as models 6 | import torch.nn as nn 7 | import os 8 | # import matplotlib.pyplot as plt 9 | from PIL import Image 10 | from tqdm import tqdm 11 | import warnings 12 | import eval 13 | import bleu 14 | import utils 15 | import string 16 | import copy 17 | import argparse 18 | 19 | from models import * 20 | 21 | img_dir = './dataset/Flickr8k_Dataset/' 22 | ann_dir = './dataset/Flickr8k_text/Flickr8k.token.txt' 23 | train_dir = './dataset/Flickr8k_text/Flickr_8k.trainImages.txt' 24 | val_dir = './dataset/Flickr8k_text/Flickr_8k.devImages.txt' 25 | test_dir = './dataset/Flickr8k_text/Flickr_8k.testImages.txt' 26 | 27 | vocab_file = './vocab.txt' 28 | 29 | SEED = 123 30 | torch.manual_seed(SEED) 31 | 32 | torch.backends.cudnn.deterministic = True 33 | torch.backends.cudnn.benchmark = False 34 | 35 | class Flickr8kDataset(Dataset): 36 | """Flickr8k dataset.""" 37 | 38 | def __init__(self, img_dir, split_dir, ann_dir, vocab_file, transform=None): 39 | """ 40 | Args: 41 | img_dir (string): Directory with all the images. 42 | ann_dir (string): Directory with all the tokens 43 | split_dir (string): Directory with all the file names which belong to a certain split(train/dev/test) 44 | vocab_file (string): File which has the entire vocabulary of the dataset. 45 | transform (callable, optional): Optional transform to be applied 46 | on a sample. 47 | """ 48 | 49 | self.img_dir = img_dir 50 | self.ann_dir = ann_dir 51 | self.split_dir = split_dir 52 | self.SOS = self.EOS = None 53 | self.word_2_token = None 54 | self.vocab_size = None 55 | self.image_file_names, self.captions, self.tokenized_captions= self.tokenizer(self.split_dir, self.ann_dir) 56 | 57 | if(transform == None): 58 | self.transform = transforms.Compose([ 59 | transforms.Resize((224,224)), 60 | # transforms.CenterCrop(224), 61 | transforms.ToTensor(), 62 | transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) 63 | ]) 64 | 65 | def tokenizer(self, split_dir, ann_dir): 66 | image_file_names = [] 67 | captions = [] 68 | tokenized_captions = [] 69 | 70 | with open(split_dir, "r") as split_f: 71 | sub_lines = split_f.readlines() 72 | 73 | with open(ann_dir, "r") as ann_f: 74 | for line in ann_f: 75 | if line.split("#")[0] + "\n" in sub_lines: 76 | caption = utils.clean_description(line.replace("-", " ").split()[1:]) 77 | image_file_names.append(line.split()[0]) 78 | captions.append(caption) 79 | 80 | 81 | vocab = [] 82 | # for caption in captions: 83 | # for word in caption: 84 | # if word not in vocab: 85 | # vocab.append(word) 86 | with open(vocab_file, "r") as vocab_f: 87 | for line in vocab_f: 88 | vocab.append(line.strip()) 89 | 90 | self.vocab_size = len(vocab) + 2 #The +2 is to accomodate for the SOS and EOS 91 | self.SOS = 0 92 | self.EOS = self.vocab_size - 1 93 | 94 | 95 | self.word_2_token = dict(zip(vocab, list(range(1, self.vocab_size - 1)))) 96 | 97 | for caption in captions: 98 | temp = [] 99 | for word in caption: 100 | temp.append(self.word_2_token[word]) 101 | temp.insert(0, self.SOS) 102 | temp.append(self.EOS) 103 | tokenized_captions.append(temp) 104 | 105 | assert(len(image_file_names) == len(captions)) 106 | 107 | return image_file_names, captions, tokenized_captions 108 | 109 | 110 | def __len__(self): 111 | return len(self.image_file_names) 112 | 113 | def __getitem__(self, idx): 114 | if torch.is_tensor(idx): 115 | idx = idx.tolist() 116 | 117 | img_name, cap_tok, caption = self.image_file_names[idx], self.tokenized_captions[idx], self.captions[idx] 118 | img_name, instance = img_name.split('#') 119 | img_name = os.path.join(self.img_dir, 120 | img_name) 121 | image = Image.open(img_name) 122 | if self.transform: 123 | image = self.transform(image) 124 | cap_tok = torch.tensor(cap_tok) 125 | sample = {'image': image, 'cap_tok': cap_tok, 'caption': caption} 126 | 127 | 128 | 129 | return sample 130 | 131 | 132 | 133 | 134 | def collater(batch): 135 | '''This functions pads the cpations and makes them equal length 136 | ''' 137 | 138 | cap_lens = torch.tensor([len(item['cap_tok']) for item in batch]) #Includes SOS and EOS as part of the length 139 | caption_list = [item['cap_tok'] for item in batch] 140 | # padded_captions = pad_sequence(caption_list, padding_value=9631) 141 | images = torch.stack([item['image'] for item in batch]) 142 | 143 | return images, caption_list, cap_lens 144 | 145 | 146 | def display_sample(sample): 147 | image = sample['image'] 148 | inv_normalize = transforms.Normalize( 149 | mean=[-0.485/0.229, -0.456/0.224, -0.406/0.255], 150 | std=[1/0.229, 1/0.224, 1/0.255] 151 | ) 152 | image = inv_normalize(image) 153 | caption = ' '.join(sample['caption']) 154 | cap_tok = sample['cap_tok'] 155 | plt.figure() 156 | plt.imshow(image.permute(1,2,0)) 157 | print("Caption: ", caption) 158 | print("Tokenized Caption: ", cap_tok) 159 | plt.show() 160 | 161 | def predict(model, device, image_name): 162 | vocab = [] 163 | with open(vocab_file, "r") as vocab_f: 164 | for line in vocab_f: 165 | vocab.append(line.strip()) 166 | image_path = os.path.join(img_dir, image_name) 167 | image = Image.open(image_path) 168 | transform = transforms.Compose([ 169 | transforms.Resize((224,224)), 170 | # transforms.CenterCrop(224), 171 | transforms.ToTensor(), 172 | transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) 173 | ]) 174 | image = transform(image) 175 | image = image.unsqueeze(0) 176 | hypotheses = eval.get_output_sentence(model, device, image, vocab) 177 | 178 | for i in range(len(hypotheses)): 179 | hypotheses[i] = [vocab[token - 1] for token in hypotheses[i]] 180 | hypotheses[i] = " ".join(hypotheses[i]) 181 | 182 | return hypotheses 183 | 184 | 185 | def clip_gradient(optimizer, grad_clip): 186 | """ 187 | Clips gradients computed during backpropagation to avoid explosion of gradients. 188 | :param optimizer: optimizer with the gradients to be clipped 189 | :param grad_clip: clip value 190 | """ 191 | for group in optimizer.param_groups: 192 | for param in group['params']: 193 | if param.grad is not None: 194 | param.grad.data.clamp_(-grad_clip, grad_clip) 195 | 196 | 197 | 198 | def adjust_optim(optimizer, n_iter, warmup_steps): 199 | optimizer.param_groups[0]['lr'] = (word_embedding_size**(-0.5)) * min(n_iter**(-0.5), n_iter*(warmup_steps**(-1.5))) 200 | 201 | 202 | 203 | 204 | def train_for_epoch(model, dataloader, optimizer, device, n_iter, args): 205 | '''Train an EncoderDecoder for an epoch 206 | 207 | Returns 208 | ------- 209 | avg_loss : float 210 | The total loss divided by the total numer of sequence 211 | ''' 212 | 213 | criterion1 = nn.CrossEntropyLoss(ignore_index=-1, reduction='sum') 214 | criterion2 = nn.MSELoss(reduction='sum') 215 | total_loss = 0 216 | total_num = 0 217 | for data in tqdm(dataloader): 218 | images, captions, cap_lens = data 219 | captions = pad_sequence(captions, padding_value=model.target_eos) #(seq_len, batch_size) 220 | images, captions, cap_lens = images.to(device), captions.to(device), cap_lens.to(device) 221 | optimizer.zero_grad() 222 | if model.decoder_type == 'rnn': 223 | logits, total_attention_weights = model(images, captions) #total_attention_weights -> (L, N, 1) 224 | total_attention_weights = total_attention_weights.sum(axis=0).squeeze(2).T 225 | else: 226 | logits = model(images, captions).permute(1, 0, 2) 227 | 228 | captions = captions[1:] 229 | mask = model.get_target_padding_mask(captions) 230 | captions = captions.masked_fill(mask,-1) 231 | loss1 = criterion1(torch.flatten(logits, 0, 1), torch.flatten(captions)) 232 | if model.decoder_type == 'rnn': 233 | loss2 = criterion2(total_attention_weights, torch.ones_like(total_attention_weights)) 234 | loss = loss1 + lamda * loss2 235 | else: 236 | if args.smoothing: 237 | eps = args.Lepsilon 238 | captions = captions.masked_fill(mask,0) #just to make the scatter work so no indexing issue occurs 239 | gold = captions.contiguous().view(-1) 240 | 241 | logits = torch.flatten(logits, 0, 1) 242 | n_class = logits.shape[-1] 243 | one_hot = torch.zeros_like(logits, device=logits.device).scatter(1, gold.view(-1, 1), 1) 244 | one_hot = one_hot * (1 - eps) + (1 - one_hot) * eps / (n_class - 1) 245 | log_prb = torch.log_softmax(logits, dim=1) 246 | 247 | captions = captions.masked_fill(mask,-1) #puttng the right mask back on 248 | gold = captions.contiguous().view(-1) 249 | non_pad_mask = gold.ne(-1) 250 | 251 | loss = -(one_hot * log_prb).sum(dim=1) 252 | loss = loss.masked_select(non_pad_mask).sum() # average later 253 | 254 | del gold, log_prb, non_pad_mask #hoping that this saves a bit of memory 255 | else: 256 | loss = loss1 257 | total_loss += loss.item() 258 | total_num += len(cap_lens) 259 | # print(total_loss/total_num) 260 | loss.backward() 261 | if grad_clip is not None: 262 | clip_gradient(optimizer, grad_clip) 263 | optimizer.step() 264 | # if model.decoder_type == 'transformer': 265 | # adjust_optim(optimizer, n_iter, warmup_steps) 266 | n_iter += 1 267 | torch.cuda.empty_cache() 268 | return total_loss/total_num, n_iter 269 | 270 | 271 | parser = argparse.ArgumentParser(description='Training Script for Encoder+LSTM decoder') 272 | parser.add_argument('--lr', type=float, help='learning rate', default=0.0001) 273 | parser.add_argument('--batch-size', type=int, help='batch size', default=64) 274 | parser.add_argument('--batch-size-val', type=int, help='batch size validation', default=64) 275 | parser.add_argument('--encoder-type', choices=['resnet18', 'resnet50', 'resnet101'], default='resnet18', 276 | help='Network to use in the encoder (default: resnet18)') 277 | parser.add_argument('--fine-tune', type=int, choices=[0,1], default=0) 278 | parser.add_argument('--decoder-type', choices=['rnn', 'transformer'], default='rnn') 279 | parser.add_argument('--beam-width', type=int, default=4) 280 | parser.add_argument('--num-epochs', type=int, default=100) 281 | parser.add_argument('--decoder-hidden-size', help="Hidden size for lstm", type=int, default=512) 282 | parser.add_argument('--experiment-name', type=str, default="autobestmodel") 283 | parser.add_argument('--num-tf-layers', help="Number of transformer layers", type=int, default=3) 284 | parser.add_argument('--num-heads', help="Number of heads", type=int, default=2) 285 | parser.add_argument('--beta1', help="Beta1 for Adam", type=float, default=0.9) 286 | parser.add_argument('--beta2', help="Beta2 for Adam", type=float, default=0.999) 287 | parser.add_argument('--dropout-lstm', help="Dropout_LSTM", type=float, default=0.5) 288 | parser.add_argument('--dropout-trans', help="Dropout_Trans", type=float, default=0.1) 289 | parser.add_argument('--smoothing', help="Label smoothing", type=int, default=1) 290 | parser.add_argument('--Lepsilon', help="Label smoothing epsilon", type=float, default=0.1) 291 | parser.add_argument('--use-checkpoint', help="Use checkpoint or start from beginning", type=int, default=0) 292 | parser.add_argument('--checkpoint-name', help="Checkpoint model file name", type=str, default=None) 293 | 294 | args = parser.parse_args() 295 | 296 | encoder_type = args.encoder_type 297 | decoder_type = args.decoder_type #transformer, rnn 298 | warmup_steps = 4000 299 | n_iter = 1 300 | 301 | if encoder_type == 'resnet18': 302 | CNN_channels = 512 #DO SOMETHING ABOUT THIS, 2048 for resnet101 303 | else: 304 | CNN_channels = 2048 305 | 306 | max_epochs = args.num_epochs 307 | beam_width = args.beam_width 308 | 309 | print("Epochs are read correctly: ", max_epochs) 310 | print("Encoder type is read correctly: ", encoder_type) 311 | print("Number of CNN channels being used: ", CNN_channels) 312 | print("Fine tune setting is set to: ", bool(args.fine_tune)) 313 | 314 | 315 | word_embedding_size = 512 316 | attention_dim = 512 317 | model_save_path = './model_saves/' 318 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 319 | lamda = 1. 320 | if decoder_type == 'rnn': 321 | learning_rate = args.lr 322 | decoder_hidden_size = args.decoder_hidden_size 323 | dropout = args.dropout_lstm 324 | else: 325 | print("Label smoothing set to: ", bool(args.smoothing)) 326 | learning_rate = 0.00004 327 | # learning_rate = (CNN_channels**(-0.5)) * min(n_iter**(-0.5), n_iter*(warmup_steps**(-1.5))) 328 | decoder_hidden_size = CNN_channels 329 | dropout = args.dropout_trans 330 | 331 | batch_size = args.batch_size 332 | batch_size_val = args.batch_size_val 333 | grad_clip = 5. 334 | transformer_layers = args.num_tf_layers 335 | heads = args.num_heads 336 | beta1 = args.beta1 337 | beta2 = args.beta2 338 | 339 | use_checkpoint = args.use_checkpoint 340 | checkpoint_path = args.checkpoint_name 341 | mode = 'train' 342 | 343 | if not os.path.isdir(model_save_path): 344 | os.mkdir(model_save_path) 345 | 346 | train_data = Flickr8kDataset(img_dir, train_dir, ann_dir, vocab_file) 347 | val_data = eval.TestDataset(img_dir, val_dir, ann_dir, vocab_file) 348 | test_data = eval.TestDataset(img_dir, test_dir, ann_dir, vocab_file) 349 | 350 | 351 | train_dataloader = DataLoader(train_data, batch_size=batch_size, shuffle=True, collate_fn=collater) 352 | val_dataloader = DataLoader(val_data, batch_size=batch_size_val, shuffle=False, collate_fn=eval.collater) 353 | test_dataloader = DataLoader(test_data, batch_size=batch_size_val, shuffle=False, collate_fn=eval.collater) 354 | 355 | encoder_class = Encoder 356 | if decoder_type == 'rnn': 357 | decoder_class = Decoder 358 | else: 359 | decoder_class = TransformerDecoder 360 | 361 | model = EncoderDecoder(encoder_class, decoder_class, train_data.vocab_size, target_sos=train_data.SOS, 362 | target_eos=train_data.EOS, fine_tune=bool(args.fine_tune), encoder_type=args.encoder_type, encoder_hidden_size=CNN_channels, 363 | decoder_hidden_size=decoder_hidden_size, 364 | word_embedding_size=word_embedding_size, attention_dim=attention_dim, decoder_type=decoder_type, cell_type='lstm', beam_width=beam_width, dropout=dropout, 365 | transformer_layers=transformer_layers, num_heads=heads) 366 | 367 | optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(beta1, beta2)) # used to experiment with (0.9, 0.98) for transformer 368 | # optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) 369 | 370 | fixed_image = "2090545563_a4e66ec76b.jpg" 371 | 372 | if mode == "train": 373 | 374 | best_bleu4 = 0. 375 | poor_iters = 0 376 | epoch = 1 377 | num_iters_change_lr = 4 378 | max_poor_iters = 10 379 | best_model = None 380 | best_optimizer = None 381 | best_loss = None 382 | best_epoch = None 383 | best_metrics = None 384 | 385 | if use_checkpoint: 386 | checkpoint = torch.load(model_save_path + checkpoint_path) 387 | model.load_state_dict(checkpoint['model_state_dict']) 388 | # optimizer.load_state_dict(checkpoint['optimizer_state_dict']) 389 | optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) 390 | epoch = checkpoint['epoch'] 391 | loss = checkpoint['loss'] 392 | print("Loss of checkpoint model: ", loss) 393 | model.to(device) 394 | print("Ground Truth captions: ", [" ".join(caption) for caption in val_data.all_captions[fixed_image]]) 395 | while epoch <= max_epochs: 396 | model.train() 397 | loss, n_iter = train_for_epoch(model, train_dataloader, optimizer, device, n_iter, args) 398 | 399 | 400 | # EVALUATE AND ADJUST LR ACCORDINGLY 401 | model.eval() 402 | print(f'Epoch {epoch}: loss={loss}') 403 | metrics = eval.get_pycoco_metrics(model, device, val_data, val_dataloader) 404 | print(metrics) 405 | is_epoch_better = metrics['Bleu_4'] > best_bleu4 406 | if is_epoch_better: 407 | poor_iters = 0 408 | best_bleu4 = metrics['Bleu_4'] 409 | best_model = copy.deepcopy(model) 410 | best_epoch = copy.deepcopy(epoch) 411 | # best_optimizer = copy.deepcopy(optimizer) 412 | best_loss = copy.deepcopy(loss) 413 | best_metrics = copy.deepcopy(metrics) 414 | else: 415 | poor_iters += 1 416 | # if poor_iters > 0 and poor_iters % num_iters_change_lr == 0: 417 | # print("Adjusting learning rate on epoch ", epoch) 418 | # utils.adjust_learning_rate(optimizer, 0.6) 419 | if poor_iters > max_poor_iters: 420 | print("Hasn't improved for ", max_poor_iters, " epochs...I give up :(") 421 | test_metrics = eval.get_pycoco_metrics(best_model, device, test_data, test_dataloader) 422 | utils.save_model_and_result(model_save_path, args.experiment_name, best_model, decoder_type, best_optimizer, best_epoch, best_bleu4, best_loss, best_metrics, test_metrics) 423 | break 424 | print("Predicted caption: ",predict(model, device, fixed_image)) 425 | 426 | # # SAVE MODEL EVERY 10 EPOCHS 427 | # if epoch % 10 == 0: 428 | # model.cpu() 429 | # utils.save_model_and_result(model_save_path, args.experiment_name, best_model, best_optimizer, best_epoch, best_bleu4, best_loss) 430 | 431 | epoch += 1 432 | if epoch > max_epochs: 433 | test_metrics = eval.get_pycoco_metrics(best_model, device, test_data, test_dataloader) 434 | utils.save_model_and_result(model_save_path, args.experiment_name, best_model, decoder_type, best_optimizer, best_epoch, best_bleu4, best_loss, best_metrics, test_metrics) 435 | print(f'Finished {max_epochs} epochs') 436 | torch.cuda.empty_cache() 437 | elif mode == "test": 438 | checkpoint = torch.load(model_save_path + checkpoint_path) 439 | # print("This model has bleu4 of: ", checkpoint['best_bleu4']) 440 | model.load_state_dict(checkpoint['model_state_dict']) 441 | model.to(device) 442 | model.eval() 443 | # predict(model, device, fixed_image) 444 | print(eval.get_pycoco_metrics(model, device, test_data, test_dataloader)) 445 | 446 | 447 | 448 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torchvision.models as models 4 | import warnings 5 | 6 | class Encoder(nn.Module): 7 | """ 8 | Encoder. 9 | """ 10 | def __init__(self, model_type, encoded_image_size=14, fine_tune=False): 11 | super(Encoder, self).__init__() 12 | self.enc_image_size = encoded_image_size 13 | model = getattr(models, model_type) 14 | resnet = model(pretrained=True) 15 | 16 | # Remove linear and pool layers (since we're not doing classification) 17 | modules = list(resnet.children())[:-2] 18 | self.resnet = nn.Sequential(*modules) 19 | 20 | # Resize image to fixed size to allow input images of variable size 21 | self.adaptive_pool = nn.AdaptiveAvgPool2d((encoded_image_size, encoded_image_size)) 22 | 23 | self.fine_tune(fine_tune) 24 | 25 | def forward(self, images): 26 | """ 27 | Forward propagation. 28 | :param images: images, a tensor of dimensions (batch_size, 3, image_size, image_size) 29 | :return: encoded images 30 | """ 31 | out = self.resnet(images) # (batch_size, 2048, image_size/32, image_size/32) 32 | out = self.adaptive_pool(out) # (batch_size, 2048, encoded_image_size, encoded_image_size) 33 | out = torch.flatten(out,2,3) #(batch_size, 2048, encoded_image_size * encoded_image_size) 34 | out = out.permute(2, 0, 1) # (encoded_image_size * encoded_image_size, batch_size, 2048) 35 | return out 36 | 37 | def fine_tune(self, fine_tune=False): 38 | """ 39 | Allow or prevent the computation of gradients for convolutional blocks 2 through 4 of the encoder. 40 | :param fine_tune: Allow? 41 | """ 42 | for p in self.resnet.parameters(): 43 | p.requires_grad = False 44 | # If fine-tuning, only fine-tune convolutional blocks 2 through 4 45 | for c in list(self.resnet.children())[-1]: 46 | for p in c.parameters(): 47 | p.requires_grad = fine_tune 48 | 49 | 50 | class AdditiveAttention(nn.Module): 51 | def __init__(self, encoder_hidden_size, decoder_hidden_size, attention_dim): 52 | super(AdditiveAttention, self).__init__() 53 | 54 | self.encoder_hidden_size = encoder_hidden_size 55 | self.decoder_hidden_size = decoder_hidden_size 56 | self.attention_dim = attention_dim 57 | 58 | self.beta_network = nn.Sequential(nn.Linear(decoder_hidden_size, 1), 59 | nn.Sigmoid()) 60 | 61 | self.softmax = nn.Softmax(dim=0) 62 | 63 | self.decoder_att_net = nn.Linear(decoder_hidden_size, attention_dim) 64 | self.encoder_att_net = nn.Linear(encoder_hidden_size, attention_dim) 65 | self.full_att_net = nn.Sequential(nn.Linear(attention_dim, 1), 66 | nn.ReLU()) 67 | 68 | 69 | def forward(self, queries, keys, values): 70 | """The forward pass of the additive attention mechanism. 71 | 72 | Arguments: 73 | queries: The current decoder hidden state. (batch_size x decoder_hidden_size) 74 | keys: The encoder hidden states for each step of the input sequence. (seq_len x batch_size x encoder_hidden_size) 75 | values: The encoder hidden states for each step of the input sequence. (seq_len x batch_size x encoder_hidden_size) 76 | 77 | Returns: 78 | context: weighted average of the values (batch_size x 1 x encoder_hidden_size) 79 | attention_weights: Normalized attention weights for each encoder hidden state. (seq_len x batch_size x 1) 80 | 81 | The attention_weights must be a softmax weighting over the seq_len annotations. 82 | 83 | Note: seq_len here refers to H*W (which by default is 14*14 = 196) 84 | """ 85 | batch_size = keys.shape[1] 86 | seq_len = keys.shape[0] 87 | expanded_queries = queries.unsqueeze(0).expand(seq_len, batch_size, self.decoder_hidden_size) 88 | att1 = self.decoder_att_net(expanded_queries) 89 | att2 = self.encoder_att_net(keys) 90 | unnormalized_attention = self.full_att_net(att1 + att2) 91 | attention_weights = self.softmax(unnormalized_attention) 92 | context = torch.bmm(attention_weights.permute(1,2,0), values.transpose(0,1)) 93 | beta = self.beta_network(queries) 94 | context = context * beta.unsqueeze(1) 95 | return context, attention_weights 96 | 97 | 98 | class MLP_init(nn.Module): 99 | def __init__(self, encoder_hidden_size, decoder_hidden_size): 100 | super(MLP_init, self).__init__() 101 | 102 | self.encoder_hidden_size = encoder_hidden_size 103 | self.decoder_hidden_size = decoder_hidden_size 104 | 105 | self.init_MLP = nn.Sequential( 106 | nn.Linear(encoder_hidden_size, decoder_hidden_size), 107 | nn.ReLU(), 108 | nn.Linear(decoder_hidden_size, decoder_hidden_size) 109 | ) 110 | 111 | def forward(self, h): 112 | return self.init_MLP(h) 113 | 114 | 115 | 116 | 117 | class Decoder(nn.Module): 118 | '''Decode source sequence embeddings into distributions over targets 119 | ''' 120 | 121 | def __init__( 122 | self, target_vocab_size, pad_id=-1, word_embedding_size=1024, 123 | encoder_hidden_state_size=1024, decoder_hidden_state_size=1024, attention_dim=512, cell_type='lstm', dropout=0.0): 124 | '''Initialize the decoder 125 | ''' 126 | super().__init__() 127 | self.target_vocab_size = target_vocab_size 128 | self.pad_id = pad_id 129 | self.word_embedding_size = word_embedding_size 130 | self.encoder_hidden_state_size = encoder_hidden_state_size 131 | self.decoder_hidden_state_size = decoder_hidden_state_size 132 | self.attention_dim = attention_dim 133 | self.cell_type = cell_type 134 | self.dropout = dropout 135 | self.embedding = self.cell = None 136 | self.ff_out = self.attention_net = self.ff_init_h = self.ff_init_c = None 137 | self.init_submodules() 138 | self.init_weights() 139 | 140 | def init_submodules(self): 141 | '''Initialize the parameterized submodules of this network 142 | ''' 143 | self.embedding = nn.Embedding(self.target_vocab_size, self.word_embedding_size, self.pad_id) 144 | if self.cell_type == 'rnn': 145 | self.cell = nn.RNNCell(input_size=self.word_embedding_size + self.encoder_hidden_state_size, hidden_size=self.decoder_hidden_state_size) 146 | elif self.cell_type == 'gru': 147 | self.cell = nn.GRUCell(input_size=self.word_embedding_size + self.encoder_hidden_state_size, hidden_size=self.decoder_hidden_state_size) 148 | else: 149 | self.cell = nn.LSTMCell(input_size=self.word_embedding_size + self.encoder_hidden_state_size, hidden_size=self.decoder_hidden_state_size) 150 | 151 | self.ff_out = nn.Linear(self.word_embedding_size , self.target_vocab_size) 152 | 153 | self.ff_init_h = MLP_init(encoder_hidden_size=self.encoder_hidden_state_size, decoder_hidden_size=self.decoder_hidden_state_size) 154 | self.ff_init_c = MLP_init(encoder_hidden_size=self.encoder_hidden_state_size, decoder_hidden_size=self.decoder_hidden_state_size) 155 | 156 | self.attention_net = AdditiveAttention(encoder_hidden_size=self.encoder_hidden_state_size, decoder_hidden_size=self.decoder_hidden_state_size, 157 | attention_dim=self.attention_dim) 158 | self.dropout = nn.Dropout(p=self.dropout) 159 | 160 | self.output_linear_1 = nn.Linear(self.decoder_hidden_state_size, self.word_embedding_size) 161 | self.output_linear_2 = nn.Linear(self.encoder_hidden_state_size, self.word_embedding_size) 162 | 163 | def init_weights(self): 164 | """ 165 | Initializes some parameters with values from the uniform distribution, for easier convergence. 166 | """ 167 | # self.embedding.weight.data.uniform_(-0.1, 0.1) 168 | # nn.init.xavier_uniform_(self.embedding.weight) 169 | # self.ff_out.bias.data.fill_(0) 170 | # self.ff_out.weight.data.uniform_(-0.1, 0.1) 171 | # nn.init.xavier_uniform_(self.ff_out.weight) 172 | # nn.init.xavier_uniform_(self.output_linear_1.weight) 173 | # nn.init.xavier_uniform_(self.output_linear_2.weight) 174 | # self.output_linear_1.weight.data.uniform_(-0.1, 0.1) 175 | # self.output_linear_2.weight.data.uniform_(-0.1, 0.1) 176 | 177 | def forward(self, E_tm1, y_tm1, htilde_tm1, h): 178 | ''' 179 | Inputs: 180 | h: Encoder hidden states. #(H*W, batch_size, num_channels) 181 | htilde_tm1: Previous decoder hidden state. #(batch_size, decoder_hidden_size) 182 | Tuple of two of these if cell type is LSTM 183 | E_tm1: Current input. #(batch_size, ) 184 | y_tm1: Previous output logits. #(batch_size, vocab_size) 185 | 186 | Returns: 187 | logits_t: Output logits #(batch_size, vocab_size) 188 | htilde_t: Current decoder hidden state. #(batch_size, decoder_hidden_size) 189 | Tuple of two of these if cell type is LSTM 190 | attention_weights: All the attention weights. #(num_encoder_hidden_states, batch_size, 1) 191 | ''' 192 | if htilde_tm1 is None: 193 | htilde_tm1 = self.get_first_hidden_state(h) 194 | if self.cell_type == 'lstm': #I don't like the way this part's been handled. Handle this later PLEASE! 195 | ctilde_tm1 = self.ff_init_c(h.mean(axis=0)) 196 | htilde_tm1 = (htilde_tm1, ctilde_tm1) 197 | 198 | if self.cell_type == 'lstm': 199 | xtilde_t, context, attention_weights = self.get_current_rnn_input(E_tm1, htilde_tm1[0], h) 200 | #context: (batch_size, 1, encoder_hidden_size) just a reminder, encoder_hidden_size is the num of channels 201 | #xtilde_t: (batch_size, embedding_size + encoder_hidden_size) 202 | #attention_weights: (num_encoder_hidden_states, batch_size, 1) just a reminder, num_encoder_hidden_states is H*W (default 14*14=196) 203 | else: 204 | xtilde_t, context, attention_weights = self.get_current_rnn_input(E_tm1, htilde_tm1, h) 205 | 206 | htilde_t = self.get_current_hidden_state(xtilde_t, htilde_tm1) #Same shape as htilde_tm1 207 | 208 | if y_tm1 is None: # Change this 209 | y_tm1 = self.embedding(torch.zeros(context.shape[0], device=h.device).long()) 210 | else: 211 | y_tm1 = self.embedding(torch.argmax(y_tm1, axis=1)) #(batch_size, embedding_size) 212 | 213 | if self.cell_type == 'lstm': 214 | logits_t = self.get_current_logits(htilde_t[0], y_tm1, context.squeeze(1)) 215 | else: 216 | logits_t = self.get_current_logits(htilde_t, y_tm1, context.squeeze(1)) #(batch_size, vocab_size) 217 | 218 | return logits_t, htilde_t, attention_weights 219 | 220 | def get_first_hidden_state(self, h): 221 | '''Get the initial decoder hidden state, prior to the first input 222 | ''' 223 | h_avg = h.mean(axis=0) 224 | htilde_tm1 = self.ff_init_h(h_avg) 225 | 226 | return htilde_tm1 227 | 228 | def get_current_rnn_input(self, E_tm1, htilde_tm1, h): 229 | '''Get the current input the decoder RNN 230 | ''' 231 | context, attention_weights = self.attention_net(htilde_tm1, h, h) 232 | xtilde_t = torch.cat((self.embedding(E_tm1),context.squeeze(1)),axis=1) 233 | return xtilde_t, context, attention_weights 234 | 235 | def get_current_hidden_state(self, xtilde_t, htilde_tm1): 236 | '''Calculate the decoder's current hidden state 237 | ''' 238 | htilde_t = self.cell(xtilde_t, htilde_tm1) 239 | return htilde_t 240 | 241 | def get_current_logits(self, htilde_t, y_tm1, ctx_t): 242 | '''Calculate an un-normalized log distribution over target words 243 | Uses the deep output layer as described in the paper 244 | ''' 245 | logits_t = self.ff_out(self.dropout(self.output_linear_1(htilde_t) + y_tm1 + self.output_linear_2(ctx_t))) 246 | return logits_t 247 | 248 | class ScaledDotAttention(nn.Module): 249 | def __init__(self, hidden_size): 250 | super(ScaledDotAttention, self).__init__() 251 | 252 | self.hidden_size = hidden_size 253 | 254 | self.Q = nn.Linear(hidden_size, hidden_size) 255 | self.K = nn.Linear(hidden_size, hidden_size) 256 | self.V = nn.Linear(hidden_size, hidden_size) 257 | self.softmax = nn.Softmax(dim=1) 258 | self.scaling_factor = torch.rsqrt(torch.tensor(self.hidden_size, dtype= torch.float)) 259 | 260 | def forward(self, queries, keys, values): 261 | """The forward pass of the scaled dot attention mechanism. 262 | 263 | Arguments: 264 | queries: The current decoder hidden state, 2D or 3D tensor. (batch_size x (k) x hidden_size) 265 | keys: The encoder hidden states for each step of the input sequence. (batch_size x seq_len x hidden_size) 266 | values: The encoder hidden states for each step of the input sequence. (batch_size x seq_len x hidden_size) 267 | 268 | Returns: 269 | context: weighted average of the values (batch_size x k x hidden_size) 270 | attention_weights: Normalized attention weights for each encoder hidden state. (batch_size x seq_len x 1) 271 | 272 | The output must be a softmax weighting over the seq_len annotations. 273 | """ 274 | 275 | # ------------ 276 | # FILL THIS IN 277 | # ------------ 278 | batch_size = queries.shape[0] 279 | q = self.Q(queries.view(batch_size, -1, queries.shape[-1])) 280 | k = self.K(keys) 281 | v = self.V(values) 282 | unnormalized_attention = k@q.transpose(2,1)*self.scaling_factor 283 | attention_weights = self.softmax(unnormalized_attention) 284 | context = attention_weights.transpose(2,1)@v 285 | return context, attention_weights 286 | 287 | 288 | class CausalScaledDotAttention(nn.Module): 289 | def __init__(self, hidden_size): 290 | super(CausalScaledDotAttention, self).__init__() 291 | 292 | self.hidden_size = hidden_size 293 | self.neg_inf = torch.tensor(-1e7) 294 | 295 | self.Q = nn.Linear(hidden_size, hidden_size) 296 | self.K = nn.Linear(hidden_size, hidden_size) 297 | self.V = nn.Linear(hidden_size, hidden_size) 298 | self.softmax = nn.Softmax(dim=1) 299 | self.scaling_factor = torch.rsqrt(torch.tensor(self.hidden_size, dtype= torch.float)) 300 | 301 | def forward(self, queries, keys, values): 302 | """The forward pass of the scaled dot attention mechanism. 303 | 304 | Arguments: 305 | queries: The current decoder hidden state, 2D or 3D tensor. (batch_size x (k) x hidden_size) 306 | keys: The encoder hidden states for each step of the input sequence. (batch_size x seq_len x hidden_size) 307 | values: The encoder hidden states for each step of the input sequence. (batch_size x seq_len x hidden_size) 308 | 309 | Returns: 310 | context: weighted average of the values (batch_size x k x hidden_size) 311 | attention_weights: Normalized attention weights for each encoder hidden state. (batch_size x seq_len x 1) 312 | 313 | The output must be a softmax weighting over the seq_len annotations. 314 | """ 315 | 316 | # ------------ 317 | # FILL THIS IN 318 | # ------------ 319 | batch_size = queries.shape[0] 320 | q = self.Q(queries.view(batch_size, -1, queries.shape[-1])) 321 | k = self.K(keys) 322 | v = self.V(values) 323 | unnormalized_attention = k@q.transpose(2,1)*self.scaling_factor 324 | mask = ~torch.triu(unnormalized_attention).bool() 325 | attention_weights = self.softmax(unnormalized_attention.masked_fill(mask, self.neg_inf)) 326 | context = attention_weights.transpose(2,1)@v 327 | return context, attention_weights 328 | 329 | 330 | 331 | class TransformerDecoder(nn.Module): 332 | def __init__(self, vocab_size, hidden_size, num_layers, num_heads, dropout): 333 | super(TransformerDecoder, self).__init__() 334 | self.vocab_size = vocab_size 335 | self.hidden_size = hidden_size 336 | 337 | self.embedding = nn.Embedding(vocab_size, hidden_size) 338 | self.num_layers = num_layers 339 | self.num_heads = num_heads 340 | 341 | self.self_attentions = nn.ModuleList([nn.ModuleList([CausalScaledDotAttention( 342 | hidden_size=hidden_size, 343 | ) for i in range(self.num_heads)]) for j in range(self.num_layers)]) 344 | self.encoder_attentions = nn.ModuleList([nn.ModuleList([ScaledDotAttention( 345 | hidden_size=hidden_size, 346 | ) for i in range(self.num_heads)]) for j in range(self.num_layers)]) 347 | self.attention_mlps = nn.ModuleList([nn.Sequential( 348 | nn.Linear(hidden_size, hidden_size), 349 | nn.ReLU(), 350 | ) for i in range(self.num_layers)]) 351 | 352 | self.linear_after_causal = nn.ModuleList([nn.Linear(self.num_heads*hidden_size, hidden_size) for j in range(self.num_layers)]) 353 | self.linear_after_scaled = nn.ModuleList([nn.Linear(self.num_heads*hidden_size, hidden_size) for j in range(self.num_layers)]) 354 | 355 | self.out = nn.Linear(hidden_size, vocab_size) 356 | 357 | self.positional_encodings = self.create_positional_encodings() 358 | 359 | self.dropout = nn.Dropout(p=dropout) 360 | 361 | self.layernorms1 = nn.ModuleList([nn.LayerNorm([self.hidden_size]) for i in range(self.num_layers)]) 362 | self.layernorms2 = nn.ModuleList([nn.LayerNorm([self.hidden_size]) for i in range(self.num_layers)]) 363 | self.layernorms3 = nn.ModuleList([nn.LayerNorm([self.hidden_size]) for i in range(self.num_layers)]) 364 | 365 | def forward(self, inputs, annotations): 366 | """Forward pass of the attention-based decoder RNN. 367 | 368 | Arguments: 369 | inputs: Input token indexes across a batch for all the time step. (batch_size x decoder_seq_len) 370 | annotations: The encoder hidden states for each step of the input. 371 | sequence. (batch_size x seq_len x hidden_size) 372 | hidden_init: Not used in the transformer decoder 373 | Returns: 374 | output: Un-normalized scores for each token in the vocabulary, across a batch for all the decoding time steps. (batch_size x decoder_seq_len x vocab_size) 375 | attentions: The stacked attention weights applied to the encoder annotations (batch_size x encoder_seq_len x decoder_seq_len) 376 | """ 377 | 378 | batch_size, seq_len = inputs.size() 379 | embed = self.embedding(inputs) # batch_size x seq_len x hidden_size 380 | 381 | # THIS LINE WAS ADDED AS A CORRECTION. 382 | embed = embed + self.positional_encodings[:seq_len] 383 | embed = self.dropout(embed) 384 | 385 | encoder_attention_weights_list = [] 386 | self_attention_weights_list = [] 387 | contexts = embed 388 | 389 | 390 | 391 | for i in range(self.num_layers): 392 | # ------------ 393 | # FILL THIS IN - START 394 | # ------------ 395 | concat_causal = torch.empty((batch_size, seq_len, 0), device='cuda') 396 | concat_scaled = torch.empty((batch_size, seq_len, 0), device='cuda') 397 | for j in range(self.num_heads): 398 | new_contexts, self_attention_weights = self.self_attentions[i][j](contexts, contexts, contexts) # batch_size x seq_len x hidden_size 399 | concat_causal = torch.cat((concat_causal, new_contexts), axis=2) 400 | 401 | new_contexts = self.linear_after_causal[i](concat_causal) #batch_size x seq_len x hidden_size*num_heads -----> batch_size x seq_len x hidden_size 402 | new_contexts = self.dropout(new_contexts) #dropout 403 | residual_contexts = self.layernorms1[i](contexts + new_contexts) #add and norm 404 | 405 | for j in range(self.num_heads): 406 | new_contexts, encoder_attention_weights = self.encoder_attentions[i][j](residual_contexts, annotations, annotations) # batch_size x seq_len x hidden_size 407 | concat_scaled = torch.cat((concat_scaled, new_contexts), axis=2) 408 | 409 | new_contexts = self.linear_after_scaled[i](concat_scaled) #batch_size x seq_len x hidden_size*num_heads -----> batch_size x seq_len x hidden_size 410 | new_contexts = self.dropout(new_contexts) #dropout 411 | residual_contexts = self.layernorms2[i](residual_contexts + new_contexts) #add and norm 412 | 413 | new_contexts = self.attention_mlps[i](residual_contexts) 414 | new_contexts = self.dropout(new_contexts) #dropout 415 | contexts = self.layernorms3[i](residual_contexts + new_contexts) #add and norm 416 | # ------------ 417 | # FILL THIS IN - END 418 | # ------------ 419 | 420 | encoder_attention_weights_list.append(encoder_attention_weights) 421 | self_attention_weights_list.append(self_attention_weights) 422 | 423 | output = self.out(contexts) 424 | encoder_attention_weights = torch.stack(encoder_attention_weights_list) 425 | self_attention_weights = torch.stack(self_attention_weights_list) 426 | 427 | return output, (encoder_attention_weights, self_attention_weights) 428 | 429 | def create_positional_encodings(self, max_seq_len=1000): 430 | """Creates positional encodings for the inputs. 431 | 432 | Arguments: 433 | max_seq_len: a number larger than the maximum string length we expect to encounter during training 434 | 435 | Returns: 436 | pos_encodings: (max_seq_len, hidden_dim) Positional encodings for a sequence with length max_seq_len. 437 | """ 438 | pos_indices = torch.arange(max_seq_len)[..., None] 439 | dim_indices = torch.arange(self.hidden_size//2)[None, ...] 440 | exponents = (2*dim_indices).float()/(self.hidden_size) 441 | trig_args = pos_indices / (10000**exponents) 442 | sin_terms = torch.sin(trig_args) 443 | cos_terms = torch.cos(trig_args) 444 | 445 | pos_encodings = torch.zeros((max_seq_len, self.hidden_size)) 446 | pos_encodings[:, 0::2] = sin_terms 447 | pos_encodings[:, 1::2] = cos_terms 448 | 449 | pos_encodings = pos_encodings.cuda() 450 | 451 | return pos_encodings 452 | 453 | class EncoderDecoder(nn.Module): 454 | '''Decode a source transcription into a target transcription 455 | ''' 456 | 457 | def __init__( 458 | self, encoder_class, decoder_class, 459 | target_vocab_size, target_sos=-2, target_eos=-1, encoder_type='resnet18', fine_tune=False, encoder_hidden_size=512, 460 | decoder_hidden_size=1024, word_embedding_size=1024, attention_dim=512, cell_type='lstm', decoder_type='rnn', beam_width=4, dropout=0.0, 461 | transformer_layers=3, num_heads=1): 462 | '''Initialize the encoder decoder combo 463 | ''' 464 | super().__init__() 465 | self.target_vocab_size = target_vocab_size 466 | self.target_sos = target_sos 467 | self.target_eos = target_eos 468 | self.encoder_type = encoder_type 469 | self.fine_tune = fine_tune 470 | self.encoder_hidden_size = encoder_hidden_size 471 | self.decoder_hidden_size = decoder_hidden_size 472 | self.word_embedding_size = word_embedding_size 473 | self.attention_dim = attention_dim 474 | self.cell_type = cell_type 475 | self.decoder_type = decoder_type 476 | self.beam_width = beam_width 477 | self.dropout = dropout 478 | self.transformer_layers = transformer_layers 479 | self.num_heads = num_heads 480 | self.encoder = self.decoder = None 481 | self.init_submodules(encoder_class, decoder_class) 482 | 483 | def init_submodules(self, encoder_class, decoder_class): 484 | '''Initialize encoder and decoder submodules 485 | ''' 486 | self.encoder = encoder_class(self.encoder_type, fine_tune=self.fine_tune) 487 | if self.decoder_type == 'rnn': 488 | self.decoder = decoder_class(self.target_vocab_size, 489 | self.target_eos, 490 | self.word_embedding_size, 491 | self.encoder_hidden_size, 492 | self.decoder_hidden_size, 493 | self.attention_dim, 494 | self.cell_type, 495 | self.dropout) 496 | else: 497 | self.decoder = decoder_class(self.target_vocab_size, 498 | self.encoder_hidden_size, 499 | self.transformer_layers, 500 | self.num_heads, 501 | self.dropout) 502 | 503 | def get_target_padding_mask(self, E): 504 | '''Determine what parts of a target sequence batch are padding 505 | 506 | `E` is right-padded with end-of-sequence symbols. This method 507 | creates a mask of those symbols, excluding the first in every sequence 508 | (the first eos symbol should not be excluded in the loss). 509 | 510 | Parameters 511 | ---------- 512 | E : torch.LongTensor 513 | A float tensor of shape ``(T - 1, N)``, where ``E[t', n]`` is 514 | the ``t'``-th token id of a gold-standard transcription for the 515 | ``n``-th source sequence. *Should* exclude the initial 516 | start-of-sequence token. 517 | 518 | Returns 519 | ------- 520 | pad_mask : torch.BoolTensor 521 | A boolean tensor of shape ``(T - 1, N)``, where ``pad_mask[t, n]`` 522 | is :obj:`True` when ``E[t, n]`` is considered padding. 523 | ''' 524 | pad_mask = E == self.target_eos # (T - 1, N) 525 | pad_mask = pad_mask & torch.cat([pad_mask[:1], pad_mask[:-1]], 0) 526 | return pad_mask 527 | 528 | def forward(self, images, captions=None, max_T=100, on_max='raise'): 529 | h = self.encoder(images) # (L, N, H) 530 | if self.training: 531 | return self.get_logits_for_teacher_forcing(h, captions) 532 | else: 533 | return self.beam_search(h, max_T, on_max) 534 | 535 | def get_logits_for_teacher_forcing(self, h, captions): 536 | '''Get un-normed distributions over next tokens via teacher forcing 537 | ''' 538 | op = [] 539 | h_cur = None 540 | cur_op = None 541 | total_attention_weights = [] 542 | if self.decoder_type == 'rnn': 543 | for i in range(len(captions)-1): 544 | cur_ip = captions[i] 545 | cur_op, h_cur, attention_weights = self.decoder(cur_ip, cur_op, h_cur, h) 546 | op.append(cur_op) 547 | total_attention_weights.append(attention_weights) 548 | return torch.stack(op), torch.stack(total_attention_weights) 549 | else: 550 | op, _ = self.decoder(captions[:-1,:].T, h.permute(1,0,2)) 551 | return op 552 | 553 | 554 | def beam_search(self, h, max_T, on_max): 555 | ''' 556 | Inputs: 557 | h: encoder hidden states. #(H*W, batch_size, L) default is (196, batch_size, 2048) 558 | ''' 559 | # beam search 560 | assert not self.training 561 | if self.decoder_type == 'rnn': 562 | htilde_tm1 = self.decoder.get_first_hidden_state(h) #(batch_size, decoder_hidden_size) 563 | logpb_tm1 = torch.where( 564 | torch.arange(self.beam_width, device=h.device) > 0, # K 565 | torch.full_like( 566 | htilde_tm1[..., 0].unsqueeze(1), -float('inf')), # k > 0 567 | torch.zeros_like( 568 | htilde_tm1[..., 0].unsqueeze(1)), # k == 0 569 | ) # (N, K) 570 | else: 571 | random_placeholder = torch.randn(h.shape[1], self.decoder_hidden_size, device=h.device) 572 | logpb_tm1 = torch.where( 573 | torch.arange(self.beam_width, device=h.device) > 0, # K 574 | torch.full_like( 575 | random_placeholder[..., 0].unsqueeze(1), -float('inf')), # k > 0 576 | torch.zeros_like( 577 | random_placeholder[..., 0].unsqueeze(1)), # k == 0 578 | ) # (N, K) 579 | 580 | assert torch.all(logpb_tm1[:, 0] == 0.) 581 | assert torch.all(logpb_tm1[:, 1:] == -float('inf')) 582 | b_tm1_1 = torch.full_like( # (t, N, K) 583 | logpb_tm1, self.target_sos, dtype=torch.long).unsqueeze(0) 584 | # We treat each beam within the batch as just another batch when 585 | # computing logits, then recover the original batch dimension by 586 | # reshaping 587 | if self.decoder_type == 'rnn': 588 | htilde_tm1 = htilde_tm1.unsqueeze(1).repeat(1, self.beam_width, 1) 589 | htilde_tm1 = htilde_tm1.flatten(end_dim=1) # (N * K, decoder_hidden_size) 590 | if self.cell_type == 'lstm': 591 | ctilde_tm1 = self.decoder.ff_init_c(h.mean(axis=0)) 592 | ctilde_tm1 = ctilde_tm1.unsqueeze(1).repeat(1, self.beam_width, 1) 593 | ctilde_tm1 = ctilde_tm1.flatten(end_dim=1) 594 | htilde_tm1 = (htilde_tm1, ctilde_tm1) 595 | h = h.unsqueeze(2).repeat(1, 1, self.beam_width, 1) 596 | h = h.flatten(1, 2) # (S, N * K, L) 597 | v_is_eos = torch.arange(self.target_vocab_size, device=h.device) 598 | v_is_eos = v_is_eos == self.target_eos # (V,) 599 | t = 0 600 | logits_tm1 = None 601 | cur_transformer_ip = None 602 | while torch.any(b_tm1_1[-1, :, 0] != self.target_eos): 603 | if t == max_T: 604 | if on_max == 'raise': 605 | raise RuntimeError( 606 | f'Beam search has not finished by t={t}. Increase the ' 607 | f'number of parameters and train longer') 608 | elif on_max == 'halt': 609 | print(f'Beam search not finished by t={t}. Halted') 610 | break 611 | finished = (b_tm1_1[-1] == self.target_eos) 612 | if self.decoder_type == 'rnn': 613 | E_tm1 = b_tm1_1[-1].flatten() # (N * K,) 614 | logits_t, htilde_t, _ = self.decoder(E_tm1, logits_tm1, htilde_tm1, h)#logits: (N * K, V), htilde_t:(N * K, decoder_hid_size) 615 | else: 616 | E_tm1 = b_tm1_1[-1].flatten().unsqueeze(1) # (N * K, 1) 617 | if cur_transformer_ip == None: 618 | cur_transformer_ip = E_tm1 619 | else: 620 | cur_transformer_ip = torch.cat([cur_transformer_ip, E_tm1], axis=1) 621 | op, _ = self.decoder(cur_transformer_ip, h.permute(1,0,2)) 622 | logits_t = op[:, -1, :] 623 | logits_tm1 = logits_t 624 | logits_t = logits_t.view( 625 | -1, self.beam_width, self.target_vocab_size) # (N, K, V) 626 | logpy_t = nn.functional.log_softmax(logits_t, -1) 627 | # We length-normalize the extensions of the unfinished paths 628 | if t: 629 | logpb_tm1 = torch.where( 630 | finished, logpb_tm1, logpb_tm1 * (t / (t + 1))) 631 | logpy_t = logpy_t / (t + 1) 632 | # For any path that's finished: 633 | # - v == gets log prob 0 634 | # - v != gets log prob -inf 635 | logpy_t = logpy_t.masked_fill( 636 | finished.unsqueeze(-1) & v_is_eos, 0.) 637 | logpy_t = logpy_t.masked_fill( 638 | finished.unsqueeze(-1) & (~v_is_eos), -float('inf')) 639 | if self.decoder_type == 'rnn': 640 | if self.cell_type == 'lstm': 641 | htilde_t = ( 642 | htilde_t[0].view( 643 | -1, self.beam_width, self.decoder_hidden_size), 644 | htilde_t[1].view( 645 | -1, self.beam_width, self.decoder_hidden_size), 646 | ) 647 | else: 648 | htilde_t = htilde_t.view( 649 | -1, self.beam_width, self.decoder_hidden_size) 650 | b_t_0, b_t_1, logpb_t = self.update_beam( 651 | htilde_t, b_tm1_1, logpb_tm1, logpy_t) 652 | del logits_t, logpy_t, finished, htilde_t 653 | if self.cell_type == 'lstm': 654 | htilde_tm1 = ( 655 | b_t_0[0].flatten(end_dim=1), 656 | b_t_0[1].flatten(end_dim=1) 657 | ) 658 | else: 659 | htilde_tm1 = b_t_0.flatten(end_dim=1) # (N * K, 2 * H) 660 | else: 661 | b_t_1, logpb_t = self.update_beam(None, b_tm1_1, logpb_tm1, logpy_t) 662 | del logits_t, logpy_t, finished 663 | logpb_tm1, b_tm1_1 = logpb_t, b_t_1 664 | t += 1 665 | return b_tm1_1 666 | 667 | def update_beam(self, htilde_t, b_tm1_1, logpb_tm1, logpy_t): 668 | '''Update the beam in a beam search for the current time step 669 | 670 | Parameters 671 | ---------- 672 | htilde_t : torch.FloatTensor 673 | A float tensor of shape 674 | ``(N, self.beam_with, self.decoder_hidden_size)`` where 675 | ``htilde_t[n, k, :]`` is the hidden state vector of the ``k``-th 676 | path in the beam search for batch element ``n`` for the current 677 | time step. ``htilde_t[n, k, :]`` was used to calculate 678 | ``logpy_t[n, k, :]``. 679 | b_tm1_1 : torch.LongTensor 680 | A long tensor of shape ``(t, N, self.beam_width)`` where 681 | ``b_tm1_1[t', n, k]`` is the ``t'``-th target token of the 682 | ``k``-th path of the search for the ``n``-th element in the batch 683 | up to the previous time step (including the start-of-sequence). 684 | logpb_tm1 : torch.FloatTensor 685 | A float tensor of shape ``(N, self.beam_width)`` where 686 | ``logpb_tm1[n, k]`` is the log-probability of the ``k``-th path 687 | of the search for the ``n``-th element in the batch up to the 688 | previous time step. Log-probabilities are sorted such that 689 | ``logpb_tm1[n, k] >= logpb_tm1[n, k']`` when ``k <= k'``. 690 | logpy_t : torch.FloatTensor 691 | A float tensor of shape 692 | ``(N, self.beam_width, self.target_vocab_size)`` where 693 | ``logpy_t[n, k, v]`` is the (normalized) conditional 694 | log-probability of the word ``v`` extending the ``k``-th path in 695 | the beam search for batch element ``n``. `logpy_t` has been 696 | modified to account for finished paths (i.e. if ``(n, k)`` 697 | indexes a finished path, 698 | ``logpy_t[n, k, v] = 0. if v == self.eos else -inf``) 699 | 700 | Returns 701 | ------- 702 | b_t_0, b_t_1, logpb_t : torch.FloatTensor, torch.LongTensor 703 | `b_t_0` is a float tensor of shape ``(N, self.beam_width, 704 | self.decoder_hidden_size)`` of the hidden states of the 705 | remaining paths after the update. `b_t_1` is a long tensor of shape 706 | ``(t + 1, N, self.beam_width)`` which provides the token sequences 707 | of the remaining paths after the update. `logpb_t` is a float 708 | tensor of the same shape as `logpb_tm1`, indicating the 709 | log-probabilities of the remaining paths in the beam after the 710 | update. Paths within a beam are ordered in decreasing log 711 | probability: 712 | ``logpb_t[n, k] >= logpb_t[n, k']`` implies ``k <= k'`` 713 | 714 | Notes 715 | ----- 716 | While ``logpb_tm1[n, k]``, ``htilde_t[n, k]``, and ``b_tm1_1[:, n, k]`` 717 | refer to the same path within a beam and so do ``logpb_t[n, k]``, 718 | ``b_t_0[n, k]``, and ``b_t_1[:, n, k]``, 719 | it is not necessarily the case that ``logpb_tm1[n, k]`` extends the 720 | path ``logpb_t[n, k]`` (nor ``b_t_1[:, n, k]`` the path 721 | ``b_tm1_1[:, n, k]``). This is because candidate paths are re-ranked in 722 | the update by log-probability. It may be the case that all extensions 723 | to ``logpb_tm1[n, k]`` are pruned in the update. 724 | 725 | ``b_t_0`` extracts the hidden states from ``htilde_t`` that remain 726 | after the update. 727 | ''' 728 | V = logpy_t.shape[2] #Vocab size 729 | K = logpy_t.shape[1] #Beam width 730 | 731 | s = logpb_tm1.unsqueeze(-1).expand_as(logpy_t) + logpy_t 732 | logy_flat = torch.flatten(s, 1, 2) 733 | top_k_val, top_k_ind = torch.topk(logy_flat, K, dim = 1) 734 | temp = top_k_ind // V #This tells us which beam that top value is from 735 | logpb_t = top_k_val 736 | 737 | temp_ = temp.expand_as(b_tm1_1) 738 | b_t_1 = torch.cat((torch.gather(b_tm1_1, 2, temp_), (top_k_ind % V).unsqueeze(0))) 739 | 740 | if htilde_t != None: 741 | if(self.cell_type == 'lstm'): 742 | temp_ = temp.unsqueeze(-1).expand_as(htilde_t[0]) 743 | b_t_0 = (torch.gather(htilde_t[0], 1, temp_), torch.gather(htilde_t[1], 1, temp_)) 744 | else: 745 | temp_ = temp.unsqueeze(-1).expand_as(htilde_t) 746 | b_t_0 = torch.gather(htilde_t, 1, temp_) 747 | 748 | return b_t_0, b_t_1, logpb_t 749 | else: 750 | return b_t_1, logpb_t --------------------------------------------------------------------------------