5 |
4 |
5 |
8 | 
\n",
259 | "\n",
272 | ""
321 | ],
322 | "text/plain": [
323 | " pos sentence_idx word tag\n",
324 | "0 NNS 1.0 Thousands O\n",
325 | "1 IN 1.0 of O\n",
326 | "2 NNS 1.0 demonstrators O\n",
327 | "3 VBP 1.0 have O\n",
328 | "4 VBN 1.0 marched O"
329 | ]
330 | },
331 | "execution_count": 5,
332 | "metadata": {
333 | "tags": []
334 | },
335 | "output_type": "execute_result"
336 | }
337 | ],
338 | "source": [
339 | "df = pd.read_csv(\"./data/ner.csv\", encoding = \"ISO-8859-1\", error_bad_lines=False)\n",
340 | "dataset=df.drop(['Unnamed: 0', 'lemma', 'next-lemma', 'next-next-lemma', 'next-next-pos',\n",
341 | " 'next-next-shape', 'next-next-word', 'next-pos', 'next-shape',\n",
342 | " 'next-word', 'prev-iob', 'prev-lemma', 'prev-pos',\n",
343 | " 'prev-prev-iob', 'prev-prev-lemma', 'prev-prev-pos', 'prev-prev-shape',\n",
344 | " 'prev-prev-word', 'prev-shape', 'prev-word','shape'],axis=1)\n",
345 | "dataset.head()"
346 | ]
347 | },
348 | {
349 | "cell_type": "code",
350 | "execution_count": 0,
351 | "metadata": {
352 | "colab": {},
353 | "colab_type": "code",
354 | "id": "sdqhHeAqcLnO"
355 | },
356 | "outputs": [],
357 | "source": [
358 | "# Creating a class to pull the words from the columns and create them into sentences\n",
359 | "\n",
360 | "class SentenceGetter(object):\n",
361 | " \n",
362 | " def __init__(self, dataset):\n",
363 | " self.n_sent = 1\n",
364 | " self.dataset = dataset\n",
365 | " self.empty = False\n",
366 | " agg_func = lambda s: [(w,p, t) for w,p, t in zip(s[\"word\"].values.tolist(),\n",
367 | " s['pos'].values.tolist(),\n",
368 | " s[\"tag\"].values.tolist())]\n",
369 | " self.grouped = self.dataset.groupby(\"sentence_idx\").apply(agg_func)\n",
370 | " self.sentences = [s for s in self.grouped]\n",
371 | " \n",
372 | " def get_next(self):\n",
373 | " try:\n",
374 | " s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n",
375 | " self.n_sent += 1\n",
376 | " return s\n",
377 | " except:\n",
378 | " return None\n",
379 | "\n",
380 | "getter = SentenceGetter(dataset)"
381 | ]
382 | },
383 | {
384 | "cell_type": "code",
385 | "execution_count": 0,
386 | "metadata": {
387 | "colab": {},
388 | "colab_type": "code",
389 | "id": "Cqln3QJecNsJ"
390 | },
391 | "outputs": [],
392 | "source": [
393 | "# Creating new lists and dicts that will be used at a later stage for reference and processing\n",
394 | "\n",
395 | "tags_vals = list(set(dataset[\"tag\"].values))\n",
396 | "tag2idx = {t: i for i, t in enumerate(tags_vals)}\n",
397 | "sentences = [' '.join([s[0] for s in sent]) for sent in getter.sentences]\n",
398 | "labels = [[s[2] for s in sent] for sent in getter.sentences]\n",
399 | "labels = [[tag2idx.get(l) for l in lab] for lab in labels]"
400 | ]
401 | },
402 | {
403 | "cell_type": "markdown",
404 | "metadata": {},
405 | "source": [
406 | "\n",
407 | "### Preparing the Dataset and Dataloader\n",
408 | "\n",
409 | "We will start with defining few key variables that will be used later during the training/fine tuning stage.\n",
410 | "Followed by creation of Dataset class - This defines how the text is pre-processed before sending it to the neural network. We will also define the Dataloader that will feed the data in batches to the neural network for suitable training and processing. \n",
411 | "Dataset and Dataloader are constructs of the PyTorch library for defining and controlling the data pre-processing and its passage to neural network. For further reading into Dataset and Dataloader read the [docs at PyTorch](https://pytorch.org/docs/stable/data.html)\n",
412 | "\n",
413 | "#### *CustomDataset* Dataset Class\n",
414 | "- This class is defined to accept the `tokenizer`, `sentences` and `labels` as input and generate tokenized output and tags that is used by the BERT model for training. \n",
415 | "- We are using the BERT tokenizer to tokenize the data in the `sentences` list for encoding. \n",
416 | "- The tokenizer uses the `encode_plus` method to perform tokenization and generate the necessary outputs, namely: `ids`, `attention_mask`\n",
417 | "- To read further into the tokenizer, [refer to this document](https://huggingface.co/transformers/model_doc/bert.html#berttokenizer)\n",
418 | "- `tags` is the encoded entity from the annonated dataset. \n",
419 | "- The *CustomDataset* class is used to create 2 datasets, for training and for validation.\n",
420 | "- *Training Dataset* is used to fine tune the model: **80% of the original data**\n",
421 | "- *Validation Dataset* is used to evaluate the performance of the model. The model has not seen this data during training. \n",
422 | "\n",
423 | "#### Dataloader\n",
424 | "- Dataloader is used to for creating training and validation dataloader that load data to the neural network in a defined manner. This is needed because all the data from the dataset cannot be loaded to the memory at once, hence the amount of dataloaded to the memory and then passed to the neural network needs to be controlled.\n",
425 | "- This control is achieved using the parameters such as `batch_size` and `max_len`.\n",
426 | "- Training and Validation dataloaders are used in the training and validation part of the flow respectively"
427 | ]
428 | },
429 | {
430 | "cell_type": "code",
431 | "execution_count": 0,
432 | "metadata": {
433 | "colab": {},
434 | "colab_type": "code",
435 | "id": "kL0b1VIQcTAC"
436 | },
437 | "outputs": [],
438 | "source": [
439 | "# Defining some key variables that will be used later on in the training\n",
440 | "\n",
441 | "MAX_LEN = 200\n",
442 | "TRAIN_BATCH_SIZE = 32\n",
443 | "VALID_BATCH_SIZE = 16\n",
444 | "EPOCHS = 5\n",
445 | "LEARNING_RATE = 2e-05\n",
446 | "tokenizer = BertTokenizer.from_pretrained('bert-base-cased')"
447 | ]
448 | },
449 | {
450 | "cell_type": "code",
451 | "execution_count": 0,
452 | "metadata": {
453 | "colab": {},
454 | "colab_type": "code",
455 | "id": "IV72GFgq_ZYb"
456 | },
457 | "outputs": [],
458 | "source": [
459 | "class CustomDataset(Dataset):\n",
460 | " def __init__(self, tokenizer, sentences, labels, max_len):\n",
461 | " self.len = len(sentences)\n",
462 | " self.sentences = sentences\n",
463 | " self.labels = labels\n",
464 | " self.tokenizer = tokenizer\n",
465 | " self.max_len = max_len\n",
466 | " \n",
467 | " def __getitem__(self, index):\n",
468 | " sentence = str(self.sentences[index])\n",
469 | " inputs = self.tokenizer.encode_plus(\n",
470 | " sentence,\n",
471 | " None,\n",
472 | " add_special_tokens=True,\n",
473 | " max_length=self.max_len,\n",
474 | " pad_to_max_length=True,\n",
475 | " return_token_type_ids=True\n",
476 | " )\n",
477 | " ids = inputs['input_ids']\n",
478 | " mask = inputs['attention_mask']\n",
479 | " label = self.labels[index]\n",
480 | " label.extend([4]*200)\n",
481 | " label=label[:200]\n",
482 | "\n",
483 | " return {\n",
484 | " 'ids': torch.tensor(ids, dtype=torch.long),\n",
485 | " 'mask': torch.tensor(mask, dtype=torch.long),\n",
486 | " 'tags': torch.tensor(label, dtype=torch.long)\n",
487 | " } \n",
488 | " \n",
489 | " def __len__(self):\n",
490 | " return self.len"
491 | ]
492 | },
493 | {
494 | "cell_type": "code",
495 | "execution_count": 12,
496 | "metadata": {
497 | "colab": {
498 | "base_uri": "https://localhost:8080/",
499 | "height": 67
500 | },
501 | "colab_type": "code",
502 | "id": "7BvniiyvX-rB",
503 | "outputId": "61fdc431-4544-4a19-a176-46f19ca58f0f"
504 | },
505 | "outputs": [
506 | {
507 | "name": "stdout",
508 | "output_type": "stream",
509 | "text": [
510 | "FULL Dataset: 35177\n",
511 | "TRAIN Dataset: 28141\n",
512 | "TEST Dataset: 7036\n"
513 | ]
514 | }
515 | ],
516 | "source": [
517 | "# Creating the dataset and dataloader for the neural network\n",
518 | "\n",
519 | "train_percent = 0.8\n",
520 | "train_size = int(train_percent*len(sentences))\n",
521 | "# train_dataset=df.sample(frac=train_size,random_state=200).reset_index(drop=True)\n",
522 | "# test_dataset=df.drop(train_dataset.index).reset_index(drop=True)\n",
523 | "train_sentences = sentences[0:train_size]\n",
524 | "train_labels = labels[0:train_size]\n",
525 | "\n",
526 | "test_sentences = sentences[train_size:]\n",
527 | "test_labels = labels[train_size:]\n",
528 | "\n",
529 | "print(\"FULL Dataset: {}\".format(len(sentences)))\n",
530 | "print(\"TRAIN Dataset: {}\".format(len(train_sentences)))\n",
531 | "print(\"TEST Dataset: {}\".format(len(test_sentences)))\n",
532 | "\n",
533 | "training_set = CustomDataset(tokenizer, train_sentences, train_labels, MAX_LEN)\n",
534 | "testing_set = CustomDataset(tokenizer, test_sentences, test_labels, MAX_LEN)"
535 | ]
536 | },
537 | {
538 | "cell_type": "code",
539 | "execution_count": 0,
540 | "metadata": {
541 | "colab": {},
542 | "colab_type": "code",
543 | "id": "F05bou5zZYtV"
544 | },
545 | "outputs": [],
546 | "source": [
547 | "train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
548 | " 'shuffle': True,\n",
549 | " 'num_workers': 0\n",
550 | " }\n",
551 | "\n",
552 | "test_params = {'batch_size': VALID_BATCH_SIZE,\n",
553 | " 'shuffle': True,\n",
554 | " 'num_workers': 0\n",
555 | " }\n",
556 | "\n",
557 | "training_loader = DataLoader(training_set, **train_params)\n",
558 | "testing_loader = DataLoader(testing_set, **test_params)"
559 | ]
560 | },
561 | {
562 | "cell_type": "markdown",
563 | "metadata": {},
564 | "source": [
565 | "\n",
566 | "### Creating the Neural Network for Fine Tuning\n",
567 | "\n",
568 | "#### Neural Network\n",
569 | " - We will be creating a neural network with the `BERTClass`. \n",
570 | " - This network will have the `BertForTokenClassification` model. \n",
571 | " - The data will be fed to the `BertForTokenClassification` as defined in the dataset. \n",
572 | " - Final layer outputs is what will be used to calcuate the loss and to determine the accuracy of models prediction. \n",
573 | " - We will initiate an instance of the network called `model`. This instance will be used for training and then to save the final trained model for future inference. \n",
574 | " \n",
575 | "#### Loss Function and Optimizer\n",
576 | " - `Optimizer` is defined in the next cell.\n",
577 | " - We do not define any `Loss function` since the specified model already outputs `Loss` for a given input. \n",
578 | " - `Optimizer` is used to update the weights of the neural network to improve its performance.\n",
579 | " \n",
580 | "#### Further Reading\n",
581 | "- You can refer to my [Pytorch Tutorials](https://github.com/abhimishra91/pytorch-tutorials) to get an intuition of Loss Function and Optimizer.\n",
582 | "- [Pytorch Documentation for Loss Function](https://pytorch.org/docs/stable/nn.html#loss-functions)\n",
583 | "- [Pytorch Documentation for Optimizer](https://pytorch.org/docs/stable/optim.html)\n",
584 | "- Refer to the links provided on the top of the notebook to read more about `BertForTokenClassification`. "
585 | ]
586 | },
587 | {
588 | "cell_type": "code",
589 | "execution_count": 0,
590 | "metadata": {
591 | "colab": {},
592 | "colab_type": "code",
593 | "id": "9vuIJrvSZble"
594 | },
595 | "outputs": [],
596 | "source": [
597 | "# Creating the customized model, by adding a drop out and a dense layer on top of distil bert to get the final output for the model. \n",
598 | "\n",
599 | "class BERTClass(torch.nn.Module):\n",
600 | " def __init__(self):\n",
601 | " super(BERTClass, self).__init__()\n",
602 | " self.l1 = transformers.BertForTokenClassification.from_pretrained('bert-base-cased', num_labels=18)\n",
603 | " # self.l2 = torch.nn.Dropout(0.3)\n",
604 | " # self.l3 = torch.nn.Linear(768, 200)\n",
605 | " \n",
606 | " def forward(self, ids, mask, labels):\n",
607 | " output_1= self.l1(ids, mask, labels = labels)\n",
608 | " # output_2 = self.l2(output_1[0])\n",
609 | " # output = self.l3(output_2)\n",
610 | " return output_1"
611 | ]
612 | },
613 | {
614 | "cell_type": "code",
615 | "execution_count": 16,
616 | "metadata": {
617 | "colab": {
618 | "base_uri": "https://localhost:8080/",
619 | "height": 1000
620 | },
621 | "colab_type": "code",
622 | "collapsed": true,
623 | "id": "CflOeT2-ZoV6",
624 | "outputId": "8045068c-9aa0-48b8-cad1-8ac7c3b1e807"
625 | },
626 | "outputs": [
627 | {
628 | "data": {
629 | "text/plain": [
630 | "BertForTokenClassification(\n",
631 | " (bert): BertModel(\n",
632 | " (embeddings): BertEmbeddings(\n",
633 | " (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
634 | " (position_embeddings): Embedding(512, 768)\n",
635 | " (token_type_embeddings): Embedding(2, 768)\n",
636 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
637 | " (dropout): Dropout(p=0.1, inplace=False)\n",
638 | " )\n",
639 | " (encoder): BertEncoder(\n",
640 | " (layer): ModuleList(\n",
641 | " (0): BertLayer(\n",
642 | " (attention): BertAttention(\n",
643 | " (self): BertSelfAttention(\n",
644 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
645 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
646 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
647 | " (dropout): Dropout(p=0.1, inplace=False)\n",
648 | " )\n",
649 | " (output): BertSelfOutput(\n",
650 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
651 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
652 | " (dropout): Dropout(p=0.1, inplace=False)\n",
653 | " )\n",
654 | " )\n",
655 | " (intermediate): BertIntermediate(\n",
656 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
657 | " )\n",
658 | " (output): BertOutput(\n",
659 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
660 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
661 | " (dropout): Dropout(p=0.1, inplace=False)\n",
662 | " )\n",
663 | " )\n",
664 | " (1): BertLayer(\n",
665 | " (attention): BertAttention(\n",
666 | " (self): BertSelfAttention(\n",
667 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
668 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
669 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
670 | " (dropout): Dropout(p=0.1, inplace=False)\n",
671 | " )\n",
672 | " (output): BertSelfOutput(\n",
673 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
674 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
675 | " (dropout): Dropout(p=0.1, inplace=False)\n",
676 | " )\n",
677 | " )\n",
678 | " (intermediate): BertIntermediate(\n",
679 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
680 | " )\n",
681 | " (output): BertOutput(\n",
682 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
683 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
684 | " (dropout): Dropout(p=0.1, inplace=False)\n",
685 | " )\n",
686 | " )\n",
687 | " (2): BertLayer(\n",
688 | " (attention): BertAttention(\n",
689 | " (self): BertSelfAttention(\n",
690 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
691 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
692 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
693 | " (dropout): Dropout(p=0.1, inplace=False)\n",
694 | " )\n",
695 | " (output): BertSelfOutput(\n",
696 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
697 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
698 | " (dropout): Dropout(p=0.1, inplace=False)\n",
699 | " )\n",
700 | " )\n",
701 | " (intermediate): BertIntermediate(\n",
702 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
703 | " )\n",
704 | " (output): BertOutput(\n",
705 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
706 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
707 | " (dropout): Dropout(p=0.1, inplace=False)\n",
708 | " )\n",
709 | " )\n",
710 | " (3): BertLayer(\n",
711 | " (attention): BertAttention(\n",
712 | " (self): BertSelfAttention(\n",
713 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
714 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
715 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
716 | " (dropout): Dropout(p=0.1, inplace=False)\n",
717 | " )\n",
718 | " (output): BertSelfOutput(\n",
719 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
720 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
721 | " (dropout): Dropout(p=0.1, inplace=False)\n",
722 | " )\n",
723 | " )\n",
724 | " (intermediate): BertIntermediate(\n",
725 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
726 | " )\n",
727 | " (output): BertOutput(\n",
728 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
729 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
730 | " (dropout): Dropout(p=0.1, inplace=False)\n",
731 | " )\n",
732 | " )\n",
733 | " (4): BertLayer(\n",
734 | " (attention): BertAttention(\n",
735 | " (self): BertSelfAttention(\n",
736 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
737 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
738 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
739 | " (dropout): Dropout(p=0.1, inplace=False)\n",
740 | " )\n",
741 | " (output): BertSelfOutput(\n",
742 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
743 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
744 | " (dropout): Dropout(p=0.1, inplace=False)\n",
745 | " )\n",
746 | " )\n",
747 | " (intermediate): BertIntermediate(\n",
748 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
749 | " )\n",
750 | " (output): BertOutput(\n",
751 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
752 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
753 | " (dropout): Dropout(p=0.1, inplace=False)\n",
754 | " )\n",
755 | " )\n",
756 | " (5): BertLayer(\n",
757 | " (attention): BertAttention(\n",
758 | " (self): BertSelfAttention(\n",
759 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
760 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
761 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
762 | " (dropout): Dropout(p=0.1, inplace=False)\n",
763 | " )\n",
764 | " (output): BertSelfOutput(\n",
765 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
766 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
767 | " (dropout): Dropout(p=0.1, inplace=False)\n",
768 | " )\n",
769 | " )\n",
770 | " (intermediate): BertIntermediate(\n",
771 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
772 | " )\n",
773 | " (output): BertOutput(\n",
774 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
775 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
776 | " (dropout): Dropout(p=0.1, inplace=False)\n",
777 | " )\n",
778 | " )\n",
779 | " (6): BertLayer(\n",
780 | " (attention): BertAttention(\n",
781 | " (self): BertSelfAttention(\n",
782 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
783 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
784 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
785 | " (dropout): Dropout(p=0.1, inplace=False)\n",
786 | " )\n",
787 | " (output): BertSelfOutput(\n",
788 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
789 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
790 | " (dropout): Dropout(p=0.1, inplace=False)\n",
791 | " )\n",
792 | " )\n",
793 | " (intermediate): BertIntermediate(\n",
794 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
795 | " )\n",
796 | " (output): BertOutput(\n",
797 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
798 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
799 | " (dropout): Dropout(p=0.1, inplace=False)\n",
800 | " )\n",
801 | " )\n",
802 | " (7): BertLayer(\n",
803 | " (attention): BertAttention(\n",
804 | " (self): BertSelfAttention(\n",
805 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
806 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
807 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
808 | " (dropout): Dropout(p=0.1, inplace=False)\n",
809 | " )\n",
810 | " (output): BertSelfOutput(\n",
811 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
812 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
813 | " (dropout): Dropout(p=0.1, inplace=False)\n",
814 | " )\n",
815 | " )\n",
816 | " (intermediate): BertIntermediate(\n",
817 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
818 | " )\n",
819 | " (output): BertOutput(\n",
820 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
821 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
822 | " (dropout): Dropout(p=0.1, inplace=False)\n",
823 | " )\n",
824 | " )\n",
825 | " (8): BertLayer(\n",
826 | " (attention): BertAttention(\n",
827 | " (self): BertSelfAttention(\n",
828 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
829 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
830 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
831 | " (dropout): Dropout(p=0.1, inplace=False)\n",
832 | " )\n",
833 | " (output): BertSelfOutput(\n",
834 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
835 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
836 | " (dropout): Dropout(p=0.1, inplace=False)\n",
837 | " )\n",
838 | " )\n",
839 | " (intermediate): BertIntermediate(\n",
840 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
841 | " )\n",
842 | " (output): BertOutput(\n",
843 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
844 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
845 | " (dropout): Dropout(p=0.1, inplace=False)\n",
846 | " )\n",
847 | " )\n",
848 | " (9): BertLayer(\n",
849 | " (attention): BertAttention(\n",
850 | " (self): BertSelfAttention(\n",
851 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
852 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
853 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
854 | " (dropout): Dropout(p=0.1, inplace=False)\n",
855 | " )\n",
856 | " (output): BertSelfOutput(\n",
857 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
858 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
859 | " (dropout): Dropout(p=0.1, inplace=False)\n",
860 | " )\n",
861 | " )\n",
862 | " (intermediate): BertIntermediate(\n",
863 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
864 | " )\n",
865 | " (output): BertOutput(\n",
866 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
867 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
868 | " (dropout): Dropout(p=0.1, inplace=False)\n",
869 | " )\n",
870 | " )\n",
871 | " (10): BertLayer(\n",
872 | " (attention): BertAttention(\n",
873 | " (self): BertSelfAttention(\n",
874 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
875 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
876 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
877 | " (dropout): Dropout(p=0.1, inplace=False)\n",
878 | " )\n",
879 | " (output): BertSelfOutput(\n",
880 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
881 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
882 | " (dropout): Dropout(p=0.1, inplace=False)\n",
883 | " )\n",
884 | " )\n",
885 | " (intermediate): BertIntermediate(\n",
886 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
887 | " )\n",
888 | " (output): BertOutput(\n",
889 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
890 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
891 | " (dropout): Dropout(p=0.1, inplace=False)\n",
892 | " )\n",
893 | " )\n",
894 | " (11): BertLayer(\n",
895 | " (attention): BertAttention(\n",
896 | " (self): BertSelfAttention(\n",
897 | " (query): Linear(in_features=768, out_features=768, bias=True)\n",
898 | " (key): Linear(in_features=768, out_features=768, bias=True)\n",
899 | " (value): Linear(in_features=768, out_features=768, bias=True)\n",
900 | " (dropout): Dropout(p=0.1, inplace=False)\n",
901 | " )\n",
902 | " (output): BertSelfOutput(\n",
903 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
904 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
905 | " (dropout): Dropout(p=0.1, inplace=False)\n",
906 | " )\n",
907 | " )\n",
908 | " (intermediate): BertIntermediate(\n",
909 | " (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
910 | " )\n",
911 | " (output): BertOutput(\n",
912 | " (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
913 | " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
914 | " (dropout): Dropout(p=0.1, inplace=False)\n",
915 | " )\n",
916 | " )\n",
917 | " )\n",
918 | " )\n",
919 | " (pooler): BertPooler(\n",
920 | " (dense): Linear(in_features=768, out_features=768, bias=True)\n",
921 | " (activation): Tanh()\n",
922 | " )\n",
923 | " )\n",
924 | " (dropout): Dropout(p=0.1, inplace=False)\n",
925 | " (classifier): Linear(in_features=768, out_features=18, bias=True)\n",
926 | ")"
927 | ]
928 | },
929 | "execution_count": 16,
930 | "metadata": {
931 | "tags": []
932 | },
933 | "output_type": "execute_result"
934 | }
935 | ],
936 | "source": [
937 | "model = BERTClass()\n",
938 | "model.to(dev)"
939 | ]
940 | },
941 | {
942 | "cell_type": "code",
943 | "execution_count": 0,
944 | "metadata": {
945 | "colab": {},
946 | "colab_type": "code",
947 | "id": "DN_u9NC5aaa_"
948 | },
949 | "outputs": [],
950 | "source": [
951 | "optimizer = torch.optim.Adam(params = model.parameters(), lr=LEARNING_RATE)"
952 | ]
953 | },
954 | {
955 | "cell_type": "markdown",
956 | "metadata": {},
957 | "source": [
958 | "\n",
959 | "### Fine Tuning the Model\n",
960 | "\n",
961 | "After all the effort of loading and preparing the data and datasets, creating the model and defining its loss and optimizer. This is probably the easier steps in the process. \n",
962 | "\n",
963 | "Here we define a training function that trains the model on the training dataset created above, specified number of times (EPOCH), An epoch defines how many times the complete data will be passed through the network. \n",
964 | "\n",
965 | "Following events happen in this function to fine tune the neural network:\n",
966 | "- The dataloader passes data to the model based on the batch size. \n",
967 | "- Subsequent output from the model and the actual category are compared to calculate the loss. \n",
968 | "- Loss value is used to optimize the weights of the neurons in the network.\n",
969 | "- After every 500 steps the loss value is printed in the console.\n",
970 | "\n",
971 | "As you can see just in 1 epoch by the final step the model was working with a miniscule loss of 0.08503091335296631 i.e. the output is extremely close to the actual output."
972 | ]
973 | },
974 | {
975 | "cell_type": "code",
976 | "execution_count": 0,
977 | "metadata": {
978 | "colab": {},
979 | "colab_type": "code",
980 | "id": "8aQ6WCk2a-Vd"
981 | },
982 | "outputs": [],
983 | "source": [
984 | "def train(epoch):\n",
985 | " model.train()\n",
986 | " for _,data in enumerate(training_loader, 0):\n",
987 | " ids = data['ids'].to(dev, dtype = torch.long)\n",
988 | " mask = data['mask'].to(dev, dtype = torch.long)\n",
989 | " targets = data['tags'].to(dev, dtype = torch.long)\n",
990 | "\n",
991 | " loss = model(ids, mask, labels = targets)[0]\n",
992 | "\n",
993 | " # optimizer.zero_grad()\n",
994 | " if _%500==0:\n",
995 | " print(f'Epoch: {epoch}, Loss: {loss.item()}')\n",
996 | " \n",
997 | " optimizer.zero_grad()\n",
998 | " loss.backward()\n",
999 | " xm.optimizer_step(optimizer)\n",
1000 | " xm.mark_step() "
1001 | ]
1002 | },
1003 | {
1004 | "cell_type": "code",
1005 | "execution_count": 31,
1006 | "metadata": {
1007 | "colab": {
1008 | "base_uri": "https://localhost:8080/",
1009 | "height": 689
1010 | },
1011 | "colab_type": "code",
1012 | "collapsed": true,
1013 | "id": "50oMGTe0bvl0",
1014 | "outputId": "292ede16-6da3-460f-d174-f7ee10c729e0"
1015 | },
1016 | "outputs": [
1017 | {
1018 | "name": "stdout",
1019 | "output_type": "stream",
1020 | "text": [
1021 | "Epoch: 0, Loss: 0.21416641771793365\n",
1022 | "Epoch: 0, Loss: 0.08791390806436539\n",
1023 | "Epoch: 0, Loss: 0.1277497559785843\n",
1024 | "Epoch: 0, Loss: 0.25511449575424194\n",
1025 | "Epoch: 0, Loss: 0.11072967946529388\n",
1026 | "Epoch: 0, Loss: 0.1202322468161583\n",
1027 | "Epoch: 0, Loss: 0.16198261082172394\n",
1028 | "Epoch: 0, Loss: 0.31682807207107544\n",
1029 | "Epoch: 1, Loss: 0.09211093187332153\n",
1030 | "Epoch: 1, Loss: 0.15079179406166077\n",
1031 | "Epoch: 1, Loss: 0.1959223747253418\n",
1032 | "Epoch: 1, Loss: 0.09143798053264618\n",
1033 | "Epoch: 1, Loss: 0.29411888122558594\n",
1034 | "Epoch: 1, Loss: 0.11708520352840424\n",
1035 | "Epoch: 1, Loss: 0.11245028674602509\n",
1036 | "Epoch: 1, Loss: 0.14728033542633057\n",
1037 | "Epoch: 2, Loss: 0.1607980579137802\n",
1038 | "Epoch: 2, Loss: 0.08060580492019653\n",
1039 | "Epoch: 2, Loss: 0.14363577961921692\n",
1040 | "Epoch: 2, Loss: 0.12225533276796341\n",
1041 | "Epoch: 2, Loss: 0.10335233807563782\n",
1042 | "Epoch: 2, Loss: 0.04923604056239128\n",
1043 | "Epoch: 2, Loss: 0.09237729012966156\n",
1044 | "Epoch: 2, Loss: 0.12473192811012268\n",
1045 | "Epoch: 3, Loss: 0.09085617959499359\n",
1046 | "Epoch: 3, Loss: 0.09351193159818649\n",
1047 | "Epoch: 3, Loss: 0.06728512048721313\n",
1048 | "Epoch: 3, Loss: 0.1666068434715271\n",
1049 | "Epoch: 3, Loss: 0.19255675375461578\n",
1050 | "Epoch: 3, Loss: 0.16131675243377686\n",
1051 | "Epoch: 3, Loss: 0.15462705492973328\n",
1052 | "Epoch: 3, Loss: 0.18679684400558472\n",
1053 | "Epoch: 4, Loss: 0.11378277838230133\n",
1054 | "Epoch: 4, Loss: 0.025372153148055077\n",
1055 | "Epoch: 4, Loss: 0.08231651782989502\n",
1056 | "Epoch: 4, Loss: 0.2682102620601654\n",
1057 | "Epoch: 4, Loss: 0.05264609679579735\n",
1058 | "Epoch: 4, Loss: 0.056522976607084274\n",
1059 | "Epoch: 4, Loss: 0.15710100531578064\n",
1060 | "Epoch: 4, Loss: 0.08503091335296631\n"
1061 | ]
1062 | }
1063 | ],
1064 | "source": [
1065 | "for epoch in range(5):\n",
1066 | " train(epoch)"
1067 | ]
1068 | },
1069 | {
1070 | "cell_type": "markdown",
1071 | "metadata": {},
1072 | "source": [
1073 | "\n",
1074 | "### Validating the Model\n",
1075 | "\n",
1076 | "During the validation stage we pass the unseen data(Testing Dataset) to the model. This step determines how good the model performs on the unseen data. \n",
1077 | "\n",
1078 | "This unseen data is the 30% of `ner.csv` which was seperated during the Dataset creation stage. \n",
1079 | "During the validation stage the weights of the model are not updated. Only the final output is compared to the actual value. This comparison is then used to calcuate the accuracy of the model. \n",
1080 | "\n",
1081 | "The metric used for measuring the performance of model for these problem statements is called F1 score. We will create a helper function for helping us with f1 score calcuation and also import a library for the same. `seqeval`"
1082 | ]
1083 | },
1084 | {
1085 | "cell_type": "code",
1086 | "execution_count": 27,
1087 | "metadata": {
1088 | "colab": {
1089 | "base_uri": "https://localhost:8080/",
1090 | "height": 34
1091 | },
1092 | "colab_type": "code",
1093 | "id": "6OckC0XNkWWm",
1094 | "outputId": "d27682ee-c34f-4811-d791-0c922afa8b05"
1095 | },
1096 | "outputs": [
1097 | {
1098 | "name": "stdout",
1099 | "output_type": "stream",
1100 | "text": [
1101 | " Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
1102 | ]
1103 | }
1104 | ],
1105 | "source": [
1106 | "from seqeval.metrics import f1_score\n",
1107 | "\n",
1108 | "def flat_accuracy(preds, labels):\n",
1109 | " flat_preds = np.argmax(preds, axis=2).flatten()\n",
1110 | " flat_labels = labels.flatten()\n",
1111 | " return np.sum(flat_preds == flat_labels)/len(flat_labels)"
1112 | ]
1113 | },
1114 | {
1115 | "cell_type": "code",
1116 | "execution_count": 0,
1117 | "metadata": {
1118 | "colab": {},
1119 | "colab_type": "code",
1120 | "id": "9zwFzzBriLMO"
1121 | },
1122 | "outputs": [],
1123 | "source": [
1124 | "def valid(model, testing_loader):\n",
1125 | " model.eval()\n",
1126 | " eval_loss = 0; eval_accuracy = 0\n",
1127 | " n_correct = 0; n_wrong = 0; total = 0\n",
1128 | " predictions , true_labels = [], []\n",
1129 | " nb_eval_steps, nb_eval_examples = 0, 0\n",
1130 | " with torch.no_grad():\n",
1131 | " for _, data in enumerate(testing_loader, 0):\n",
1132 | " ids = data['ids'].to(dev, dtype = torch.long)\n",
1133 | " mask = data['mask'].to(dev, dtype = torch.long)\n",
1134 | " targets = data['tags'].to(dev, dtype = torch.long)\n",
1135 | "\n",
1136 | " output = model(ids, mask, labels=targets)\n",
1137 | " loss, logits = output[:2]\n",
1138 | " logits = logits.detach().cpu().numpy()\n",
1139 | " label_ids = targets.to('cpu').numpy()\n",
1140 | " predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n",
1141 | " true_labels.append(label_ids)\n",
1142 | " accuracy = flat_accuracy(logits, label_ids)\n",
1143 | " eval_loss += loss.mean().item()\n",
1144 | " eval_accuracy += accuracy\n",
1145 | " nb_eval_examples += ids.size(0)\n",
1146 | " nb_eval_steps += 1\n",
1147 | " eval_loss = eval_loss/nb_eval_steps\n",
1148 | " print(\"Validation loss: {}\".format(eval_loss))\n",
1149 | " print(\"Validation Accuracy: {}\".format(eval_accuracy/nb_eval_steps))\n",
1150 | " pred_tags = [tags_vals[p_i] for p in predictions for p_i in p]\n",
1151 | " valid_tags = [tags_vals[l_ii] for l in true_labels for l_i in l for l_ii in l_i]\n",
1152 | " print(\"F1-Score: {}\".format(f1_score(pred_tags, valid_tags)))"
1153 | ]
1154 | },
1155 | {
1156 | "cell_type": "code",
1157 | "execution_count": null,
1158 | "metadata": {},
1159 | "outputs": [],
1160 | "source": [
1161 | "# To get the results on the validation set. This data is not seen by the model\n",
1162 | "\n",
1163 | "valid(model, testing_loader)"
1164 | ]
1165 | }
1166 | ],
1167 | "metadata": {
1168 | "accelerator": "TPU",
1169 | "colab": {
1170 | "collapsed_sections": [],
1171 | "name": "transformers_ner.ipynb",
1172 | "provenance": []
1173 | },
1174 | "kernelspec": {
1175 | "display_name": "Python 3",
1176 | "language": "python",
1177 | "name": "python3"
1178 | },
1179 | "language_info": {
1180 | "codemirror_mode": {
1181 | "name": "ipython",
1182 | "version": 3
1183 | },
1184 | "file_extension": ".py",
1185 | "mimetype": "text/x-python",
1186 | "name": "python",
1187 | "nbconvert_exporter": "python",
1188 | "pygments_lexer": "ipython3",
1189 | "version": "3.7.6"
1190 | }
1191 | },
1192 | "nbformat": 4,
1193 | "nbformat_minor": 1
1194 | }
--------------------------------------------------------------------------------
/transformers_sentiment_wandb.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {
6 | "colab": {},
7 | "colab_type": "code",
8 | "id": "UNiYBVL25rQK"
9 | },
10 | "source": [
11 | "# Fine Tuning Transformer for Sentiment Classification"
12 | ]
13 | },
14 | {
15 | "cell_type": "markdown",
16 | "metadata": {},
17 | "source": [
18 | "### Introduction\n",
19 | "\n",
20 | "In this tutorial we will be fine tuning a transformer model for the **Sentiment classification** problem. **Sentiment classification** is a special case of **Multiclass Classification**. In this case the classes represent the sentiment represented by the text.\n",
21 | "The number of classes are generally lesser than a standard multiclass classification proboem where the classes represent the polarity, in form of `postive`, `negative` and in some cases and additional `neutral` polarity.\n",
22 | "\n",
23 | "This is one of the most common business problems when trying to ascertain the sentiment of a statement made by your customer in a business setup.\n",
24 | "\n",
25 | "#### Flow of the notebook\n",
26 | "\n",
27 | "* As with all the tutorials previously, this notebook also follows a easy to follow steps. Making the process of fine tuning and training a Transformers model a straight forward task.\n",
28 | "* However, unlike the other notebooks, in the tutorial, most of the sections have been created into functions, and they are called from the `main()` in the end of the notebook. \n",
29 | "* This is done to leverage the [Weights and Biases Service](https://www.wandb.com/) WandB in short.\n",
30 | "* It is a experiment tracking, parameter optimization and artifact management service. That can be very easily integrated to any of the Deep learning or Machine learning frameworks. \n",
31 | "\n",
32 | "The notebook will be divided into separate sections to provide a organized walk through for the process used. This process can be modified for individual use cases. The sections are:\n",
33 | "\n",
34 | "1. [Preparing Environment and Importing Libraries](#section01)\n",
35 | "2. [Pre-Processing and Preparing the Dataset for data processing: Class](#section02)\n",
36 | "3. [Defining a Model/Network](#section07)\n",
37 | "4. [Fine Tuning the Model: Function](#section03)\n",
38 | "5. [Validating the Model Performance: Function](#section04)\n",
39 | "6. [Main Function](#section05)\n",
40 | " * [Initializing WandB](#section501)\n",
41 | " * [Importing and Pre-Processing the domain data](#section502)\n",
42 | " * [Creation of Dataset and Dataloader](#section503)\n",
43 | " * [Neural Network and Optimizer](#section504)\n",
44 | " * [Training Model and Logging to WandB](#section505)\n",
45 | " * [Validation and generation of Summary](#section506)\n",
46 | "\n",
47 | "\n",
48 | "#### Technical Details\n",
49 | "\n",
50 | "This script leverages on multiple tools designed by other teams. Details of the tools used below. Please ensure that these elements are present in your setup to successfully implement this script.\n",
51 | "\n",
52 | "- **Data**:\n",
53 | "\t- We are using the **IMDB Dataset** available at [Kaggle](https://www.kaggle.com/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews)\n",
54 | "\t- This dataset is a collection of moview reviews obtained from IMDB website, the reviews are labled with a positive or negative sentiment. \n",
55 | "\t- There are approx. `50000` rows of data. Where each row has the following data-point:\n",
56 | "\t\t- **review** : Review of a movie\n",
57 | "\t\t- **sentiment** : positive or negative\n",
58 | "\n",
59 | "\n",
60 | "- **Language Model Used**: \n",
61 | " - This notebook uses ***RoBERTa*** as its base transformer model. [Research Paper](https://arxiv.org/abs/1907.11692) \n",
62 | " - ***RoBERTa*** was an incremental improvement in the ***BERT*** architecture with multiple tweaks in different domains.\n",
63 | " - Some of the changes in RoBERTa were: Bigger training data, Dymanic Masking, Different Self Supervised training objective.\n",
64 | " - You can have a detailed read of these changes at the following [link](https://medium.com/towards-artificial-intelligence/a-robustly-optimized-bert-pretraining-approach-f6b6e537e6a6). \n",
65 | " - We will be leveraging on the ***RoBERTa*** implementation from the HuggingFace team. \n",
66 | " - [Documentation for python](https://huggingface.co/transformers/model_doc/t5.html)\n",
67 | "\n",
68 | "\n",
69 | "- **Hardware Requirements**: \n",
70 | "\t- Python 3.6 and above\n",
71 | "\t- Pytorch, Transformers and\n",
72 | "\t- All the stock Python ML Library\n",
73 | "\t- GPU/TPU enabled setup \n",
74 | " \n",
75 | "\n",
76 | "- **Script Objective**:\n",
77 | "\t- The objective of this script is to fine tune ***RoBERTa*** to be able to classify wether the sentiment of a given text is positive or negative.\n",
78 | "\n",
79 | "---\n",
80 | "NOTE: \n",
81 | "We are using the Weights and Biases Tool-set in this tutorial. The different components will be explained as we go through the article. This is an incremental work done in the summarization notebook.\n",
82 | "\n",
83 | "[Link](https://app.wandb.ai/abhimishra-91/transformers_tutorials_sentiment?workspace=user-abhimishra-91) to the Project on WandB"
84 | ]
85 | },
86 | {
87 | "cell_type": "markdown",
88 | "metadata": {},
89 | "source": [
90 | "\n",
91 | "### Preparing Environment and Importing Libraries\n",
92 | "\n",
93 | "At this step we will be installing the necessary libraries followed by importing the libraries and modules needed to run our script. \n",
94 | "We will be installing:\n",
95 | "* transformers\n",
96 | "* wandb\n",
97 | "* packages to support tpu for pytorch\n",
98 | "\n",
99 | "Libraries imported are:\n",
100 | "* Pandas\n",
101 | "* Pytorch\n",
102 | "* Pytorch Utils for Dataset and Dataloader\n",
103 | "* Transformers\n",
104 | "* Roberta Model and Tokenizer\n",
105 | "* wandb\n",
106 | "\n",
107 | "Followed by that we will preapre the device to support TPU execution for training.\n",
108 | "\n",
109 | "Finally, we will be logging into the [wandb](https://www.wandb.com/) serice using the login command"
110 | ]
111 | },
112 | {
113 | "cell_type": "code",
114 | "execution_count": null,
115 | "metadata": {
116 | "colab": {
117 | "base_uri": "https://localhost:8080/",
118 | "height": 844
119 | },
120 | "colab_type": "code",
121 | "id": "4k86Li6zL9Jz",
122 | "outputId": "6fb4449b-12cc-4917-84f0-6e17ff56b9e7"
123 | },
124 | "outputs": [],
125 | "source": [
126 | "# Installing NLP-Transformers library\n",
127 | "!pip install -q transformers\n",
128 | "\n",
129 | "# Installing wandb library for experiment tracking and hyper parameter optimization\n",
130 | "!pip install -q wandb\n",
131 | "\n",
132 | "# Code for TPU packages install\n",
133 | "!curl -q https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n",
134 | "!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev"
135 | ]
136 | },
137 | {
138 | "cell_type": "code",
139 | "execution_count": 3,
140 | "metadata": {
141 | "colab": {},
142 | "colab_type": "code",
143 | "id": "hCOWCYgQvgkP"
144 | },
145 | "outputs": [],
146 | "source": [
147 | "# Importing stock libraries\n",
148 | "import numpy as np\n",
149 | "import pandas as pd\n",
150 | "import torch\n",
151 | "import torch.nn.functional as F\n",
152 | "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\n",
153 | "\n",
154 | "# Importing lackages from our NLP-Hugging Package\n",
155 | "from transformers import RobertaConfig, RobertaModel, RobertaTokenizerFast, RobertaForSequenceClassification\n",
156 | "\n",
157 | "# Importing wand for logging and hyper-parameter tuning\n",
158 | "import wandb"
159 | ]
160 | },
161 | {
162 | "cell_type": "code",
163 | "execution_count": 4,
164 | "metadata": {
165 | "colab": {},
166 | "colab_type": "code",
167 | "id": "i7NMSGVKDwue"
168 | },
169 | "outputs": [],
170 | "source": [
171 | "# Setting up the accelerators\n",
172 | "\n",
173 | "# # GPU\n",
174 | "# from torch import cuda\n",
175 | "# device = 'cuda' if cuda.is_available() else 'cpu'\n",
176 | "\n",
177 | "# TPU\n",
178 | "import torch_xla\n",
179 | "import torch_xla.core.xla_model as xm\n",
180 | "device = xm.xla_device()"
181 | ]
182 | },
183 | {
184 | "cell_type": "code",
185 | "execution_count": 5,
186 | "metadata": {
187 | "colab": {
188 | "base_uri": "https://localhost:8080/",
189 | "height": 87
190 | },
191 | "colab_type": "code",
192 | "id": "X9L1AxSQDugC",
193 | "outputId": "76ceca60-14c8-4baf-b410-ec2c5c179bc9"
194 | },
195 | "outputs": [
196 | {
197 | "name": "stdout",
198 | "output_type": "stream",
199 | "text": [
200 | "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://app.wandb.ai/authorize\n",
201 | "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter: \n",
202 | "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n",
203 | "\u001b[32mSuccessfully logged in to Weights & Biases!\u001b[0m\n"
204 | ]
205 | }
206 | ],
207 | "source": [
208 | "# login to wandb\n",
209 | "!wandb login"
210 | ]
211 | },
212 | {
213 | "cell_type": "markdown",
214 | "metadata": {},
215 | "source": [
216 | "\n",
217 | "### Pre-Processing and Preparing the Dataset for data processing: Class\n",
218 | "\n",
219 | "* We will start with creation of Preprocess class - This defines how the text is pre-processed before working on the tokenization, dataset and dataloader aspects of the workflow. In this class the dataframe is loaded and then the `sentiment` column is used to create a new column in the dataframe called `encoded_polarity` such that if:\n",
220 | " * `sentiment = positive` then `encoded_polarity = 0`\n",
221 | " * `sentiment = negative` then `encoded_polarity = 1`\n",
222 | "\n",
223 | "* Followed by this, the `sentiment` column is removed from the dataframe.\n",
224 | "* The `dataframe` and `encoded_polarity` dictionary are returned. \n",
225 | "* This method is called in the `run()` function.\n",
226 | "\n",
227 | "* After this we will work on the Dataset class - This defines how the text is pre-processed before sending it to the neural network. This dataset will be used the the Dataloader method that will feed the data in batches to the neural network for suitable training and processing. \n",
228 | "* The Dataloader and Dataset will be used inside the `run()`.\n",
229 | "* Dataset and Dataloader are constructs of the PyTorch library for defining and controlling the data pre-processing and its passage to neural network. For further reading into Dataset and Dataloader read the [docs at PyTorch](https://pytorch.org/docs/stable/data.html)\n",
230 | "\n",
231 | "#### *CustomDataset* Dataset Class\n",
232 | "- This class is defined to accept the Dataframe as input and generate tokenized output that is used by the Roberta model for training. \n",
233 | "- We are using the Roberta tokenizer to tokenize the data in the `review` column of the dataframe. \n",
234 | "- The tokenizer uses the `encode_plus` method to perform tokenization and generate the necessary outputs, namely: `ids`, `attention_mask`\n",
235 | "- To read further into the tokenizer, [refer to this document](https://huggingface.co/transformers/model_doc/roberta.html#robertatokenizer)\n",
236 | "- `encoded_polarity` transformed into the `targets` tensor. \n",
237 | "- The *CustomDataset* class is used to create 2 datasets, for training and for validation.\n",
238 | "- *Training Dataset* is used to fine tune the model: **70% of the original data**\n",
239 | "- *Validation Dataset* is used to evaluate the performance of the model. The model has not seen this data during training. \n",
240 | "\n",
241 | "#### return_dataloader: Called inside the `run()`\n",
242 | "- `return_dataloader` function is used to for creating training and validation dataloader that load data to the neural network in a defined manner. This is needed because all the data from the dataset cannot be loaded to the memory at once, hence the amount of data loaded to the memory and then passed to the neural network needs to be controlled.\n",
243 | "- Internally the `return_dataloader` function calls the pytorch Dataloader class and the CustomDataset class to create the dataloaders for training and validation. \n",
244 | "- This control is achieved using the parameters such as `batch_size` and `max_len`.\n",
245 | "- Training and Validation dataloaders are used in the training and validation part of the flow respectively"
246 | ]
247 | },
248 | {
249 | "cell_type": "code",
250 | "execution_count": 6,
251 | "metadata": {
252 | "colab": {},
253 | "colab_type": "code",
254 | "id": "yDe5ntrM9Kud"
255 | },
256 | "outputs": [],
257 | "source": [
258 | "# Preprocess class defines how the dataframe will be processed to generate and removal of features based on thier usage.\n",
259 | "# A new encoded_polarity column is added that adds 0 and 1 to the column based on the positive and negative\n",
260 | "# The processing method will return both the dictionary, and the updated dataframe for further usage.\n",
261 | "\n",
262 | "class Preprocess:\n",
263 | " def __init__(self, df):\n",
264 | " \"\"\"\n",
265 | " Constructor for the class\n",
266 | " :param df: Input Dataframe to be pre-processed\n",
267 | " \"\"\"\n",
268 | " self.df = df\n",
269 | " self.encoded_dict = dict()\n",
270 | "\n",
271 | " def encoding(self, x):\n",
272 | " if x not in self.encoded_dict.keys():\n",
273 | " self.encoded_dict[x] = len(self.encoded_dict)\n",
274 | " return self.encoded_dict[x]\n",
275 | "\n",
276 | " def processing(self):\n",
277 | " self.df['encoded_polarity'] = self.df['sentiment'].apply(lambda x: self.encoding(x))\n",
278 | " self.df.drop(['sentiment'], axis=1, inplace=True)\n",
279 | " return self.encoded_dict, self.df"
280 | ]
281 | },
282 | {
283 | "cell_type": "code",
284 | "execution_count": 7,
285 | "metadata": {
286 | "colab": {},
287 | "colab_type": "code",
288 | "id": "l00E-IDDdg0s"
289 | },
290 | "outputs": [],
291 | "source": [
292 | "# Creating a CustomDataset class that is used to read the updated dataframe and tokenize the text. \n",
293 | "# The class is used in the return_dataloader function\n",
294 | "\n",
295 | "class CustomDataset(Dataset):\n",
296 | " def __init__(self, dataframe, tokenizer, max_len):\n",
297 | " self.len = len(dataframe)\n",
298 | " self.data = dataframe\n",
299 | " self.tokenizer = tokenizer\n",
300 | " self.max_len = max_len\n",
301 | " \n",
302 | " def __getitem__(self, index):\n",
303 | " text = str(self.data.review[index])\n",
304 | " text = \" \".join(text.split())\n",
305 | " inputs = self.tokenizer.encode_plus(\n",
306 | " text,\n",
307 | " None,\n",
308 | " add_special_tokens=True,\n",
309 | " max_length=self.max_len,\n",
310 | " pad_to_max_length=True,\n",
311 | " return_token_type_ids=True\n",
312 | " )\n",
313 | " ids = inputs['input_ids']\n",
314 | " mask = inputs['attention_mask']\n",
315 | "\n",
316 | " return {\n",
317 | " 'ids': torch.tensor(ids, dtype=torch.long),\n",
318 | " 'mask': torch.tensor(mask, dtype=torch.long),\n",
319 | " 'targets': torch.tensor(self.data.encoded_polarity[index], dtype=torch.float)\n",
320 | " } \n",
321 | " \n",
322 | " def __len__(self):\n",
323 | " return self.len"
324 | ]
325 | },
326 | {
327 | "cell_type": "code",
328 | "execution_count": 8,
329 | "metadata": {
330 | "colab": {},
331 | "colab_type": "code",
332 | "id": "Oj5MC7xPhMoq"
333 | },
334 | "outputs": [],
335 | "source": [
336 | "# Creating a function that returns the dataloader based on the dataframe and the specified train and validation batch size. \n",
337 | "\n",
338 | "def return_dataloader(df, tokenizer, train_batch_size, validation_batch_size, MAX_LEN, train_size=0.7):\n",
339 | " train_size = 0.7\n",
340 | " train_dataset=df.sample(frac=train_size,random_state=200)\n",
341 | " val_dataset=df.drop(train_dataset.index).reset_index(drop=True)\n",
342 | " train_dataset = train_dataset.reset_index(drop=True)\n",
343 | "\n",
344 | " print(\"FULL Dataset: {}\".format(df.shape))\n",
345 | " print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
346 | " print(\"VAL Dataset: {}\".format(val_dataset.shape))\n",
347 | "\n",
348 | " training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN)\n",
349 | " validation_set = CustomDataset(val_dataset, tokenizer, MAX_LEN)\n",
350 | "\n",
351 | " train_params = {'batch_size': train_batch_size,\n",
352 | " 'shuffle': True,\n",
353 | " 'num_workers': 1\n",
354 | " }\n",
355 | "\n",
356 | " val_params = {'batch_size': validation_batch_size,\n",
357 | " 'shuffle': True,\n",
358 | " 'num_workers': 1\n",
359 | " }\n",
360 | "\n",
361 | " training_loader = DataLoader(training_set, **train_params)\n",
362 | " validation_loader = DataLoader(validation_set, **val_params)\n",
363 | " \n",
364 | " return training_loader, validation_loader"
365 | ]
366 | },
367 | {
368 | "cell_type": "markdown",
369 | "metadata": {},
370 | "source": [
371 | "\n",
372 | "### Defining a Model/Network\n",
373 | "\n",
374 | "#### Neural Network\n",
375 | " - We will be creating a neural network with the `ModelClass`. \n",
376 | " - This network will have the Roberta Language model and a few by a `dropout` and `Linear` layer to obtain the final outputs. \n",
377 | " - The data will be fed to the Roberta Language model as defined in the dataset. \n",
378 | " - Final layer outputs is what will be compared to the `encoded_polarity` to determine the accuracy of models prediction. \n",
379 | " - We will initiate an instance of the network called `model`. This instance will be used for training and then to save the final trained model for future inference. \n",
380 | " - The `return_model` function is used in the `run()` to instantiate the model and set it up for TPU execution."
381 | ]
382 | },
383 | {
384 | "cell_type": "code",
385 | "execution_count": 9,
386 | "metadata": {
387 | "colab": {},
388 | "colab_type": "code",
389 | "id": "MxaCC_XGh0ev"
390 | },
391 | "outputs": [],
392 | "source": [
393 | "# Creating the customized model, by adding a drop out and a dense layer on top of roberta to get the final output for the model. \n",
394 | "\n",
395 | "class ModelClass(torch.nn.Module):\n",
396 | " def __init__(self):\n",
397 | " super(ModelClass, self).__init__()\n",
398 | " self.model_layer = RobertaModel.from_pretrained(\"roberta-base\")\n",
399 | " self.pre_classifier = torch.nn.Linear(768, 768)\n",
400 | " self.dropout = torch.nn.Dropout(0.3)\n",
401 | " self.classifier = torch.nn.Linear(768, 2)\n",
402 | "\n",
403 | " def forward(self, input_ids, attention_mask):\n",
404 | " output_1 = self.model_layer(input_ids=input_ids, attention_mask=attention_mask)\n",
405 | " hidden_state = output_1[0]\n",
406 | " pooler = hidden_state[:, 0]\n",
407 | " pooler = self.pre_classifier(pooler)\n",
408 | " pooler = torch.nn.ReLU()(pooler)\n",
409 | " pooler = self.dropout(pooler)\n",
410 | " output = self.classifier(pooler)\n",
411 | " return output\n"
412 | ]
413 | },
414 | {
415 | "cell_type": "code",
416 | "execution_count": 10,
417 | "metadata": {
418 | "colab": {},
419 | "colab_type": "code",
420 | "id": "tr7ueXG-B3Vz"
421 | },
422 | "outputs": [],
423 | "source": [
424 | "# Function to return model based on the defination of Model Class\n",
425 | "\n",
426 | "def return_model(device):\n",
427 | " model = ModelClass()\n",
428 | " model = model.to(device)\n",
429 | " return model"
430 | ]
431 | },
432 | {
433 | "cell_type": "code",
434 | "execution_count": 11,
435 | "metadata": {
436 | "colab": {},
437 | "colab_type": "code",
438 | "id": "rbY5DHfbDSuZ"
439 | },
440 | "outputs": [],
441 | "source": [
442 | "# Function to calcuate the accuracy of the model\n",
443 | "\n",
444 | "def calcuate_accu(big_idx, targets):\n",
445 | " n_correct = (big_idx==targets).sum().item()\n",
446 | " return n_correct"
447 | ]
448 | },
449 | {
450 | "cell_type": "markdown",
451 | "metadata": {},
452 | "source": [
453 | "\n",
454 | "### Fine Tuning the Model: Function\n",
455 | "\n",
456 | "Here we define a training function that trains the model on the training dataset created above, specified number of times (EPOCH), An epoch defines how many times the complete data will be passed through the network. \n",
457 | "\n",
458 | "This function is called in the `run()`\n",
459 | "\n",
460 | "Following events happen in this function to fine tune the neural network:\n",
461 | "- The `epoch`, `model`, `device` details, `testing_ dataloader`, `optimizer` and `loss_function` are passed to the `train ()` when its called from the `run()`\n",
462 | "- The dataloader passes data to the model based on the batch size.\n",
463 | "- The output from the neural network: `outputs` is compared to the `targets` tensor and loss is calcuated using `loss_function()`\n",
464 | "- Loss value is used to optimize the weights of the neurons in the network.\n",
465 | "- After every 100 steps the loss value and accuracy is logged in the wandb service. This log is then used to generate graphs for analysis. Such as [these](https://app.wandb.ai/abhimishra-91/transformers_tutorials_sentiment?workspace=user-abhimishra-91)\n",
466 | "- After every epoch the loss and accuracy value is printed in the console. Also, logged into the wandb service."
467 | ]
468 | },
469 | {
470 | "cell_type": "code",
471 | "execution_count": 12,
472 | "metadata": {
473 | "colab": {},
474 | "colab_type": "code",
475 | "id": "7GE89LJnkid7"
476 | },
477 | "outputs": [],
478 | "source": [
479 | "# Function to fine tune the model based on the epochs, model, tokenizer and other arguments\n",
480 | "\n",
481 | "def train(epoch, model, device, training_loader, optimizer, loss_function):\n",
482 | " n_correct = 0\n",
483 | " nb_tr_examples, nb_tr_steps = 0, 0\n",
484 | " tr_loss = 0\n",
485 | " model.train()\n",
486 | " for _,data in enumerate(training_loader, 0):\n",
487 | " ids = data['ids'].to(device, dtype = torch.long)\n",
488 | " mask = data['mask'].to(device, dtype = torch.long)\n",
489 | " targets = data['targets'].to(device, dtype = torch.long)\n",
490 | "\n",
491 | " outputs = model(ids, mask).squeeze()\n",
492 | " optimizer.zero_grad()\n",
493 | " loss = loss_function(outputs, targets)\n",
494 | " tr_loss += loss.item()\n",
495 | " big_val, big_idx = torch.max(outputs.data, dim=1)\n",
496 | " n_correct += calcuate_accu(big_idx, targets)\n",
497 | "\n",
498 | " nb_tr_steps += 1\n",
499 | " nb_tr_examples+=targets.size(0)\n",
500 | " \n",
501 | " if _%100==0:\n",
502 | " loss_step = tr_loss/nb_tr_steps\n",
503 | " accu_step = (n_correct*100)/nb_tr_examples \n",
504 | " wandb.log({\"Training Loss per 100 steps\": loss_step})\n",
505 | " wandb.log({\"Training Accuracy per 100 steps\": accu_step})\n",
506 | "\n",
507 | " optimizer.zero_grad()\n",
508 | " loss.backward()\n",
509 | " \n",
510 | " # # When using GPU or GPU\n",
511 | " # optimizer.step()\n",
512 | " \n",
513 | " # When using TPU\n",
514 | " xm.optimizer_step(optimizer)\n",
515 | " xm.mark_step()\n",
516 | "\n",
517 | " print(f'The Total Accuracy for Epoch {epoch}: {(n_correct*100)/nb_tr_examples}')\n",
518 | " epoch_loss = tr_loss/nb_tr_steps\n",
519 | " epoch_accu = (n_correct*100)/nb_tr_examples\n",
520 | " wandb.log({\"Training Loss Epoch\": epoch_loss})\n",
521 | " wandb.log({\"Training Accuracy Epoch\": epoch_accu})"
522 | ]
523 | },
524 | {
525 | "cell_type": "markdown",
526 | "metadata": {},
527 | "source": [
528 | "\n",
529 | "### Validating the Model Performance: Function\n",
530 | "\n",
531 | "During the validation stage we pass the unseen data(Validation Dataset), trained model, and device details to the function to perform the validation run. This step generates new encoded_sentiment value for dataset that it has not seen during the training session. \n",
532 | "\n",
533 | "This is then compared to the actual encoded_sentiment, to give us the Validation Accuracy and Loss.\n",
534 | "\n",
535 | "This function is called in the `run()`\n",
536 | "\n",
537 | "This unseen data is the 30% of `IMBD Dataset` which was seperated during the Dataset creation stage. \n",
538 | "During the validation stage the weights of the model are not updated. We use the generate method for generating new text for the summary. \n",
539 | "\n",
540 | "The generated validation accuracy and loss are logged to wandb for every 100th step and per epoch. "
541 | ]
542 | },
543 | {
544 | "cell_type": "code",
545 | "execution_count": 13,
546 | "metadata": {
547 | "colab": {},
548 | "colab_type": "code",
549 | "id": "2_1Hw4k-0M4r"
550 | },
551 | "outputs": [],
552 | "source": [
553 | "# Function to run the validation dataloader to validate the performance of the fine tuned model. \n",
554 | "\n",
555 | "def valid(epoch, model, device, validation_loader, loss_function):\n",
556 | " n_correct = 0; total = 0\n",
557 | " nb_tr_examples, nb_tr_steps = 0, 0\n",
558 | " tr_loss = 0\n",
559 | " model.eval()\n",
560 | " with torch.no_grad():\n",
561 | " for _,data in enumerate(validation_loader, 0):\n",
562 | " ids = data['ids'].to(device, dtype = torch.long)\n",
563 | " mask = data['mask'].to(device, dtype = torch.long)\n",
564 | " targets = data['targets'].to(device, dtype = torch.long)\n",
565 | "\n",
566 | " outputs = model(ids, mask).squeeze()\n",
567 | " loss = loss_function(outputs, targets)\n",
568 | " tr_loss += loss.item()\n",
569 | " big_val, big_idx = torch.max(outputs.data, dim=1)\n",
570 | " n_correct += calcuate_accu(big_idx, targets)\n",
571 | "\n",
572 | " nb_tr_steps += 1\n",
573 | " nb_tr_examples+=targets.size(0)\n",
574 | " \n",
575 | " if _%100==0:\n",
576 | " loss_step = tr_loss/nb_tr_steps\n",
577 | " accu_step = (n_correct*100)/nb_tr_examples \n",
578 | " wandb.log({\"Validation Loss per 100 steps\": loss_step})\n",
579 | " wandb.log({\"Validation Accuracy per 100 steps\": accu_step})\n",
580 | " \n",
581 | " epoch_loss = tr_loss/nb_tr_steps\n",
582 | " epoch_accu = (n_correct*100)/nb_tr_examples\n",
583 | " wandb.log({\"Validation Loss Epoch\": epoch_loss})\n",
584 | " wandb.log({\"Validation Accuracy Epoch\": epoch_accu})\n",
585 | " print(f'The Validation Accuracy: {(n_correct*100)/nb_tr_examples}')"
586 | ]
587 | },
588 | {
589 | "cell_type": "markdown",
590 | "metadata": {},
591 | "source": [
592 | "\n",
593 | "### Run Function\n",
594 | "\n",
595 | "The `run()` as the name suggests is the central location to run all the functions/flows created above in the notebook. The following steps are executed in the `run()`:\n",
596 | "\n",
597 | "\n",
598 | "\n",
599 | "#### Initializing WandB \n",
600 | "\n",
601 | "* The `run()` begins with initializing WandB run under a specific project. This command initiates a new run for each execution of this command. \n",
602 | "\n",
603 | "* We have seend wandb in action in one of the previous notebooks. Leveraging this notebook to log some additional metrics. \n",
604 | "\n",
605 | "* This particular tutorial is logged in the project: **[transformers_tutorials_sentiment](https://app.wandb.ai/abhimishra-91/transformers_tutorials_sentiment?workspace=user-abhimishra-91)**\n",
606 | "\n",
607 | "**One of the dadshboard from the project**\n",
608 | "\n",
609 | "\n",
610 | "* Visit the project page to see the details of different runs and what information is logged by the service. \n",
611 | "\n",
612 | "* Following the initialization of the WandB service we define configuration parameters that will be used across the tutorial such as `batch_size`, `epoch`, `learning_rate` etc.\n",
613 | "\n",
614 | "* These parameters are also passed to the WandB config. The config construct with all the parameters can be optimized using the Sweep service from WandB. Currently, that is outof scope of this tutorial. \n",
615 | "\n",
616 | "\n",
617 | "\n",
618 | "#### Importing and Pre-Processing the domain data\n",
619 | "\n",
620 | "We will be working with the data and preparing it for fine tuning purposes. \n",
621 | "*Assuming that the `IMDB Dataset.csv` is already downloaded in your `data` folder*\n",
622 | "\n",
623 | "* The file is imported as a dataframe and give it the headers as per the documentation.\n",
624 | "* Cleaning the file to remove the unwanted columns.\n",
625 | "* All these steps are done using the `Preprocess Class` defined above\n",
626 | "* The final Dataframe will be something like this:\n",
627 | "\n",
628 | "|review|encoded_polarity|\n",
629 | "|--|--|\n",
630 | "|summary-1|0|\n",
631 | "|summary-2|1|\n",
632 | "|summary-3|1|\n",
633 | "\n",
634 | "\n",
635 | "\n",
636 | "#### Creation of Dataset and Dataloader\n",
637 | "\n",
638 | "* The updated dataframe is divided into 70-20 ratio for test and validation. \n",
639 | "* Both the data-frames are passed to the `CustomerDataset` class for tokenization of the review and its sentiment.\n",
640 | "* The tokenization is done using roberta tokenizer.\n",
641 | "* Train and Validation parameters are defined and passed to the `pytorch Dataloader contstruct` to create `train` and `validation` data loaders.\n",
642 | "* These dataloaders will be passed to `train()` and `validate()` respectively for training and validation action.\n",
643 | "* The shape of datasets is printed in the console.\n",
644 | "* All these actions are performed using the `return_dataloader()` and `CustomDataset class` defined above.\n",
645 | "\n",
646 | "\n",
647 | "\n",
648 | "#### Neural Network and Optimizer\n",
649 | "\n",
650 | "* In this stage we define the model and optimizer that will be used for training and to update the weights of the network. \n",
651 | "* We are using the `roberta-base-uncased` transformer model for our project. You can read about the `RoBERTa model` and its features above. \n",
652 | "* The model is returned and instiated using the `return_model()` and `ModelClass`.\n",
653 | "* We are using the `Adam` optimizer for our project. This has been a standard for all our tutorials and is something that can be changed updated to see how different optimizer perform with different learning rates. \n",
654 | "* There is also a scope for doing more with Optimizer such a decay, momentum to dynamically update the Learning rate and other parameters. All those concepts have been kept out of scope for these tutorials. \n",
655 | "\n",
656 | "\n",
657 | "\n",
658 | "#### Training Model and Logging to WandB\n",
659 | "\n",
660 | "* Followed by that we call the `train()` with all the necessary parameters.\n",
661 | "* Loss and accuracy at every 100th step is logged to the WandB service. \n",
662 | "* Accuracy and end of every epoch is logged in WandB and also printed in the console.\n",
663 | "\n",
664 | "\n",
665 | "\n",
666 | "#### Validation\n",
667 | "\n",
668 | "* After the training is completed, the validation step is initiated.\n",
669 | "* As defined in the validation function, the model weights are not updated. We use the fine tuned model to generate encoded sentiment.\n",
670 | "* An output is printed on the console giving the accuracy at the end of Validation. "
671 | ]
672 | },
673 | {
674 | "cell_type": "code",
675 | "execution_count": 14,
676 | "metadata": {
677 | "colab": {},
678 | "colab_type": "code",
679 | "id": "zgZVNTmu8w0J"
680 | },
681 | "outputs": [],
682 | "source": [
683 | "def run():\n",
684 | " \n",
685 | " # WandB – Initialize a new run\n",
686 | " wandb.init(project=\"transformers_tutorials_sentiment\")\n",
687 | " \n",
688 | " # Defining some key variables that will be used later on in the training\n",
689 | " config = wandb.config \n",
690 | " config.MAX_LEN = 512\n",
691 | " config.TRAIN_BATCH_SIZE = 4\n",
692 | " config.VALID_BATCH_SIZE = 2\n",
693 | " config.EPOCHS = 2\n",
694 | " config.LEARNING_RATE = 1e-05\n",
695 | " tokenizer = RobertaTokenizerFast.from_pretrained('roberta-base')\n",
696 | "\n",
697 | " # Reading the dataset and pre-processing it for usage\n",
698 | " df = pd.read_csv('/content/drive/My Drive/Colab Notebooks/IMDB Dataset.csv', encoding='latin-1')\n",
699 | " pre = Preprocess(df)\n",
700 | " encoding_dict, df = pre.processing()\n",
701 | "\n",
702 | " # Creating the training and validation dataloader using the functions defined above\n",
703 | " training_loader, validation_loader = return_dataloader(df, tokenizer, config.TRAIN_BATCH_SIZE, config.VALID_BATCH_SIZE, config.MAX_LEN)\n",
704 | "\n",
705 | " # Defining the model based on the function and ModelClass defined above\n",
706 | " model = return_model(device)\n",
707 | "\n",
708 | " # Creating the loss function and optimizer\n",
709 | " loss_function = torch.nn.CrossEntropyLoss()\n",
710 | " optimizer = torch.optim.Adam(params = model.parameters(), lr=config.LEARNING_RATE)\n",
711 | "\n",
712 | " # Fine tuning the model using the train function:\n",
713 | " for epoch in range(config.EPOCHS):\n",
714 | " train(epoch, model, device, training_loader, optimizer, loss_function)\n",
715 | "\n",
716 | " # Running the validation function to validate the performance of the trained model\n",
717 | " valid(epoch, model, device, validation_loader, loss_function)"
718 | ]
719 | },
720 | {
721 | "cell_type": "code",
722 | "execution_count": 15,
723 | "metadata": {
724 | "colab": {
725 | "base_uri": "https://localhost:8080/",
726 | "height": 155
727 | },
728 | "colab_type": "code",
729 | "id": "MzQHSMj7IDIz",
730 | "outputId": "005fedb8-9be9-4809-d2e4-661823b98021",
731 | "scrolled": true
732 | },
733 | "outputs": [
734 | {
735 | "data": {
736 | "text/html": [
737 | "\n",
738 | " Logging results to Weights & Biases (Documentation).\n",
273 | " \n",
274 | "
\n",
320 | "\n",
275 | " \n",
281 | " \n",
282 | " \n",
283 | " \n",
276 | " pos \n",
277 | " sentence_idx \n",
278 | " word \n",
279 | " tag \n",
280 | " \n",
284 | " \n",
290 | " 0 \n",
285 | " NNS \n",
286 | " 1.0 \n",
287 | " Thousands \n",
288 | " O \n",
289 | " \n",
291 | " \n",
297 | " 1 \n",
292 | " IN \n",
293 | " 1.0 \n",
294 | " of \n",
295 | " O \n",
296 | " \n",
298 | " \n",
304 | " 2 \n",
299 | " NNS \n",
300 | " 1.0 \n",
301 | " demonstrators \n",
302 | " O \n",
303 | " \n",
305 | " \n",
311 | " 3 \n",
306 | " VBP \n",
307 | " 1.0 \n",
308 | " have \n",
309 | " O \n",
310 | " \n",
312 | " \n",
318 | " \n",
319 | "4 \n",
313 | " VBN \n",
314 | " 1.0 \n",
315 | " marched \n",
316 | " O \n",
317 | "
\n",
739 | " Project page: https://app.wandb.ai/abhimishra-91/transformers_tutorials_sentiment
\n",
740 | " Run page: https://app.wandb.ai/abhimishra-91/transformers_tutorials_sentiment/runs/1zwn4gbg
\n",
741 | " "
742 | ],
743 | "text/plain": [
744 | "