2 |
3 | # RATransformers 🐭
4 |
5 |  
6 |
7 | **RATransformers**, short for Relation-Aware Transformers, is a package built on top of [transformers 🤗](https://github.com/huggingface/transformers)
8 | that enables the training/fine-tuning of models with extra relation-aware input features.
9 |
10 |
11 | ### Example - Encoding a table in TableQA (Question Answering on Tabular Data)
12 | 
13 |
14 | [[Notebook Link](https://github.com/JoaoLages/RATransformers/blob/main/notebooks/TableQA_tabfact_example.ipynb)]
15 |
16 | In this example we can see that passing the table as text with no additional information to the model is a poor representation.
17 |
18 | With RATransformers 🐭 you are able to encode the table in a more structured way by passing specific relations within the input.
19 | RATransformers 🐭 also allows you to pass further features related with each input word/token.
20 |
21 | Check more examples in [[here](https://github.com/JoaoLages/RATransformers/blob/main/notebooks/)].
22 |
23 | ## Installation
24 |
25 | Install directly from PyPI:
26 |
27 | pip install ratransformers
28 |
29 | ## Usage
30 |
31 | ```python
32 | from ratransformers import RATransformer
33 | from transformers import AutoModelForSequenceClassification
34 |
35 |
36 | ratransformer = RATransformer(
37 | "nielsr/tapex-large-finetuned-tabfact", # define the 🤗 model you want to load
38 | relation_kinds=['is_value_of_column', 'is_from_same_row'], # define the relations that you want to model in the input
39 | model_cls=AutoModelForSequenceClassification, # define the model class
40 | pretrained_tokenizer_name_or_path='facebook/bart-large' # define the tokenizer you want to load (in case it is not the same as the model)
41 | )
42 | model = ratransformer.model
43 | tokenizer = ratransformer.tokenizer
44 | ```
45 |
46 | With only these steps your RATransformer 🐭 is ready to be trained.
47 |
48 | More implementation details in [the examples here](https://github.com/JoaoLages/RATransformers/blob/main/notebooks/).
49 |
50 | ## How does it work?
51 | We modify the self-attention layers of the transformer model as explained in the section 3 of [the RAT-SQL paper](https://arxiv.org/pdf/1911.04942.pdf).
52 |
53 | ## Supported Models
54 | Currently we support a limited number of transformer models:
55 | - [BART](https://huggingface.co/docs/transformers/model_doc/bart)
56 | - [BERT](https://huggingface.co/docs/transformers/model_doc/bert)
57 | - [GPT-2](https://huggingface.co/docs/transformers/model_doc/gpt2)
58 | - [RoBERTa](https://huggingface.co/docs/transformers/model_doc/roberta)
59 | - [T5](https://huggingface.co/docs/transformers/model_doc/t5)
60 | - [LongT5](https://huggingface.co/docs/transformers/model_doc/longt5)
61 |
62 | Want another model? Feel free to open an [Issue](https://github.com/JoaoLages/RATransformers/issues) or create a [Pull Request](https://github.com/JoaoLages/RATransformers/pulls) and let's get started 🚀
63 |
--------------------------------------------------------------------------------
/src/ratransformers/gpt2.py:
--------------------------------------------------------------------------------
1 | from transformers.models.gpt2.modeling_gpt2 import GPT2Attention
2 | import torch.nn as nn
3 | import torch
4 |
5 |
6 | class GPT2RelationalAttention(GPT2Attention):
7 | def __init__(self, *args, num_relation_kinds: int, use_same_relation_kv_emb: bool = True, **kwargs):
8 | super().__init__(*args, **kwargs)
9 | self.num_relation_kinds = num_relation_kinds
10 | self.relation_k_emb = nn.Embedding(num_relation_kinds + 1, self.head_dim, padding_idx=0)
11 | if use_same_relation_kv_emb:
12 | self.relation_v_emb = self.relation_k_emb
13 | else:
14 | self.relation_v_emb = nn.Embedding(num_relation_kinds + 1, self.head_dim, padding_idx=0)
15 | self.input_relation_kinds = [] # will hold (batch, seq_length, seq_length, num_relation_kinds)
16 |
17 | def forward(
18 | self,
19 | hidden_states,
20 | layer_past=None,
21 | attention_mask=None,
22 | head_mask=None,
23 | encoder_hidden_states=None,
24 | encoder_attention_mask=None,
25 | use_cache=False,
26 | output_attentions=False,
27 | ):
28 | if encoder_hidden_states is not None:
29 | if not hasattr(self, "q_attn"):
30 | raise ValueError(
31 | "If class is used as cross attention, the weights `q_attn` have to be defined. "
32 | "Please make sure to instantiate class with `GPT2Attention(..., is_cross_attention=True)`."
33 | )
34 |
35 | query = self.q_attn(hidden_states)
36 | key, value = self.c_attn(encoder_hidden_states).split(self.split_size, dim=2)
37 | attention_mask = encoder_attention_mask
38 | else:
39 | query, key, value = self.c_attn(hidden_states).split(self.split_size, dim=2)
40 |
41 | batch_size, seq_length = hidden_states.shape[:2]
42 |
43 | assert len(self.input_relation_kinds) == 1
44 | input_relation_kinds = self.input_relation_kinds[0]
45 | assert input_relation_kinds.shape == (batch_size, seq_length, seq_length)
46 |
47 | query = self._split_heads(query, self.num_heads, self.head_dim)
48 | key = self._split_heads(key, self.num_heads, self.head_dim)
49 | value = self._split_heads(value, self.num_heads, self.head_dim)
50 |
51 | if layer_past is not None:
52 | past_key, past_value = layer_past
53 | key = torch.cat((past_key, key), dim=-2)
54 | value = torch.cat((past_value, value), dim=-2)
55 |
56 | if use_cache is True:
57 | present = (key, value)
58 | else:
59 | present = None
60 |
61 | attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask)
62 |
63 | attn_output = self._merge_heads(attn_output, self.num_heads, self.head_dim)
64 | attn_output = self.c_proj(attn_output)
65 | attn_output = self.resid_dropout(attn_output)
66 |
67 | outputs = (attn_output, present)
68 | if output_attentions:
69 | outputs += (attn_weights,)
70 |
71 | return outputs # a, present, (attentions)
72 |
73 | def _attn(self, query, key, value, attention_mask=None, head_mask=None):
74 |
75 | input_relation_kinds = self.input_relation_kinds[0]
76 | relation_k_embeds = self.relation_k_emb(input_relation_kinds)
77 |
78 | attn_weights = torch.matmul(query, key.transpose(-1, -2))
79 |
80 | # q_t is [batch, seq_length, n_heads, dim_per_head]
81 | q_t = query.permute(0, 2, 1, 3)
82 |
83 | # r_t is [batch, seq_length, dim_per_head, seq_length]
84 | r_t = relation_k_embeds.transpose(-2, -1)
85 |
86 | q_tr_t_matmul = torch.matmul(q_t, r_t) # [batch, seq_length, n_heads, seq_length]
87 | q_tr_tmatmul_t = q_tr_t_matmul.permute(0, 2, 1, 3) # [batch, n_heads, seq_length, seq_length]
88 |
89 | # Add to scores
90 | attn_weights += q_tr_tmatmul_t
91 |
92 | if self.scale_attn_weights:
93 | attn_weights = attn_weights / (float(value.size(-1)) ** 0.5)
94 |
95 | if not self.is_cross_attention:
96 | # if only "normal" attention layer implements causal mask
97 | query_length, key_length = query.size(-2), key.size(-2)
98 | causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length].bool()
99 | attn_weights = torch.where(causal_mask, attn_weights, self.masked_bias.to(attn_weights.dtype))
100 |
101 | if attention_mask is not None:
102 | # Apply the attention mask
103 | attn_weights = attn_weights + attention_mask
104 |
105 | attn_weights = nn.Softmax(dim=-1)(attn_weights)
106 | attn_weights = self.attn_dropout(attn_weights)
107 |
108 | # Mask heads if we want to
109 | if head_mask is not None:
110 | attn_weights = attn_weights * head_mask
111 |
112 | attn_output = torch.matmul(attn_weights, value)
113 |
114 | relation_v_embeds = self.relation_v_emb(input_relation_kinds)
115 |
116 | # w_t is [batch, seq_length, n_heads, seq_length]
117 | w_t = attn_weights.permute(0, 2, 1, 3)
118 |
119 | # [batch, seq_length, n_heads, seq_length]
120 | w_tr_matmul = torch.matmul(w_t, relation_v_embeds)
121 |
122 | attn_output += w_tr_matmul.permute(0, 2, 1, 3)
123 |
124 | return attn_output, attn_weights
--------------------------------------------------------------------------------
/src/ratransformers/bert.py:
--------------------------------------------------------------------------------
1 | import math
2 |
3 | from transformers.models.bert.modeling_bert import BertSelfAttention
4 | import torch.nn as nn
5 | import torch
6 |
7 |
8 | class BertRelationalSelfAttention(BertSelfAttention):
9 | def __init__(self, *args, num_relation_kinds: int, use_same_relation_kv_emb: bool = True, **kwargs):
10 | super().__init__(*args, **kwargs)
11 | self.num_relation_kinds = num_relation_kinds
12 | self.relation_k_emb = nn.Embedding(num_relation_kinds + 1, self.attention_head_size, padding_idx=0)
13 | if use_same_relation_kv_emb:
14 | self.relation_v_emb = self.relation_k_emb
15 | else:
16 | self.relation_v_emb = nn.Embedding(num_relation_kinds + 1, self.attention_head_size, padding_idx=0)
17 | self.input_relation_kinds = [] # will hold (batch, seq_length, seq_length, num_relation_kinds)
18 |
19 | def forward(
20 | self,
21 | hidden_states,
22 | attention_mask=None,
23 | head_mask=None,
24 | encoder_hidden_states=None,
25 | encoder_attention_mask=None,
26 | past_key_value=None,
27 | output_attentions=False,
28 | ):
29 |
30 | batch_size, seq_length = hidden_states.shape[:2]
31 |
32 | assert len(self.input_relation_kinds) == 1
33 | input_relation_kinds = self.input_relation_kinds[0]
34 | assert input_relation_kinds.shape == (batch_size, seq_length, seq_length)
35 |
36 | # (batch_size, seq_length, seq_length, self.num_relation_kinds, self.inner_dim // num_relation_kinds)
37 | relation_k_embeds = self.relation_k_emb(input_relation_kinds)
38 | relation_v_embeds = self.relation_v_emb(input_relation_kinds)
39 |
40 | mixed_query_layer = self.query(hidden_states)
41 |
42 | # If this is instantiated as a cross-attention module, the keys
43 | # and values come from an encoder; the attention mask needs to be
44 | # such that the encoder's padding tokens are not attended to.
45 | is_cross_attention = encoder_hidden_states is not None
46 |
47 | if is_cross_attention and past_key_value is not None:
48 | # reuse k,v, cross_attentions
49 | key_layer = past_key_value[0]
50 | value_layer = past_key_value[1]
51 | attention_mask = encoder_attention_mask
52 | elif is_cross_attention:
53 | key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
54 | value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
55 | attention_mask = encoder_attention_mask
56 | elif past_key_value is not None:
57 | key_layer = self.transpose_for_scores(self.key(hidden_states))
58 | value_layer = self.transpose_for_scores(self.value(hidden_states))
59 | key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
60 | value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
61 | else:
62 | key_layer = self.transpose_for_scores(self.key(hidden_states))
63 | value_layer = self.transpose_for_scores(self.value(hidden_states))
64 |
65 | query_layer = self.transpose_for_scores(mixed_query_layer)
66 |
67 | if self.is_decoder:
68 | # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
69 | # Further calls to cross_attention layer can then reuse all cross-attention
70 | # key/value_states (first "if" case)
71 | # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
72 | # all previous decoder key/value_states. Further calls to uni-directional self-attention
73 | # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
74 | # if encoder bi-directional self-attention `past_key_value` is always `None`
75 | past_key_value = (key_layer, value_layer)
76 |
77 | # Take the dot product between "query" and "key" to get the raw attention scores.
78 | attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
79 |
80 | # q_t is [batch, seq_length, n_heads, dim_per_head]
81 | q_t = query_layer.permute(0, 2, 1, 3)
82 |
83 | # r_t is [batch, seq_length, dim_per_head, seq_length]
84 | r_t = relation_k_embeds.transpose(-2, -1)
85 |
86 | q_tr_t_matmul = torch.matmul(q_t, r_t) # [batch, seq_length, n_heads, seq_length]
87 | q_tr_tmatmul_t = q_tr_t_matmul.permute(0, 2, 1, 3) # [batch, n_heads, seq_length, seq_length]
88 |
89 | # Add to scores
90 | attention_scores += q_tr_tmatmul_t
91 |
92 | if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
93 | seq_length = hidden_states.size()[1]
94 | position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1)
95 | position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1)
96 | distance = position_ids_l - position_ids_r
97 | positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
98 | positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility
99 |
100 | if self.position_embedding_type == "relative_key":
101 | relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
102 | attention_scores = attention_scores + relative_position_scores
103 | elif self.position_embedding_type == "relative_key_query":
104 | relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
105 | relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
106 | attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
107 |
108 | attention_scores = attention_scores / math.sqrt(self.attention_head_size)
109 | if attention_mask is not None:
110 | # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
111 | attention_scores = attention_scores + attention_mask
112 |
113 | # Normalize the attention scores to probabilities.
114 | attention_probs = nn.Softmax(dim=-1)(attention_scores)
115 |
116 | # This is actually dropping out entire tokens to attend to, which might
117 | # seem a bit unusual, but is taken from the original Transformer paper.
118 | attention_probs = self.dropout(attention_probs)
119 |
120 | # Mask heads if we want to
121 | if head_mask is not None:
122 | attention_probs = attention_probs * head_mask
123 |
124 | context_layer = torch.matmul(attention_probs, value_layer)
125 |
126 | # w_t is [batch, seq_length, n_heads, seq_length]
127 | w_t = attention_probs.permute(0, 2, 1, 3)
128 |
129 | # [batch, seq_length, n_heads, seq_length]
130 | w_tr_matmul = torch.matmul(w_t, relation_v_embeds)
131 |
132 | context_layer += w_tr_matmul.permute(0, 2, 1, 3)
133 |
134 | context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
135 | new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
136 | context_layer = context_layer.view(*new_context_layer_shape)
137 |
138 | outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
139 |
140 | if self.is_decoder:
141 | outputs = outputs + (past_key_value,)
142 | return outputs
143 |
--------------------------------------------------------------------------------
/notebooks/NER_conll_example.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "id": "0b4a5857",
7 | "metadata": {
8 | "pycharm": {
9 | "name": "#%%\n"
10 | }
11 | },
12 | "outputs": [],
13 | "source": [
14 | "!pip install datasets"
15 | ]
16 | },
17 | {
18 | "cell_type": "code",
19 | "execution_count": 2,
20 | "id": "636ff184",
21 | "metadata": {
22 | "pycharm": {
23 | "name": "#%%\n"
24 | }
25 | },
26 | "outputs": [
27 | {
28 | "name": "stderr",
29 | "output_type": "stream",
30 | "text": [
31 | "Reusing dataset conll2003 (/home/ola/.cache/huggingface/datasets/conll2003/conll2003/1.0.0/63f4ebd1bcb7148b1644497336fd74643d4ce70123334431a3c053b7ee4e96ee)\n"
32 | ]
33 | },
34 | {
35 | "data": {
36 | "application/vnd.jupyter.widget-view+json": {
37 | "model_id": "da46312ea7114898bc51966b2fe78766",
38 | "version_major": 2,
39 | "version_minor": 0
40 | },
41 | "text/plain": [
42 | " 0%| | 0/3 [00:00 Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
27 | """Input shape: Batch x Time x Channel"""
28 |
29 | # if key_value_states are provided this layer is used as a cross-attention layer
30 | # for the decoder
31 | is_cross_attention = key_value_states is not None
32 | bsz, tgt_len, embed_dim = hidden_states.size()
33 |
34 | assert len(self.input_relation_kinds) == 1
35 | input_relation_kinds = self.input_relation_kinds[0]
36 | assert input_relation_kinds.shape == (bsz, tgt_len, tgt_len)
37 |
38 | # (batch_size, seq_length, seq_length, self.num_relation_kinds, self.inner_dim // num_relation_kinds)
39 | relation_k_embeds = self.relation_k_emb(input_relation_kinds)
40 | relation_v_embeds = self.relation_v_emb(input_relation_kinds)
41 |
42 | # get query proj
43 | query_states = self.q_proj(hidden_states) * self.scaling
44 | # get key, value proj
45 | if is_cross_attention and past_key_value is not None:
46 | # reuse k,v, cross_attentions
47 | key_states = past_key_value[0]
48 | value_states = past_key_value[1]
49 | elif is_cross_attention:
50 | # cross_attentions
51 | key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
52 | value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
53 | elif past_key_value is not None:
54 | # reuse k, v, self_attention
55 | key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
56 | value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
57 | key_states = torch.cat([past_key_value[0], key_states], dim=2)
58 | value_states = torch.cat([past_key_value[1], value_states], dim=2)
59 | else:
60 | # self_attention
61 | key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
62 | value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
63 |
64 | if self.is_decoder:
65 | # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
66 | # Further calls to cross_attention layer can then reuse all cross-attention
67 | # key/value_states (first "if" case)
68 | # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
69 | # all previous decoder key/value_states. Further calls to uni-directional self-attention
70 | # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
71 | # if encoder bi-directional self-attention `past_key_value` is always `None`
72 | past_key_value = (key_states, value_states)
73 |
74 | proj_shape = (bsz * self.num_heads, -1, self.head_dim)
75 | query_states = self._shape(query_states, tgt_len, bsz)
76 | src_len = key_states.size(2)
77 |
78 | # compute scores
79 | attn_weights = torch.matmul(
80 | query_states, key_states.transpose(3, 2)
81 | ) # equivalent of torch.einsum("bnqd,bnkd->bnqk", query_states, key_states), compatible with onnx op>9
82 |
83 | # q_t is [batch, seq_length, n_heads, dim_per_head]
84 | q_t = query_states.permute(0, 2, 1, 3)
85 |
86 | # r_t is [batch, seq_length, dim_per_head, seq_length]
87 | r_t = relation_k_embeds.transpose(-2, -1)
88 |
89 | q_tr_t_matmul = torch.matmul(q_t, r_t) # [batch, seq_length, n_heads, seq_length]
90 | q_tr_tmatmul_t = q_tr_t_matmul.permute(0, 2, 1, 3) # [batch, n_heads, seq_length, seq_length]
91 |
92 | # Add to scores
93 | attn_weights += q_tr_tmatmul_t
94 | attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
95 |
96 | if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):
97 | raise ValueError(
98 | f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}"
99 | )
100 |
101 | if attention_mask is not None:
102 | if attention_mask.size() != (bsz, 1, tgt_len, src_len):
103 | raise ValueError(
104 | f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}"
105 | )
106 | attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
107 | attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
108 |
109 | attn_weights = nn.functional.softmax(attn_weights, dim=-1)
110 |
111 | if layer_head_mask is not None:
112 | if layer_head_mask.size() != (self.num_heads,):
113 | raise ValueError(
114 | f"Head mask for a single layer should be of size {(self.num_heads,)}, but is {layer_head_mask.size()}"
115 | )
116 | attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
117 | attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
118 |
119 | if output_attentions:
120 | # this operation is a bit awkward, but it's required to
121 | # make sure that attn_weights keeps its gradient.
122 | # In order to do so, attn_weights have to be reshaped
123 | # twice and have to be reused in the following
124 | attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
125 | attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
126 | else:
127 | attn_weights_reshaped = None
128 |
129 | attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
130 |
131 | attn_output = torch.bmm(attn_probs, value_states.view(*proj_shape))
132 |
133 | # w_t is [batch, seq_length, n_heads, seq_length]
134 | w_t = attn_probs.view(bsz, self.num_heads, tgt_len, src_len).permute(0, 2, 1, 3)
135 |
136 | # [batch, seq_length, n_heads, seq_length]
137 | w_tr_matmul = torch.matmul(w_t, relation_v_embeds)
138 |
139 | attn_output += w_tr_matmul.permute(0, 2, 1, 3).view(bsz * self.num_heads, tgt_len, self.head_dim)
140 |
141 | if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim):
142 | raise ValueError(
143 | f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output.size()}"
144 | )
145 |
146 | attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
147 | attn_output = attn_output.transpose(1, 2)
148 | attn_output = attn_output.reshape(bsz, tgt_len, embed_dim)
149 |
150 | attn_output = self.out_proj(attn_output)
151 |
152 | return attn_output, attn_weights_reshaped, past_key_value
--------------------------------------------------------------------------------
/notebooks/TableQA_tabfact_example.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "id": "6ac23e34",
6 | "metadata": {},
7 | "source": [
8 | "In this notebook we will show how to use RATransformers 🐭 to encode your data as the image shows.\n",
9 | "\n",
10 | ""
11 | ]
12 | },
13 | {
14 | "cell_type": "code",
15 | "execution_count": null,
16 | "id": "98ee93fc",
17 | "metadata": {},
18 | "outputs": [],
19 | "source": [
20 | "!pip install pandas"
21 | ]
22 | },
23 | {
24 | "cell_type": "code",
25 | "execution_count": 1,
26 | "id": "7841fcd4",
27 | "metadata": {},
28 | "outputs": [],
29 | "source": [
30 | "import ratransformers\n",
31 | "import pandas as pd\n",
32 | "from transformers import BartTokenizerFast, BartForSequenceClassification\n",
33 | "\n",
34 | "\n",
35 | "ratransformer = ratransformers.RATransformer(\n",
36 | " \"nielsr/tapex-large-finetuned-tabfact\", # define the 🤗 model you want to load\n",
37 | " relation_kinds=['is_value_of_column', 'is_from_same_row'], # define the relations that you want to model in the input\n",
38 | " tokenizer_cls=BartTokenizerFast, # define the tokenizer class \n",
39 | " model_cls=BartForSequenceClassification, # define the model class\n",
40 | " pretrained_tokenizer_name_or_path='facebook/bart-large' # define the tokenizer you want to load (in case it is not the same as the model)\n",
41 | ")\n",
42 | "model = ratransformer.model\n",
43 | "tokenizer = ratransformer.tokenizer\n",
44 | "\n",
45 | "# create table\n",
46 | "data = {'Actors': [\"Brad Pitt\", \"Leonardo Di Caprio\", \"George Clooney\"], 'Number of movies': [\"87\", \"53\", \"69\"]}\n",
47 | "table = pd.DataFrame.from_dict(data)\n",
48 | "\n",
49 | "# turn into dict\n",
50 | "table_dict = {\"header\": list(table.columns), \"rows\": [list(row.values) for i,row in table.iterrows()]}"
51 | ]
52 | },
53 | {
54 | "cell_type": "code",
55 | "execution_count": 2,
56 | "id": "df24f04d",
57 | "metadata": {},
58 | "outputs": [
59 | {
60 | "data": {
61 | "text/html": [
62 | ""
254 | ]
255 | },
256 | "metadata": {},
257 | "output_type": "display_data"
258 | },
259 | {
260 | "data": {
261 | "text/plain": [
262 | "{'eval_loss': 1.0085068941116333,\n",
263 | " 'eval_runtime': 1324.2969,\n",
264 | " 'eval_samples_per_second': 0.781}"
265 | ]
266 | },
267 | "execution_count": 10,
268 | "metadata": {},
269 | "output_type": "execute_result"
270 | }
271 | ],
272 | "source": [
273 | "# Set trainer\n",
274 | "trainer = Seq2SeqTrainer(\n",
275 | " model=model, \n",
276 | " args=training_args,\n",
277 | " train_dataset=train_d, \n",
278 | " eval_dataset=val_d, \n",
279 | " tokenizer=tokenizer,\n",
280 | " callbacks=[EarlyStoppingCallback()]\n",
281 | ")\n",
282 | "\n",
283 | "# get performance before training\n",
284 | "trainer.evaluate()"
285 | ]
286 | },
287 | {
288 | "cell_type": "code",
289 | "execution_count": 11,
290 | "id": "23b5a658",
291 | "metadata": {
292 | "scrolled": false
293 | },
294 | "outputs": [
295 | {
296 | "data": {
297 | "text/html": [
298 | "\n",
299 | " \n",
300 | " \n",
301 | "
\n",
302 | " [ 2000/100000 2:56:39 < 144:24:58, 0.19 it/s, Epoch 0/24]\n",
303 | "
\n",
305 | " \n",
306 | " \n",
307 | " | Step | \n",
308 | " Training Loss | \n",
309 | " Validation Loss | \n",
310 | "
\n",
311 | " \n",
312 | " \n",
313 | " \n",
314 | " | 1000 | \n",
315 | " 0.099700 | \n",
316 | " 0.325624 | \n",
317 | "
\n",
318 | " \n",
319 | " | 2000 | \n",
320 | " 0.060500 | \n",
321 | " 0.348871 | \n",
322 | "
\n",
323 | " \n",
324 | "
"
325 | ],
326 | "text/plain": [
327 | ""
328 | ]
329 | },
330 | "metadata": {},
331 | "output_type": "display_data"
332 | },
333 | {
334 | "data": {
335 | "text/plain": [
336 | "TrainOutput(global_step=2000, training_loss=0.10654444885253907, metrics={'train_runtime': 10601.4604, 'train_samples_per_second': 9.433, 'total_flos': 0, 'epoch': 0.47})"
337 | ]
338 | },
339 | "execution_count": 11,
340 | "metadata": {},
341 | "output_type": "execute_result"
342 | }
343 | ],
344 | "source": [
345 | "# train until early stopping\n",
346 | "trainer.train()"
347 | ]
348 | },
349 | {
350 | "cell_type": "code",
351 | "execution_count": 12,
352 | "id": "06da266f",
353 | "metadata": {},
354 | "outputs": [
355 | {
356 | "data": {
357 | "text/html": [
358 | "\n",
359 | " \n",
360 | " \n",
361 | "
\n",
362 | " [259/259 20:01]\n",
363 | "
\n",
424 | " \n",
425 | "
\n",
426 | " [259/259 15:35]\n",
427 | "