├── .gitignore ├── .ipynb_checkpoints ├── ConfusionMatrix-checkpoint.ipynb └── README-checkpoint.md ├── Bert ├── .ipynb_checkpoints │ ├── Bert-checkpoint.ipynb │ └── Bert1-checkpoint.ipynb ├── Bert.ipynb └── Bert1.ipynb ├── ConfusionMatrix.ipynb ├── NLP ├── 分词 │ ├── .ipynb_checkpoints │ │ └── starter_code-checkpoint.ipynb │ ├── starter_code.ipynb │ └── 综合类中文词库.xlsx ├── 拼写纠错 │ ├── .ipynb_checkpoints │ │ ├── learn-checkpoint.ipynb │ │ ├── spell-errors-checkpoint.txt │ │ ├── starter_code-checkpoint.ipynb │ │ ├── testdata-checkpoint.txt │ │ └── vocab-checkpoint.txt │ ├── learn.ipynb │ ├── spell-errors.txt │ ├── starter_code.ipynb │ ├── testdata.txt │ └── vocab.txt └── 问答系统 │ ├── .ipynb_checkpoints │ ├── code-checkpoint.ipynb │ ├── learn_nltk_python-checkpoint.ipynb │ └── starter_code-checkpoint.ipynb │ ├── code.ipynb │ ├── learn_nltk_python.ipynb │ └── starter_code.ipynb ├── NeuralFrameWork ├── .ipynb_checkpoints │ ├── 2-checkpoint.ipynb │ ├── 3-checkpoint.ipynb │ ├── All-checkpoint.ipynb │ ├── Build-neural-network-from-scrach-Lesson-02 (2)-checkpoint.ipynb │ ├── Lesson-01-checkpoint.ipynb │ ├── Lesson-01-代码复现与参考答案-checkpoint.ipynb │ └── Lesson-03-checkpoint.ipynb ├── 2.ipynb ├── 3.ipynb ├── All.ipynb ├── Build-neural-network-from-scrach-Lesson-02 (2).ipynb ├── Lesson-01-代码复现与参考答案.ipynb ├── Lesson-01.ipynb └── Lesson-03.ipynb ├── README.md ├── TF_AlexNet ├── .ipynb_checkpoints │ ├── AlexNetModel-checkpoint.ipynb │ ├── train_AlexNet-checkpoint.ipynb │ └── train_AlexNet_GPU-checkpoint.ipynb ├── AlexNetModel.ipynb ├── class_indices.json ├── train_AlexNet.ipynb └── train_AlexNet_GPU.ipynb ├── TF_GNN └── GCN │ ├── .ipynb_checkpoints │ ├── Cora_Dataset-checkpoint.ipynb │ ├── Cora_EDA-checkpoint.ipynb │ ├── GCN_numpy-checkpoint.ipynb │ └── GCN_tensorflow-checkpoint.ipynb │ ├── Cora_Dataset.ipynb │ ├── Cora_EDA.ipynb │ ├── GCN_numpy.ipynb │ └── GCN_tensorflow.ipynb ├── TF_GoogLeNet ├── .ipynb_checkpoints │ ├── Model-checkpoint.ipynb │ ├── Model_add_bn-checkpoint.ipynb │ ├── predict-checkpoint.ipynb │ ├── trainCPU-checkpoint.ipynb │ └── trainGPU-checkpoint.ipynb ├── Model.ipynb ├── Model_add_bn.ipynb ├── class_indices.json ├── model.png ├── predict.ipynb ├── trainCPU.ipynb └── trainGPU.ipynb ├── TF_VGG ├── .ipynb_checkpoints │ ├── VGGModel-checkpoint.ipynb │ ├── class_indices-checkpoint.json │ ├── fine_train_vgg16-checkpoint.ipynb │ ├── predict-checkpoint.ipynb │ ├── read_ckpt-checkpoint.ipynb │ ├── trainVGG-checkpoint.ipynb │ └── train_VGG_GPU-checkpoint.ipynb ├── VGGModel.ipynb ├── class_indices.json ├── fine_train_vgg16.ipynb ├── predict.ipynb ├── read_ckpt.ipynb ├── trainVGG.ipynb └── train_VGG_GPU.ipynb ├── Torch_ResNet ├── .gitignore ├── .ipynb_checkpoints │ ├── model-checkpoint.ipynb │ ├── predict-checkpoint.ipynb │ └── train-checkpoint.ipynb ├── class_indices.json ├── model.ipynb ├── predict.ipynb └── train.ipynb ├── Transformer ├── .ipynb_checkpoints │ ├── Transformer&Bert_exam_solution-checkpoint.ipynb │ ├── Transformer-checkpoint.ipynb │ ├── Transformer2-checkpoint.ipynb │ ├── Transformer3-checkpoint.ipynb │ ├── learn_spacy-checkpoint.ipynb │ └── self_attention-checkpoint.ipynb ├── Transformer&Bert_exam_solution.ipynb ├── Transformer.ipynb ├── Transformer2.ipynb ├── Transformer3.ipynb ├── learn_spacy.ipynb └── self_attention.ipynb ├── dgl_GCN ├── .ipynb_checkpoints │ ├── gcn-checkpoint.ipynb │ └── gcn_mp-checkpoint.ipynb ├── gcn.ipynb └── gcn_mp.ipynb ├── dgl_torch_GNN └── study_dgl │ ├── .ipynb_checkpoints │ ├── STGCN-checkpoint.ipynb │ ├── predict-checkpoint.ipynb │ ├── resnet-checkpoint.ipynb │ ├── train-checkpoint.ipynb │ └── 图生成模型-checkpoint.ipynb │ ├── STGCN.ipynb │ ├── predict.ipynb │ ├── resnet.ipynb │ ├── train.ipynb │ └── 图生成模型.ipynb ├── node2vec ├── .ipynb_checkpoints │ ├── learn_networkx-checkpoint.ipynb │ └── main-checkpoint.ipynb ├── emb │ └── karate.emb ├── karate.edgelist ├── learn_networkx.ipynb └── main.ipynb ├── tensor_numpy_GCN_NN ├── .ipynb_checkpoints │ ├── gcn-checkpoint.ipynb │ ├── numpy_nn-checkpoint.ipynb │ └── torch_tensor_nn-checkpoint.ipynb ├── gcn.ipynb ├── numpy_nn.ipynb └── torch_tensor_nn.ipynb └── tulip.jpg /.gitignore: -------------------------------------------------------------------------------- 1 | */save_weights/* 2 | */callbacks/* 3 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/README-checkpoint.md: -------------------------------------------------------------------------------- 1 | # DL-paper 2 | 花图片数据集 3 | 数据集下载：https://pan.baidu.com/s/1iQM6MWXz7bG27-uFWGlt8w 提取码:dw2d -------------------------------------------------------------------------------- /Bert/.ipynb_checkpoints/Bert-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# -*- coding: utf-8 -*-\n", 10 | "\"\"\"BERT-Torch\n", 11 | "\n", 12 | "Automatically generated by Colaboratory.\n", 13 | "\n", 14 | "Original file is located at\n", 15 | " https://colab.research.google.com/drive/1LVhb99B-YQJ1bGnaWIX-2bgANy78zAAt\n", 16 | "\"\"\"\n", 17 | "\n", 18 | "'''\n", 19 | " code by Tae Hwan Jung(Jeff Jung) @graykode, modify by wmathor\n", 20 | " Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch\n", 21 | " https://github.com/JayParks/transformer, https://github.com/dhlee347/pytorchic-bert\n", 22 | " https://github.com/wmathor/nlp-tutorial/tree/master/5-2.BERT\n", 23 | "'''\n", 24 | "import re\n", 25 | "import math\n", 26 | "import torch\n", 27 | "import numpy as np\n", 28 | "from random import *\n", 29 | "import torch.nn as nn\n", 30 | "import torch.optim as optim\n", 31 | "import torch.utils.data as Data\n", 32 | "\n", 33 | "text = (\n", 34 | " 'Hello, how are you? I am Romeo.\\n' # R\n", 35 | " 'Hello, Romeo My name is Juliet. Nice to meet you.\\n' # J\n", 36 | " 'Nice meet you too. How are you today?\\n' # R\n", 37 | " 'Great. My baseball team won the competition.\\n' # J\n", 38 | " 'Oh Congratulations, Juliet\\n' # R\n", 39 | " 'Thank you Romeo\\n' # J\n", 40 | " 'Where are you going today?\\n' # R\n", 41 | " 'I am going shopping. What about you?\\n' # J\n", 42 | " 'I am going to visit my grandmother. she is not very well' # R\n", 43 | ")\n", 44 | "sentences = re.sub(\"[.,!?\\\\-]\", '', text.lower()).split('\\n') # filter '.', ',', '?', '!'\n", 45 | "word_list = list(set(\" \".join(sentences).split())) # ['hello', 'how', 'are', 'you',...]\n", 46 | "word2idx = {'[PAD]': 0, '[CLS]': 1, '[SEP]': 2, '[MASK]': 3}\n", 47 | "for i, w in enumerate(word_list):\n", 48 | " word2idx[w] = i + 4\n", 49 | "idx2word = {i: w for i, w in enumerate(word2idx)}\n", 50 | "vocab_size = len(word2idx)\n", 51 | "\n", 52 | "token_list = list()\n", 53 | "for sentence in sentences:\n", 54 | " arr = [word2idx[s] for s in sentence.split()]\n", 55 | " token_list.append(arr)\n", 56 | "\n", 57 | "# BERT Parameters\n", 58 | "maxlen = 30 # 粗暴 同一个batch里 所有句子长度 一样， 这里把所有batch所有长度粗暴统一\n", 59 | "batch_size = 6\n", 60 | "max_pred = 5 # max tokens of prediction 上限，做mask 不是15%\n", 61 | "n_layers = 6\n", 62 | "n_heads = 12\n", 63 | "d_model = 768 # 三个embedding维度相等\n", 64 | "d_ff = 768 * 4 # 4*d_model, FeedForward dimension\n", 65 | "d_k = d_v = 64 # dimension of K(=Q), V\n", 66 | "n_segments = 2 # 一个样本有多少句话构成\n", 67 | "\n", 68 | "\n", 69 | "# sample IsNext and NotNext to be same in small batch size\n", 70 | "def make_data():\n", 71 | " batch = []\n", 72 | " positive = negative = 0\n", 73 | " while positive != batch_size / 2 or negative != batch_size / 2:\n", 74 | " tokens_a_index, tokens_b_index = randrange(len(sentences)), randrange(\n", 75 | " len(sentences)) # sample random index in sentences\n", 76 | " tokens_a, tokens_b = token_list[tokens_a_index], token_list[tokens_b_index]\n", 77 | " input_ids = [word2idx['[CLS]']] + tokens_a + [word2idx['[SEP]']] + tokens_b + [word2idx['[SEP]']]\n", 78 | " segment_ids = [0] * (1 + len(tokens_a) + 1) + [1] * (len(tokens_b) + 1)\n", 79 | "\n", 80 | " # MASK LM 大于13小于33 5但是句子最长为30 所以说 1-5\n", 81 | " n_pred = min(max_pred, max(1, int(len(input_ids) * 0.15))) # 15 % of tokens in one sentence\n", 82 | " # 去掉标志索引\n", 83 | " cand_maked_pos = [i for i, token in enumerate(input_ids)\n", 84 | " if token != word2idx['[CLS]'] and token != word2idx['[SEP]']] # candidate masked position\n", 85 | " shuffle(cand_maked_pos)\n", 86 | " masked_tokens, masked_pos = [], []\n", 87 | " # 拿出前n_pred个要掩盖的词80%进行掩盖10%进行替换\n", 88 | " for pos in cand_maked_pos[:n_pred]:\n", 89 | " masked_pos.append(pos)\n", 90 | " masked_tokens.append(input_ids[pos])\n", 91 | " if random() < 0.8: # 80%\n", 92 | " input_ids[pos] = word2idx['[MASK]'] # make mask\n", 93 | " elif random() > 0.9: # 10%\n", 94 | " index = randint(0, vocab_size - 1) # random index in vocabulary\n", 95 | " while index < 4: # can't involve 'CLS', 'SEP', 'PAD'\n", 96 | " index = randint(0, vocab_size - 1)\n", 97 | " input_ids[pos] = index # replace\n", 98 | "\n", 99 | " # Zero Paddings 补全\n", 100 | " n_pad = maxlen - len(input_ids)\n", 101 | " input_ids.extend([0] * n_pad)\n", 102 | " segment_ids.extend([0] * n_pad)\n", 103 | "\n", 104 | " # Zero Padding (100% - 15%) tokens其他75%为0\n", 105 | " if max_pred > n_pred:\n", 106 | " n_pad = max_pred - n_pred\n", 107 | " masked_tokens.extend([0] * n_pad)\n", 108 | " masked_pos.extend([0] * n_pad)\n", 109 | " # 两个连续就positive\n", 110 | " if tokens_a_index + 1 == tokens_b_index and positive < batch_size / 2:\n", 111 | " batch.append([input_ids, segment_ids, masked_tokens, masked_pos, True]) # IsNext\n", 112 | " positive += 1\n", 113 | " elif tokens_a_index + 1 != tokens_b_index and negative < batch_size / 2:\n", 114 | " batch.append([input_ids, segment_ids, masked_tokens, masked_pos, False]) # NotNext\n", 115 | " negative += 1\n", 116 | " return batch\n", 117 | "\n", 118 | "\n", 119 | "# Proprecessing Finished\n", 120 | "\n", 121 | "batch = make_data()\n", 122 | "input_ids, segment_ids, masked_tokens, masked_pos, isNext = zip(*batch)\n", 123 | "input_ids, segment_ids, masked_tokens, masked_pos, isNext = \\\n", 124 | " torch.LongTensor(input_ids), torch.LongTensor(segment_ids), torch.LongTensor(masked_tokens), \\\n", 125 | " torch.LongTensor(masked_pos), torch.LongTensor(isNext)\n", 126 | "\n", 127 | "\n", 128 | "class MyDataSet(Data.Dataset):\n", 129 | " def __init__(self, input_ids, segment_ids, masked_tokens, masked_pos, isNext):\n", 130 | " self.input_ids = input_ids\n", 131 | " self.segment_ids = segment_ids\n", 132 | " self.masked_tokens = masked_tokens\n", 133 | " self.masked_pos = masked_pos\n", 134 | " self.isNext = isNext\n", 135 | "\n", 136 | " def __len__(self):\n", 137 | " return len(self.input_ids)\n", 138 | "\n", 139 | " def __getitem__(self, idx):\n", 140 | " return self.input_ids[idx], self.segment_ids[idx], self.masked_tokens[idx], self.masked_pos[idx], self.isNext[\n", 141 | " idx]\n", 142 | "\n", 143 | "\n", 144 | "loader = Data.DataLoader(MyDataSet(input_ids, segment_ids, masked_tokens, masked_pos, isNext), batch_size, True)\n", 145 | "\n", 146 | "\n", 147 | "def get_attn_pad_mask(seq_q, seq_k):\n", 148 | " batch_size, seq_len = seq_q.size()\n", 149 | " # eq(zero) is PAD token\n", 150 | " pad_attn_mask = seq_q.data.eq(0).unsqueeze(1) # [batch_size, 1, seq_len]\n", 151 | " return pad_attn_mask.expand(batch_size, seq_len, seq_len) # [batch_size, seq_len, seq_len]\n", 152 | "\n", 153 | "\n", 154 | "def gelu(x):\n", 155 | " \"\"\"\n", 156 | " Implementation of the gelu activation function.\n", 157 | " For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):\n", 158 | " 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))\n", 159 | " Also see https://arxiv.org/abs/1606.08415\n", 160 | " \"\"\"\n", 161 | " return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))\n", 162 | "\n", 163 | "\n", 164 | "class Embedding(nn.Module):\n", 165 | " def __init__(self):\n", 166 | " super(Embedding, self).__init__()\n", 167 | " self.tok_embed = nn.Embedding(vocab_size, d_model) # token embedding\n", 168 | " self.pos_embed = nn.Embedding(maxlen, d_model) # position embedding\n", 169 | " self.seg_embed = nn.Embedding(n_segments, d_model) # segment(token type) embedding\n", 170 | " self.norm = nn.LayerNorm(d_model)\n", 171 | "\n", 172 | " def forward(self, x, seg):\n", 173 | " seq_len = x.size(1)\n", 174 | " pos = torch.arange(seq_len, dtype=torch.long)\n", 175 | "\n", 176 | " pos = pos.unsqueeze(0).expand_as(x) # [seq_len] -> [batch_size, seq_len]\n", 177 | "\n", 178 | " embedding = self.tok_embed(x) + self.pos_embed(pos.cuda()) + self.seg_embed(seg)\n", 179 | " return self.norm(embedding)\n", 180 | "\n", 181 | "\n", 182 | "class ScaledDotProductAttention(nn.Module):\n", 183 | " def __init__(self):\n", 184 | " super(ScaledDotProductAttention, self).__init__()\n", 185 | "\n", 186 | " def forward(self, Q, K, V, attn_mask):\n", 187 | " scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size, n_heads, seq_len, seq_len]\n", 188 | " scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.\n", 189 | " attn = nn.Softmax(dim=-1)(scores)\n", 190 | " context = torch.matmul(attn, V)\n", 191 | " return context\n", 192 | "\n", 193 | "\n", 194 | "class MultiHeadAttention(nn.Module):\n", 195 | " def __init__(self):\n", 196 | " super(MultiHeadAttention, self).__init__()\n", 197 | " self.W_Q = nn.Linear(d_model, d_k * n_heads)\n", 198 | " self.W_K = nn.Linear(d_model, d_k * n_heads)\n", 199 | " self.W_V = nn.Linear(d_model, d_v * n_heads)\n", 200 | "\n", 201 | " def forward(self, Q, K, V, attn_mask):\n", 202 | " # q: [batch_size, seq_len, d_model], k: [batch_size, seq_len, d_model], v: [batch_size, seq_len, d_model]\n", 203 | " residual, batch_size = Q, Q.size(0)\n", 204 | " # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n", 205 | " q_s = self.W_Q(Q).view(batch_size, -1, n_heads, d_k).transpose(1, 2) # q_s: [batch_size, n_heads, seq_len, d_k]\n", 206 | " k_s = self.W_K(K).view(batch_size, -1, n_heads, d_k).transpose(1, 2) # k_s: [batch_size, n_heads, seq_len, d_k]\n", 207 | " v_s = self.W_V(V).view(batch_size, -1, n_heads, d_v).transpose(1, 2) # v_s: [batch_size, n_heads, seq_len, d_v]\n", 208 | "\n", 209 | " attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1,\n", 210 | " 1) # attn_mask : [batch_size, n_heads, seq_len, seq_len]\n", 211 | "\n", 212 | " # context: [batch_size, n_heads, seq_len, d_v], attn: [batch_size, n_heads, seq_len, seq_len]\n", 213 | " context = ScaledDotProductAttention()(q_s, k_s, v_s, attn_mask)\n", 214 | " context = context.transpose(1, 2).contiguous().view(batch_size, -1,\n", 215 | " n_heads * d_v) # context: [batch_size, seq_len, n_heads, d_v]\n", 216 | " output = nn.Linear(n_heads * d_v, d_model).cuda()(context)\n", 217 | " return nn.LayerNorm(d_model).cuda()(output + residual) # output: [batch_size, seq_len, d_model]\n", 218 | "\n", 219 | "\n", 220 | "class PoswiseFeedForwardNet(nn.Module):\n", 221 | " def __init__(self):\n", 222 | " super(PoswiseFeedForwardNet, self).__init__()\n", 223 | " self.fc1 = nn.Linear(d_model, d_ff)\n", 224 | " self.fc2 = nn.Linear(d_ff, d_model)\n", 225 | "\n", 226 | " def forward(self, x):\n", 227 | " # (batch_size, seq_len, d_model) -> (batch_size, seq_len, d_ff) -> (batch_size, seq_len, d_model)\n", 228 | " return self.fc2(gelu(self.fc1(x)))\n", 229 | "\n", 230 | "\n", 231 | "class EncoderLayer(nn.Module):\n", 232 | " def __init__(self):\n", 233 | " super(EncoderLayer, self).__init__()\n", 234 | " self.enc_self_attn = MultiHeadAttention()\n", 235 | " self.pos_ffn = PoswiseFeedForwardNet()\n", 236 | "\n", 237 | " def forward(self, enc_inputs, enc_self_attn_mask):\n", 238 | " enc_outputs = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs,\n", 239 | " enc_self_attn_mask) # enc_inputs to same Q,K,V\n", 240 | " enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size, seq_len, d_model]\n", 241 | " return enc_outputs\n", 242 | "\n", 243 | "\n", 244 | "class BERT(nn.Module):\n", 245 | " def __init__(self):\n", 246 | " super(BERT, self).__init__()\n", 247 | " self.embedding = Embedding()\n", 248 | " self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n", 249 | " self.fc = nn.Sequential(\n", 250 | " nn.Linear(d_model, d_model),\n", 251 | " nn.Dropout(0.5),\n", 252 | " nn.Tanh(),\n", 253 | " )\n", 254 | " self.classifier = nn.Linear(d_model, 2)\n", 255 | " self.linear = nn.Linear(d_model, d_model)\n", 256 | " self.activ2 = gelu\n", 257 | " # fc2 is shared with embedding layer\n", 258 | " embed_weight = self.embedding.tok_embed.weight\n", 259 | " self.fc2 = nn.Linear(d_model, vocab_size, bias=False)\n", 260 | " self.fc2.weight = embed_weight\n", 261 | "\n", 262 | " def forward(self, input_ids, segment_ids, masked_pos):\n", 263 | " output = self.embedding(input_ids, segment_ids) # [bach_size, seq_len, d_model]\n", 264 | " enc_self_attn_mask = get_attn_pad_mask(input_ids, input_ids) # [batch_size, maxlen, maxlen]\n", 265 | " for layer in self.layers:\n", 266 | " # output: [batch_size, max_len, d_model]\n", 267 | " output = layer(output, enc_self_attn_mask)\n", 268 | " # it will be decided by first token(CLS)\n", 269 | " h_pooled = self.fc.cuda()(output[:, 0]) # [batch_size, d_model]\n", 270 | " logits_clsf = self.classifier.cuda()(h_pooled) # [batch_size, 2] predict isNext\n", 271 | "\n", 272 | " masked_pos = masked_pos[:, :, None].expand(-1, -1, d_model) # [batch_size, maxlen, d_model]\n", 273 | " h_masked = torch.gather(output, 1, masked_pos) # masking position [batch_size, maxlen, d_model]\n", 274 | " h_masked = self.activ2(self.linear.cuda()(h_masked)) # [batch_size, maxlen, d_model]\n", 275 | " logits_lm = self.fc2(h_masked) # [batch_size, maxlen, vocab_size]\n", 276 | " return logits_lm, logits_clsf\n", 277 | "\n", 278 | "\n", 279 | "model = BERT()\n", 280 | "model.eval()\n", 281 | "for k,s in model._modules.items():\n", 282 | " s.eval()\n", 283 | " if k==\"embedding\":\n", 284 | " for k1,s1 in s._modules.items():\n", 285 | " s1.cuda()\n", 286 | " elif k==\"layers\":\n", 287 | " for k2,s2 in s._modules.items():\n", 288 | " s2.cuda()\n", 289 | " for k3,s3 in s2._modules.items():\n", 290 | " s3.cuda()\n", 291 | "\n", 292 | "\n", 293 | "criterion = nn.CrossEntropyLoss()\n", 294 | "optimizer = optim.Adadelta(model.parameters(), lr=0.001)\n", 295 | "\n", 296 | "for epoch in range(50):\n", 297 | " for input_ids, segment_ids, masked_tokens, masked_pos, isNext in loader:\n", 298 | " logits_lm, logits_clsf = model(input_ids.cuda(), segment_ids.cuda(), masked_pos.cuda())\n", 299 | " loss_lm = criterion(logits_lm.view(-1, vocab_size), masked_tokens.view(-1).cuda()) # for masked LM\n", 300 | " loss_lm = (loss_lm.float()).mean()\n", 301 | " loss_clsf = criterion(logits_clsf, isNext.cuda()) # for sentence classification\n", 302 | " loss = loss_lm + loss_clsf\n", 303 | " if (epoch + 1) % 10 == 0:\n", 304 | " print('Epoch:', '%04d' % (epoch + 1), 'loss =', '{:.6f}'.format(loss))\n", 305 | " optimizer.zero_grad()\n", 306 | " loss.backward()\n", 307 | " optimizer.step()\n", 308 | "\n", 309 | "# Predict mask tokens ans isNext\n", 310 | "input_ids, segment_ids, masked_tokens, masked_pos, isNext = batch[1]\n", 311 | "print(text)\n", 312 | "print('================================')\n", 313 | "print([idx2word[w] for w in input_ids if idx2word[w] != '[PAD]'])\n", 314 | "\n", 315 | "logits_lm, logits_clsf = model(torch.LongTensor([input_ids]).cuda(), \\\n", 316 | " torch.LongTensor([segment_ids]).cuda(), torch.LongTensor([masked_pos]).cuda())\n", 317 | "logits_lm = logits_lm.data.max(2)[1][0].data.cpu().numpy()\n", 318 | "print('masked tokens list : ', [pos for pos in masked_tokens if pos != 0])\n", 319 | "print('predict masked tokens list : ', [pos for pos in logits_lm if pos != 0])\n", 320 | "\n", 321 | "logits_clsf = logits_clsf.data.max(1)[1].data.cpu().numpy()[0]\n", 322 | "print('isNext : ', True if isNext else False)\n", 323 | "print('predict isNext : ', True if logits_clsf else False)\n", 324 | "if __name__ == '__main__':\n", 325 | " pass\n" 326 | ] 327 | } 328 | ], 329 | "metadata": { 330 | "kernelspec": { 331 | "display_name": "Python 3", 332 | "language": "python", 333 | "name": "python3" 334 | }, 335 | "language_info": { 336 | "codemirror_mode": { 337 | "name": "ipython", 338 | "version": 3 339 | }, 340 | "file_extension": ".py", 341 | "mimetype": "text/x-python", 342 | "name": "python", 343 | "nbconvert_exporter": "python", 344 | "pygments_lexer": "ipython3", 345 | "version": "3.7.6" 346 | } 347 | }, 348 | "nbformat": 4, 349 | "nbformat_minor": 4 350 | } 351 | -------------------------------------------------------------------------------- /Bert/.ipynb_checkpoints/Bert1-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /Bert/Bert1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import re\n", 10 | "import math\n", 11 | "import torch\n", 12 | "import numpy as np\n", 13 | "import torch.nn as nn\n", 14 | "import torch.optim as optim\n", 15 | "import torch.utils.data as Data" 16 | ] 17 | }, 18 | { 19 | "cell_type": "code", 20 | "execution_count": 5, 21 | "metadata": {}, 22 | "outputs": [ 23 | { 24 | "data": { 25 | "text/plain": [ 26 | "40" 27 | ] 28 | }, 29 | "execution_count": 5, 30 | "metadata": {}, 31 | "output_type": "execute_result" 32 | } 33 | ], 34 | "source": [ 35 | "text = (\n", 36 | " 'Hello, how are you? I am Romeo.\\n' # R\n", 37 | " 'Hello, Romeo My name is Juliet. Nice to meet you.\\n' # J\n", 38 | " 'Nice meet you too. How are you today?\\n' # R\n", 39 | " 'Great. My baseball team won the competition.\\n' # J\n", 40 | " 'Oh Congratulations, Juliet\\n' # R\n", 41 | " 'Thank you Romeo\\n' # J\n", 42 | " 'Where are you going today?\\n' # R\n", 43 | " 'I am going shopping. What about you?\\n' # J\n", 44 | " 'I am going to visit my grandmother. she is not very well' # R\n", 45 | ")\n", 46 | "\n", 47 | "# 过滤 '.' ',' '?' '!' '\\' '-'\n", 48 | "sentence = re.sub(\"[.,!?\\\\-]\" , '', text.lower()).split('\\n')\n", 49 | "word_list = list(set(\" \".join(sentence).split())) \n", 50 | "word2id = {'[PAD]':0, '[CLS]':1, '[SEP]':2 , '[MASK]':3}\n", 51 | "for i,w in enumerate(word_list):\n", 52 | " word2id[w] = i+4\n", 53 | "id2_word = {i:w for i,w in enumerate(word2id)}\n", 54 | "vocab_size = len(word2id)\n", 55 | "vocab_size" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [] 64 | } 65 | ], 66 | "metadata": { 67 | "kernelspec": { 68 | "display_name": "Python 3", 69 | "language": "python", 70 | "name": "python3" 71 | }, 72 | "language_info": { 73 | "codemirror_mode": { 74 | "name": "ipython", 75 | "version": 3 76 | }, 77 | "file_extension": ".py", 78 | "mimetype": "text/x-python", 79 | "name": "python", 80 | "nbconvert_exporter": "python", 81 | "pygments_lexer": "ipython3", 82 | "version": "3.7.6" 83 | } 84 | }, 85 | "nbformat": 4, 86 | "nbformat_minor": 4 87 | } 88 | -------------------------------------------------------------------------------- /NLP/分词/综合类中文词库.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Coding-Zuo/DL-paper/3015361b2ed710b4b49811ea0a0bb535426d9b09/NLP/分词/综合类中文词库.xlsx -------------------------------------------------------------------------------- /NLP/问答系统/.ipynb_checkpoints/learn_nltk_python-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /NeuralFrameWork/.ipynb_checkpoints/2-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /NeuralFrameWork/.ipynb_checkpoints/3-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /NeuralFrameWork/.ipynb_checkpoints/All-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /NeuralFrameWork/All.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from sklearn.datasets import load_boston\n", 10 | "import pandas as pd\n", 11 | "import numpy as np\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "import random" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 2, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "def random_linear(x):\n", 23 | " k, b = random.random(),random.random()\n", 24 | " return k*x+b\n", 25 | "\n", 26 | "def sigmoid(x):\n", 27 | " return 1/(1+np.exp(-x))" 28 | ] 29 | }, 30 | { 31 | "cell_type": "markdown", 32 | "metadata": {}, 33 | "source": [ 34 | "## K-Neighbor-Nearst" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": null, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [] 50 | } 51 | ], 52 | "metadata": { 53 | "kernelspec": { 54 | "display_name": "Python 3", 55 | "language": "python", 56 | "name": "python3" 57 | }, 58 | "language_info": { 59 | "codemirror_mode": { 60 | "name": "ipython", 61 | "version": 3 62 | }, 63 | "file_extension": ".py", 64 | "mimetype": "text/x-python", 65 | "name": "python", 66 | "nbconvert_exporter": "python", 67 | "pygments_lexer": "ipython3", 68 | "version": "3.7.6" 69 | } 70 | }, 71 | "nbformat": 4, 72 | "nbformat_minor": 4 73 | } 74 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DL-paper 2 | 花图片数据集 3 | 数据集下载：https://pan.baidu.com/s/1iQM6MWXz7bG27-uFWGlt8w 提取码:dw2d -------------------------------------------------------------------------------- /TF_AlexNet/.ipynb_checkpoints/AlexNetModel-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_AlexNet/.ipynb_checkpoints/train_AlexNet-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_AlexNet/.ipynb_checkpoints/train_AlexNet_GPU-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_AlexNet/AlexNetModel.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from tensorflow.keras import layers,models,Model,Sequential\n", 10 | "import tensorflow as tf" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "def AlexNet_v1(im_height=224,im_width=224,class_num=1000):\n", 20 | " # tensorflow中的tensor通道排序是 NHWC\n", 21 | " input_image = layers.Input(shape=(im_height,im_width,3),dtype='float32') # output(None,224,224,3)\n", 22 | " x = layers.ZeroPadding2D(((1,2),(1,2)))(input_image) # 输入通道的上方补一行0 下方补两行0 左边补一行0 右边补两行0 # output(None,227,227,3)\n", 23 | " x = layers.Conv2D(48,kernel_size=11,strides=4,activation='relu')(x) # output(None,55,55,48)\n", 24 | " x = layers.MaxPool2D(pool_size=3,strides=2)(x) # output(None,27,27,48)\n", 25 | " \n", 26 | " x = layers.Conv2D(128,kernel_size=5,padding='same',activation='relu')(x) # output(None,27,27,128)\n", 27 | " x = layers.MaxPool2D(pool_size=3,strides=2)(x) # output(None,13,13,192)\n", 28 | "\n", 29 | " x = layers.Conv2D(192,kernel_size=3,padding='same',activation='relu')(x) # output(None,13,13,192)\n", 30 | " x = layers.Conv2D(192,kernel_size=3,padding='same',activation='relu')(x) # output(None,13,13,192)\n", 31 | " x = layers.Conv2D(128,kernel_size=3,padding='same',activation='relu')(x) # output(None,13,13,128)\n", 32 | " x = layers.MaxPool2D(pool_size=3,strides=2)(x) # output(None,6,6,128)\n", 33 | " \n", 34 | " x = layers.Flatten()(x) # output(None,6*6*128)\n", 35 | " x = layers.Dropout(0.2)(x)\n", 36 | " x = layers.Dense(2048,activation='relu')(x) # output(None,2048)\n", 37 | " x = layers.Dropout(0.2)(x)\n", 38 | " x = layers.Dense(2048,activation='relu')(x) # output(None,2048)\n", 39 | " x = layers.Dense(class_num)(x)\n", 40 | " \n", 41 | " predict = layers.Softmax()(x)\n", 42 | " \n", 43 | " model = models.Model(inputs=input_image,outputs=predict)\n", 44 | " return model" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 3, 50 | "metadata": {}, 51 | "outputs": [], 52 | "source": [ 53 | "class AlexNet_v2(Model):\n", 54 | " def __init__(self,class_num=1000):\n", 55 | " super(AlexNet_v2,self).__init__()\n", 56 | " self.features = Sequential([\n", 57 | " layers.ZeroPadding2D(((1,2),(1,2))),\n", 58 | " layers.Conv2D(48,kernel_size=11,strides=4,activation='relu'),\n", 59 | " layers.MaxPool2D(pool_size=3,strides=2),\n", 60 | " layers.Conv2D(128,kernel_size=5,padding='same',activation='relu'),\n", 61 | " layers.MaxPool2D(pool_size=3,strides=2),\n", 62 | " layers.Conv2D(192,kernel_size=3,padding='same',activation='relu'),\n", 63 | " layers.Conv2D(192,kernel_size=3,padding='same',activation='relu'),\n", 64 | " layers.Conv2D(128,kernel_size=3,padding='same',activation='relu'),\n", 65 | " layers.Conv2D(pool_size=3,strides=2)\n", 66 | " ])\n", 67 | " self.flatten = layers.Flatten()\n", 68 | " self.classifier = Sequential([\n", 69 | " layers.Dropout(0.2),\n", 70 | " layers.Dense(1024,activation='relu'),\n", 71 | " layers.Dropout(0.2),\n", 72 | " layers.Dense(128,activation='relu'),\n", 73 | " layers.Dense(class_num),\n", 74 | " layers.Softmax()\n", 75 | " ])\n", 76 | " \n", 77 | " def call(self,inputs,**kwargs):\n", 78 | " x = self.features(inputs)\n", 79 | " x = self.flatten(x)\n", 80 | " x = self.classifier(x)\n", 81 | " return x" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 4, 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "def AlexNet_v1_2gpu(im_height=224,im_width=224,class_num=1000):\n", 91 | " # tensorflow中的tensor通道排序是 NHWC\n", 92 | " input_image = layers.Input(shape=(im_height,im_width,3),dtype='float32') # output(None,224,224,3)\n", 93 | " x = layers.ZeroPadding2D(((1,2),(1,2)))(input_image) # 输入通道的上方补一行0 下方补两行0 左边补一行0 右边补两行0 # output(None,227,227,3)\n", 94 | " x = layers.Conv2D(96,kernel_size=11,strides=4,activation='relu')(x) # output(None,55,55,48)\n", 95 | " x = layers.MaxPool2D(pool_size=3,strides=2)(x) # output(None,27,27,48)\n", 96 | "\n", 97 | " x = layers.Conv2D(256,kernel_size=5,padding='same',activation='relu')(x) # output(None,27,27,128)\n", 98 | " x = layers.MaxPool2D(pool_size=3,strides=2)(x) # output(None,13,13,192)\n", 99 | "\n", 100 | " x = layers.Conv2D(384,kernel_size=3,padding='same',activation='relu')(x) # output(None,13,13,192)\n", 101 | " x = layers.Conv2D(384,kernel_size=3,padding='same',activation='relu')(x) # output(None,13,13,192)\n", 102 | " x = layers.Conv2D(256,kernel_size=3,padding='same',activation='relu')(x) # output(None,13,13,128)\n", 103 | " x = layers.MaxPool2D(pool_size=3,strides=2)(x) # output(None,6,6,128)\n", 104 | "\n", 105 | " x = layers.Flatten()(x) # output(None,6*6*128)\n", 106 | " x = layers.Dropout(0.2)(x)\n", 107 | " x = layers.Dense(4096,activation='relu')(x) # output(None,2048)\n", 108 | " x = layers.Dropout(0.2)(x)\n", 109 | " x = layers.Dense(4096,activation='relu')(x) # output(None,2048)\n", 110 | " x = layers.Dense(class_num)(x)\n", 111 | " \n", 112 | " predict = layers.Softmax()(x)\n", 113 | " \n", 114 | " model = models.Model(inputs=input_image,outputs=predict)\n", 115 | " return model" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": null, 128 | "metadata": {}, 129 | "outputs": [], 130 | "source": [] 131 | } 132 | ], 133 | "metadata": { 134 | "kernelspec": { 135 | "display_name": "Python 3", 136 | "language": "python", 137 | "name": "python3" 138 | }, 139 | "language_info": { 140 | "codemirror_mode": { 141 | "name": "ipython", 142 | "version": 3 143 | }, 144 | "file_extension": ".py", 145 | "mimetype": "text/x-python", 146 | "name": "python", 147 | "nbconvert_exporter": "python", 148 | "pygments_lexer": "ipython3", 149 | "version": "3.7.6" 150 | } 151 | }, 152 | "nbformat": 4, 153 | "nbformat_minor": 4 154 | } 155 | -------------------------------------------------------------------------------- /TF_AlexNet/class_indices.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": "daisy", 3 | "1": "dandelion", 4 | "2": "roses", 5 | "3": "sunflowers", 6 | "4": "tulips" 7 | } -------------------------------------------------------------------------------- /TF_AlexNet/train_AlexNet_GPU.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 5, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "importing Jupyter notebook from AlexNetModel.ipynb\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "import import_ipynb\n", 18 | "from AlexNetModel import AlexNet_v1,AlexNet_v2,AlexNet_v1_2gpu\n", 19 | "import tensorflow as tf\n", 20 | "import os\n", 21 | "import time\n", 22 | "import glob\n", 23 | "import random" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 6, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "os.environ['CUDA_DEVICE_ORDER'] = \"PCI_BUS_ID\" # os.environ[“CUDA_DEVICE_ORDER”] = “PCI_BUS_ID” # 按照PCI_BUS_ID顺序从0开始排列GPU设备\n", 33 | "os.environ['CUDA_VISIBLE_DEVICES'] = '0,1' # 设置当前使用的GPU设备仅为0号设备 设备名称为'/gpu:0'" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 7, 39 | "metadata": {}, 40 | "outputs": [ 41 | { 42 | "data": { 43 | "text/plain": [ 44 | "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'),\n", 45 | " PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU')]" 46 | ] 47 | }, 48 | "execution_count": 7, 49 | "metadata": {}, 50 | "output_type": "execute_result" 51 | } 52 | ], 53 | "source": [ 54 | "gpus = tf.config.experimental.list_physical_devices('GPU')\n", 55 | "logical_gpus = tf.config.experimental.list_physical_devices('GPU')\n", 56 | "if gpus:\n", 57 | " try:\n", 58 | " for gpu in gpus:\n", 59 | " tf.config.experimental.set_memory_growth(gpu,True)\n", 60 | " except RuntimeError as e:\n", 61 | " print(e)\n", 62 | " exit(-1)\n", 63 | "logical_gpus" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 8, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "data_root = os.path.abspath(os.path.join(os.getcwd(),\"../../datasets\"))\n", 73 | "image_path = data_root+\"/flower_data/\"\n", 74 | "train_dir = image_path+'train'\n", 75 | "validation_dir = image_path+'val'" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 9, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "im_height = 224\n", 85 | "im_width = 224\n", 86 | "batch_size = 32\n", 87 | "epochs = 10" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 10, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "data": { 97 | "text/plain": [ 98 | "{0: 'dandelion', 1: 'daisy', 2: 'roses', 3: 'tulips', 4: 'sunflowers'}" 99 | ] 100 | }, 101 | "execution_count": 10, 102 | "metadata": {}, 103 | "output_type": "execute_result" 104 | } 105 | ], 106 | "source": [ 107 | "# class dict\n", 108 | "data_class = [cla for cla in os.listdir(train_dir) if '.txt' not in cla]\n", 109 | "class_num = len(data_class)\n", 110 | "class_dict = dict((value,index) for index,value in enumerate(data_class))\n", 111 | "inverse_dict = dict((value,key) for key,value in class_dict.items())\n", 112 | "inverse_dict" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "execution_count": 11, 118 | "metadata": {}, 119 | "outputs": [], 120 | "source": [ 121 | "# load train images list\n", 122 | "train_image_list = glob.glob(train_dir+\"/*/*.jpg\")\n", 123 | "random.shuffle(train_image_list)\n", 124 | "train_num = len(train_image_list)\n", 125 | "train_label_list = [class_dict[path.split(os.path.sep)[-2]] for path in train_image_list]" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 12, 131 | "metadata": {}, 132 | "outputs": [], 133 | "source": [ 134 | "# load validation images list\n", 135 | "val_image_list = glob.glob(validation_dir+\"/*/*.jpg\")\n", 136 | "random.shuffle(val_image_list)\n", 137 | "val_num = len(val_image_list)\n", 138 | "val_label_list = [class_dict[path.split(os.path.sep)[-2]] for path in val_image_list]" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 13, 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "def process_path(img_path,label):\n", 148 | " label = tf.one_hot(label,depth=class_num)\n", 149 | " image = tf.io.read_file(img_path)\n", 150 | " image = tf.image.decode_jpeg(image)\n", 151 | " image = tf.image.convert_image_dtype(image,tf.float32)\n", 152 | " image = tf.image.resize(image,[im_height,im_width])\n", 153 | " return image,label" 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "execution_count": 14, 159 | "metadata": {}, 160 | "outputs": [ 161 | { 162 | "data": { 163 | "text/plain": [ 164 | "-1" 165 | ] 166 | }, 167 | "execution_count": 14, 168 | "metadata": {}, 169 | "output_type": "execute_result" 170 | } 171 | ], 172 | "source": [ 173 | "AUTOTUNE = tf.data.experimental.AUTOTUNE\n", 174 | "AUTOTUNE" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": null, 180 | "metadata": {}, 181 | "outputs": [], 182 | "source": [ 183 | "strategy = tf.distribute.MirroredStrategy() \n", 184 | "# batch_size_per_replica = 32\n", 185 | "# # Global batch size\n", 186 | "# GLOBAL_BATCH_SIZE = batch_size_per_replica * strategy.num_replicas_in_sync\n", 187 | "# # Buffer size for data loader\n", 188 | "# BUFFER_SIZE = batch_size_per_replica * strategy.num_replicas_in_sync * 16" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 15, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "# load train dataset\n", 198 | "train_dataset = tf.data.Dataset.from_tensor_slices((train_image_list,train_label_list))\n", 199 | "\"\"\"\n", 200 | "通过‘’tf.data.Dataset.prefetch‘转换，tf.data’ API提供了一个软件流水线操作机制，可以用来解耦数据产生的时间和数据消耗的时间。\n", 201 | "特别地，转换使用一个后台线程和一个内部缓冲区，以便在请求输入数据集的元素之前预取它们。\n", 202 | "预取元素的数量应该等于(或者可能大于)单个训练步骤所消耗的批数。您可以手动调整这个值，\n", 203 | "或者将其设置为tf.data.experimental.AUTOTUNE,它将提示tf.data runtime在运行时动态地调整值。\n", 204 | "\"\"\"\n", 205 | "train_dataset = train_dataset.shuffle(buffer_size=train_num).map(process_path,num_parallel_calls=AUTOTUNE)\\\n", 206 | " .repeat().batch(batch_size).prefetch(AUTOTUNE)" 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": 16, 212 | "metadata": {}, 213 | "outputs": [], 214 | "source": [ 215 | "val_dataset = tf.data.Dataset.from_tensor_slices((val_image_list,val_label_list))\n", 216 | "val_dataset = val_dataset.map(process_path,num_parallel_calls=AUTOTUNE).repeat().batch(batch_size)" 217 | ] 218 | }, 219 | { 220 | "cell_type": "code", 221 | "execution_count": 17, 222 | "metadata": {}, 223 | "outputs": [ 224 | { 225 | "name": "stdout", 226 | "output_type": "stream", 227 | "text": [ 228 | "Model: \"model\"\n", 229 | "_________________________________________________________________\n", 230 | "Layer (type) Output Shape Param # \n", 231 | "=================================================================\n", 232 | "input_1 (InputLayer) [(None, 224, 224, 3)] 0 \n", 233 | "_________________________________________________________________\n", 234 | "zero_padding2d (ZeroPadding2 (None, 227, 227, 3) 0 \n", 235 | "_________________________________________________________________\n", 236 | "conv2d (Conv2D) (None, 55, 55, 96) 34944 \n", 237 | "_________________________________________________________________\n", 238 | "max_pooling2d (MaxPooling2D) (None, 27, 27, 96) 0 \n", 239 | "_________________________________________________________________\n", 240 | "conv2d_1 (Conv2D) (None, 27, 27, 256) 614656 \n", 241 | "_________________________________________________________________\n", 242 | "max_pooling2d_1 (MaxPooling2 (None, 13, 13, 256) 0 \n", 243 | "_________________________________________________________________\n", 244 | "conv2d_2 (Conv2D) (None, 13, 13, 384) 885120 \n", 245 | "_________________________________________________________________\n", 246 | "conv2d_3 (Conv2D) (None, 13, 13, 384) 1327488 \n", 247 | "_________________________________________________________________\n", 248 | "conv2d_4 (Conv2D) (None, 13, 13, 256) 884992 \n", 249 | "_________________________________________________________________\n", 250 | "max_pooling2d_2 (MaxPooling2 (None, 6, 6, 256) 0 \n", 251 | "_________________________________________________________________\n", 252 | "flatten (Flatten) (None, 9216) 0 \n", 253 | "_________________________________________________________________\n", 254 | "dropout (Dropout) (None, 9216) 0 \n", 255 | "_________________________________________________________________\n", 256 | "dense (Dense) (None, 4096) 37752832 \n", 257 | "_________________________________________________________________\n", 258 | "dropout_1 (Dropout) (None, 4096) 0 \n", 259 | "_________________________________________________________________\n", 260 | "dense_1 (Dense) (None, 4096) 16781312 \n", 261 | "_________________________________________________________________\n", 262 | "dense_2 (Dense) (None, 5) 20485 \n", 263 | "_________________________________________________________________\n", 264 | "softmax (Softmax) (None, 5) 0 \n", 265 | "=================================================================\n", 266 | "Total params: 58,301,829\n", 267 | "Trainable params: 58,301,829\n", 268 | "Non-trainable params: 0\n", 269 | "_________________________________________________________________\n", 270 | "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n", 271 | "INFO:tensorflow:Reduce to /job:localhost/replica:0/task:0/device:CPU:0 then broadcast to ('/job:localhost/replica:0/task:0/device:CPU:0',).\n" 272 | ] 273 | } 274 | ], 275 | "source": [ 276 | "\n", 277 | "with strategy.scope(): \n", 278 | " model = AlexNet_v1_2gpu(im_height=im_height,im_width=im_width,class_num=5)\n", 279 | " model.summary() \n", 280 | " model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0005),\n", 281 | " loss=tf.keras.losses.CategoricalCrossentropy(from_logits=False), #因为已经softmax所以false\n", 282 | " metrics=[\"accuracy\"])" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": 19, 288 | "metadata": {}, 289 | "outputs": [ 290 | { 291 | "name": "stdout", 292 | "output_type": "stream", 293 | "text": [ 294 | "Train for 103 steps, validate for 11 steps\n", 295 | "Epoch 1/10\n", 296 | "INFO:tensorflow:batch_all_reduce: 16 all-reduces with algorithm = nccl, num_packs = 1, agg_small_grads_max_bytes = 0 and agg_small_grads_max_group = 10\n", 297 | "103/103 [==============================] - 11s 105ms/step - loss: 0.5762 - accuracy: 0.7743 - val_loss: 0.8086 - val_accuracy: 0.6960\n", 298 | "Epoch 2/10\n", 299 | "103/103 [==============================] - 8s 77ms/step - loss: 0.5439 - accuracy: 0.7967 - val_loss: 0.9274 - val_accuracy: 0.6790\n", 300 | "Epoch 3/10\n", 301 | "103/103 [==============================] - 8s 77ms/step - loss: 0.4502 - accuracy: 0.8277 - val_loss: 0.9727 - val_accuracy: 0.6676\n", 302 | "Epoch 4/10\n", 303 | "103/103 [==============================] - 8s 77ms/step - loss: 0.3838 - accuracy: 0.8474 - val_loss: 0.8582 - val_accuracy: 0.6989\n", 304 | "Epoch 5/10\n", 305 | "103/103 [==============================] - 8s 76ms/step - loss: 0.3826 - accuracy: 0.8553 - val_loss: 0.8793 - val_accuracy: 0.7386\n", 306 | "Epoch 6/10\n", 307 | "103/103 [==============================] - 8s 76ms/step - loss: 0.2671 - accuracy: 0.8929 - val_loss: 1.3767 - val_accuracy: 0.6818\n", 308 | "Epoch 7/10\n", 309 | "103/103 [==============================] - 8s 76ms/step - loss: 0.2501 - accuracy: 0.9053 - val_loss: 1.3453 - val_accuracy: 0.6903\n", 310 | "Epoch 8/10\n", 311 | "103/103 [==============================] - 8s 76ms/step - loss: 0.2611 - accuracy: 0.9002 - val_loss: 1.3407 - val_accuracy: 0.6818\n", 312 | "Epoch 9/10\n", 313 | "103/103 [==============================] - 8s 78ms/step - loss: 0.2049 - accuracy: 0.9196 - val_loss: 1.2213 - val_accuracy: 0.7131\n", 314 | "Epoch 10/10\n", 315 | "103/103 [==============================] - 8s 75ms/step - loss: 0.2036 - accuracy: 0.9296 - val_loss: 1.2214 - val_accuracy: 0.6790\n" 316 | ] 317 | } 318 | ], 319 | "source": [ 320 | "history = model.fit(x=train_dataset,\n", 321 | " steps_per_epoch=train_num // batch_size,\n", 322 | " epochs=epochs,\n", 323 | " validation_data=val_dataset,\n", 324 | " validation_steps=val_num // batch_size)" 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": null, 330 | "metadata": {}, 331 | "outputs": [], 332 | "source": [] 333 | } 334 | ], 335 | "metadata": { 336 | "kernelspec": { 337 | "display_name": "Python 3", 338 | "language": "python", 339 | "name": "python3" 340 | }, 341 | "language_info": { 342 | "codemirror_mode": { 343 | "name": "ipython", 344 | "version": 3 345 | }, 346 | "file_extension": ".py", 347 | "mimetype": "text/x-python", 348 | "name": "python", 349 | "nbconvert_exporter": "python", 350 | "pygments_lexer": "ipython3", 351 | "version": "3.7.6" 352 | } 353 | }, 354 | "nbformat": 4, 355 | "nbformat_minor": 4 356 | } 357 | -------------------------------------------------------------------------------- /TF_GNN/GCN/.ipynb_checkpoints/Cora_Dataset-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## CORA数据集由机器学习论文组成。这些文件可分为以下七个类别之一：\n", 8 | " 基于案例的\n", 9 | " 遗传算法\n", 10 | " 神经网络\n", 11 | " 概率方法\n", 12 | " 强化学习\n", 13 | " 规则_学习\n", 14 | " 理论\n", 15 | " \n", 16 | " 这些论文的选择方式是，在最终语料库中，每篇论文至少引用一篇其他论文或被至少一篇其他论文引用。\n", 17 | " 整个语料库共有2708篇论文。\n", 18 | " 在对停顿词进行词干处理和删除之后，我们剩下1433个独特单词的词汇表。文档频率低于10的所有单词都被删除。\n", 19 | " \n", 20 | " 该目录包含两个文件：\n", 21 | " .content文件包含以下格式的论文说明：+\n", 22 | " \n", 23 | " 每行的第一个条目包含论文的唯一字符串ID，后跟指示词汇表中的每个单词在论文中是否存在(由1表示)或不存在(由0表示)的二进制值。\n", 24 | " 最后，该行的最后一个条目包含纸张的类别标签。\n", 25 | " 每行的数据格式如下: + 。paper id是论文的唯一标识；word_attributes是是一个维度为1433的词向量，\n", 26 | " 词向量的每个元素对应一个词，0表示该元素对应的词不在Paper中，1表示该元素对应的词在Paper中。class_label是论文的类别，\n", 27 | " 每篇Paper被映射到如下7个分类之一: Case_Based、Genetic_Algorithms、Neural_Networks、Probabilistic_Methods、Reinforcement_Learning、Rule_Learning、Theory。\n", 28 | " \n", 29 | " .cites文件包含语料库的引文图表。每行以以下格式描述一个链接：<被引用论文ID><引用论文ID>\n", 30 | " \n", 31 | " 每行包含两个纸质ID。第一个条目是被引用论文的ID，第二个ID代表包含引文的论文。链接的方向是从右到左。如果一行由“Pap1 Pap2”表示，则链接为“Pap2->Pap1”。" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "import scipy.sparse as sp\n", 41 | "import numpy as np" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 3, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "class CoraData():\n", 51 | " def __init__(self, data_root='/home/zuoyuhui/datasets/cora/'):\n", 52 | " self._data_root = data_root\n", 53 | " self._data = self.process_data()\n", 54 | " \n", 55 | " def load_data(self,dataset='cora'):\n", 56 | " print('Loading {} dataset...'.format(dataset))\n", 57 | " idx_features_labels = np.genfromtxt(\"{}{}.cites\".format(self._data_root,dataset),dtype=np.dtype(str))\n", 58 | " edges = np.genfromtxt(\"{}{}.cites\".format(self._data_root,dataset),dtype=np.int32)\n", 59 | " return idx_features_labels, edges\n", 60 | " \n", 61 | " def process_data(self):\n", 62 | " print(\"Process data...\")\n", 63 | " \n", 64 | " idx_features_labels, edges = self.load_data()\n", 65 | " \n", 66 | " features = idx_features_labels[:,1:-1].astype(np.float32)\n", 67 | " features = self.normalize_feature(features)\n", 68 | " \n", 69 | " y = idx_features_labels[:,-1]\n", 70 | " labels = self.encode_onehot(y)\n", 71 | " \n", 72 | " idx = np.array(idx_features_labels[:,0],dtype=np.int32)\n", 73 | " idx_map = {j:i for i,j in enumerate(idx)}\n", 74 | " edge_indexs = np.array(list(map(idx_map.get,edges.flatten())),dtype=np.int32)\n", 75 | " edge_indexs = edge_indexs.reshape(edges.shape)\n", 76 | " \n", 77 | " edge_index_len =len(edge_indexs)\n", 78 | " for i in range(edge_index_len):\n", 79 | " edge_indexs = np.concatenate((edge_indexs,[[edge_indexs[i][1],edge_indexs[i][0]]]))\n", 80 | " \n", 81 | " adjacency = sp.coo_matrix((np.ones(len(edge_indexs)),\n", 82 | " (edge_indexs[:,0],edge_indexs[:,1])),\n", 83 | " shape=(features.shape[0],features.shape[0]),dtype='float32')\n", 84 | " adjacency = self.normalize_adj(adjacency)\n", 85 | " \n", 86 | " train_index = np.arange(150)\n", 87 | " val_index = np.arange(150,500)\n", 88 | " test_index = np.arange(500,2708)\n", 89 | " \n", 90 | " train_mask = np.zeros(edge_indexs.shape[0],dtype=np.bool)\n", 91 | " val_mask = np.zeros(edge_indexs.shape[0],dtype=np.bool)\n", 92 | " test_mask = np.zeros(edge_indexs.shape[0],dtype=np.bool)\n", 93 | " train_mask[train_index]=True\n", 94 | " val_mask[val_index]=True\n", 95 | " test_mask[test_index]=True\n", 96 | " \n", 97 | " print('Dataset has {} nodes, {} edges, {} features.'.format(features.shape[0], adjacency.shape[0], features.shape[1]))\n", 98 | "\n", 99 | " return features, labels, adjacency, train_mask, val_mask, test_mask\n", 100 | " \n", 101 | " \n", 102 | " def encode_onehot(self,label):\n", 103 | " classes = set(labels)\n", 104 | " class_dict = {c: np.identity(len(classes))[i,:] for i,c in enumerate(classes)}\n", 105 | " labels_onehot = np.array(list(map(classes_dict.get, labels)), dtype=np.int32)\n", 106 | " return labels_onehot\n", 107 | " \n", 108 | " def normalize_feature(self,features):\n", 109 | " noraml_features = features/features.sum(1).reshape(-1,1)\n", 110 | " return noraml_features\n", 111 | " \n", 112 | " def normalize_adj(self,adjacency):\n", 113 | " \"\"\"计算 L=D^-0.5 * (A+I) * D^-0.5\"\"\"\n", 114 | " adjacency += sp.eye(adjacency.shape[0]) #增加自连接\n", 115 | " degree = np.array(adjacency.sum(1))\n", 116 | " d_hat = sp.diags(np.power(degree,-0.5).flatten())\n", 117 | " return d_hat.dot(adjacency).dot(d_hat).tocsr().todense()\n", 118 | " \n", 119 | " def data(self):\n", 120 | " \"\"\"返回Data数据对象，包括features, labes, adjacency, train_mask, val_mask, test_mask\"\"\"\n", 121 | " return self._data" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [] 130 | } 131 | ], 132 | "metadata": { 133 | "kernelspec": { 134 | "display_name": "Python 3", 135 | "language": "python", 136 | "name": "python3" 137 | }, 138 | "language_info": { 139 | "codemirror_mode": { 140 | "name": "ipython", 141 | "version": 3 142 | }, 143 | "file_extension": ".py", 144 | "mimetype": "text/x-python", 145 | "name": "python", 146 | "nbconvert_exporter": "python", 147 | "pygments_lexer": "ipython3", 148 | "version": "3.7.6" 149 | } 150 | }, 151 | "nbformat": 4, 152 | "nbformat_minor": 4 153 | } 154 | -------------------------------------------------------------------------------- /TF_GNN/GCN/.ipynb_checkpoints/Cora_EDA-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_GNN/GCN/Cora_Dataset.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## CORA数据集由机器学习论文组成。这些文件可分为以下七个类别之一：\n", 8 | " 基于案例的\n", 9 | " 遗传算法\n", 10 | " 神经网络\n", 11 | " 概率方法\n", 12 | " 强化学习\n", 13 | " 规则_学习\n", 14 | " 理论\n", 15 | " \n", 16 | " 这些论文的选择方式是，在最终语料库中，每篇论文至少引用一篇其他论文或被至少一篇其他论文引用。\n", 17 | " 整个语料库共有2708篇论文。\n", 18 | " 在对停顿词进行词干处理和删除之后，我们剩下1433个独特单词的词汇表。文档频率低于10的所有单词都被删除。\n", 19 | " \n", 20 | " 该目录包含两个文件：\n", 21 | " .content文件包含以下格式的论文说明：+\n", 22 | " \n", 23 | " 每行的第一个条目包含论文的唯一字符串ID，后跟指示词汇表中的每个单词在论文中是否存在(由1表示)或不存在(由0表示)的二进制值。\n", 24 | " 最后，该行的最后一个条目包含纸张的类别标签。\n", 25 | " 每行的数据格式如下: + 。paper id是论文的唯一标识；word_attributes是是一个维度为1433的词向量，\n", 26 | " 词向量的每个元素对应一个词，0表示该元素对应的词不在Paper中，1表示该元素对应的词在Paper中。class_label是论文的类别，\n", 27 | " 每篇Paper被映射到如下7个分类之一: Case_Based、Genetic_Algorithms、Neural_Networks、Probabilistic_Methods、Reinforcement_Learning、Rule_Learning、Theory。\n", 28 | " \n", 29 | " .cites文件包含语料库的引文图表。每行以以下格式描述一个链接：<被引用论文ID><引用论文ID>\n", 30 | " \n", 31 | " 每行包含两个纸质ID。第一个条目是被引用论文的ID，第二个ID代表包含引文的论文。链接的方向是从右到左。如果一行由“Pap1 Pap2”表示，则链接为“Pap2->Pap1”。" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "import scipy.sparse as sp\n", 41 | "import numpy as np" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 3, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [ 50 | "class CoraData():\n", 51 | " def __init__(self, data_root='/home/zuoyuhui/datasets/cora/'):\n", 52 | " self._data_root = data_root\n", 53 | " self._data = self.process_data()\n", 54 | " \n", 55 | " def load_data(self,dataset='cora'):\n", 56 | " print('Loading {} dataset...'.format(dataset))\n", 57 | " idx_features_labels = np.genfromtxt(\"{}{}.cites\".format(self._data_root,dataset),dtype=np.dtype(str))\n", 58 | " edges = np.genfromtxt(\"{}{}.cites\".format(self._data_root,dataset),dtype=np.int32)\n", 59 | " return idx_features_labels, edges\n", 60 | " \n", 61 | " def process_data(self):\n", 62 | " print(\"Process data...\")\n", 63 | " \n", 64 | " idx_features_labels, edges = self.load_data()\n", 65 | " \n", 66 | " features = idx_features_labels[:,1:-1].astype(np.float32)\n", 67 | " features = self.normalize_feature(features)\n", 68 | " \n", 69 | " y = idx_features_labels[:,-1]\n", 70 | " labels = self.encode_onehot(y)\n", 71 | " \n", 72 | " idx = np.array(idx_features_labels[:,0],dtype=np.int32)\n", 73 | " idx_map = {j:i for i,j in enumerate(idx)}\n", 74 | " edge_indexs = np.array(list(map(idx_map.get,edges.flatten())),dtype=np.int32)\n", 75 | " edge_indexs = edge_indexs.reshape(edges.shape)\n", 76 | " \n", 77 | " edge_index_len =len(edge_indexs)\n", 78 | " for i in range(edge_index_len):\n", 79 | " edge_indexs = np.concatenate((edge_indexs,[[edge_indexs[i][1],edge_indexs[i][0]]]))\n", 80 | " \n", 81 | " adjacency = sp.coo_matrix((np.ones(len(edge_indexs)),\n", 82 | " (edge_indexs[:,0],edge_indexs[:,1])),\n", 83 | " shape=(features.shape[0],features.shape[0]),dtype='float32')\n", 84 | " adjacency = self.normalize_adj(adjacency)\n", 85 | " \n", 86 | " train_index = np.arange(150)\n", 87 | " val_index = np.arange(150,500)\n", 88 | " test_index = np.arange(500,2708)\n", 89 | " \n", 90 | " train_mask = np.zeros(edge_indexs.shape[0],dtype=np.bool)\n", 91 | " val_mask = np.zeros(edge_indexs.shape[0],dtype=np.bool)\n", 92 | " test_mask = np.zeros(edge_indexs.shape[0],dtype=np.bool)\n", 93 | " train_mask[train_index]=True\n", 94 | " val_mask[val_index]=True\n", 95 | " test_mask[test_index]=True\n", 96 | " \n", 97 | " print('Dataset has {} nodes, {} edges, {} features.'.format(features.shape[0], adjacency.shape[0], features.shape[1]))\n", 98 | "\n", 99 | " return features, labels, adjacency, train_mask, val_mask, test_mask\n", 100 | " \n", 101 | " \n", 102 | " def encode_onehot(self,label):\n", 103 | " classes = set(labels)\n", 104 | " class_dict = {c: np.identity(len(classes))[i,:] for i,c in enumerate(classes)}\n", 105 | " labels_onehot = np.array(list(map(classes_dict.get, labels)), dtype=np.int32)\n", 106 | " return labels_onehot\n", 107 | " \n", 108 | " def normalize_feature(self,features):\n", 109 | " noraml_features = features/features.sum(1).reshape(-1,1)\n", 110 | " return noraml_features\n", 111 | " \n", 112 | " def normalize_adj(self,adjacency):\n", 113 | " \"\"\"计算 L=D^-0.5 * (A+I) * D^-0.5\"\"\"\n", 114 | " adjacency += sp.eye(adjacency.shape[0]) #增加自连接\n", 115 | " degree = np.array(adjacency.sum(1))\n", 116 | " d_hat = sp.diags(np.power(degree,-0.5).flatten())\n", 117 | " return d_hat.dot(adjacency).dot(d_hat).tocsr().todense()\n", 118 | " \n", 119 | " def data(self):\n", 120 | " \"\"\"返回Data数据对象，包括features, labes, adjacency, train_mask, val_mask, test_mask\"\"\"\n", 121 | " return self._data" 122 | ] 123 | }, 124 | { 125 | "cell_type": "code", 126 | "execution_count": null, 127 | "metadata": {}, 128 | "outputs": [], 129 | "source": [] 130 | } 131 | ], 132 | "metadata": { 133 | "kernelspec": { 134 | "display_name": "Python 3", 135 | "language": "python", 136 | "name": "python3" 137 | }, 138 | "language_info": { 139 | "codemirror_mode": { 140 | "name": "ipython", 141 | "version": 3 142 | }, 143 | "file_extension": ".py", 144 | "mimetype": "text/x-python", 145 | "name": "python", 146 | "nbconvert_exporter": "python", 147 | "pygments_lexer": "ipython3", 148 | "version": "3.7.6" 149 | } 150 | }, 151 | "nbformat": 4, 152 | "nbformat_minor": 4 153 | } 154 | -------------------------------------------------------------------------------- /TF_GoogLeNet/.ipynb_checkpoints/Model-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import tensorflow as tf\n", 10 | "from tensorflow.keras import layers,models,Model,Sequential" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "class Inception(layers.Layer):\n", 20 | " def __init__(self, ch1x1, ch3x3reduce, ch3x3, ch5x5reduce, ch5x5, pool_proj, **kwargs):\n", 21 | " super(Inception, self).__init__(**kwargs)\n", 22 | " self.branch1 = layers.Conv2D(ch1x1, kernel_size=1, activation='relu')\n", 23 | " \n", 24 | " self.branch2 = Sequential([\n", 25 | " layers.Conv2D(ch3x3reduce,kernel_size=1,activation='relu'),\n", 26 | " layers.Conv2D(ch3x3,kernel_size=3,activation='relu')\n", 27 | " ])\n", 28 | " \n", 29 | " self.branch3 = Sequential([\n", 30 | " layers.Conv2D(ch5x5reduce,kernel_size=1,activation='relu'),\n", 31 | " layers.Conv2D(ch5x5,kernel_size=5,padding='same',activation='relu')\n", 32 | " ])\n", 33 | " \n", 34 | " self.branch4 = Sequential([\n", 35 | " layers.MaxPool2D(pool_size=3,strides=1,padding='same'),\n", 36 | " layers.Conv2D(pool_proj,kernel_size=1,activation='relu')\n", 37 | " ])\n", 38 | " \n", 39 | " def call(self,inputs,**kwargs):\n", 40 | " branch1 = self.branch1(inputs)\n", 41 | " branch2 = self.branch2(inputs)\n", 42 | " branch3 = self.branch3(inputs)\n", 43 | " branch4 = self.branch4(inputs)\n", 44 | " outputs = layers.concatenate([branch1,branch2,branch3,branch4])\n", 45 | " return outputs" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 3, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "class InceptionAux(layers.Layer):\n", 55 | " def __init__(self, num_classes, **kwargs):\n", 56 | " super(InceptionAux, self).__init__(**kwargs)\n", 57 | " self.averagePool = layers.AvgPool2D(pool_size=5,strides=3)\n", 58 | " self.conv = layers.Conv2D(128,kernel_size=1,activation='relu')\n", 59 | " \n", 60 | " self.fc1 = layers.Dense(1024,activation='relu')\n", 61 | " self.fc2 = layers.Dense(num_classes)\n", 62 | " self.softmax = layers.Softmax()\n", 63 | " \n", 64 | " def call(self,inputs,**kwargs):\n", 65 | " # aux1:N x 512 x 14 x 14, aux2:N x 528 x 14 x 14\n", 66 | " x = self.averagePool(inputs)\n", 67 | " # aux1:N x 512 x 4 x 4, aux2: N x 528 x 4 x 4\n", 68 | " x = self.conv(x)\n", 69 | " # N x 128 x 4 x 4\n", 70 | " x = layers.Flatten()(x)\n", 71 | " x = layers.Dropout(rate=0.5)(x)\n", 72 | " # N x 2048\n", 73 | " x = self.fc1(x)\n", 74 | " x = layers.Dropout(rate=0.5)(x)\n", 75 | " # N x 1024\n", 76 | " x = self.fc2(x)\n", 77 | " x = self.softmax(x)\n", 78 | " return x" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 6, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "def GoogLeNetV1(im_height=224, im_width=224, class_num=1000, aux_logits=False):\n", 88 | " # same卷积：卷积后的特征图大小跟卷积核大小无关，只跟卷积所用的步长有关，当卷积步长是1时，卷积前后特征图大小保持不变\n", 89 | " # W1 = math.ceil(W / s)\n", 90 | " # H1 = math.ceil(H / s)\n", 91 | " # valid卷积：VALID（根据图片实际大小执行卷积，不对图像边界填充，一般卷积后特征图变小）\n", 92 | " # W1 = math.ceil((W-k+1) / s)\n", 93 | " # H1 = math.ceil((H-k+1) / s)\n", 94 | " input_image = layers.Input(shape=(im_height,im_width,3),dtype='float32')\n", 95 | " #(None,224,224,3)\n", 96 | " x = layers.Conv2D(64,kernel_size=7,strides=2,padding='same',activation='relu',name='conv2d_1')(input)\n", 97 | " #(None,112,112,64)\n", 98 | " x = layers.MaxPool2D(pool_size=2,strides=2,name='maxpool_1')(x)\n", 99 | " #(None,56,56,64)\n", 100 | " x = layers.Conv2D(64, kernel_size=1,activation='relu',name=\"conv2d_2\")(x)\n", 101 | " #(None,56,56,64)\n", 102 | " x = layers.Conv2D(192,kernel_size=3,padding='same',activation='relu',name=\"conv2d_3\")(x)\n", 103 | " #(None,56,56,192)\n", 104 | " x = layers.MaxPool2D(pool_size=3,strides=2,padding='same',name='maxpool_2')(x)\n", 105 | " #(None,28,28,192)\n", 106 | " \n", 107 | " x = Inception(ch1x1=64, ch3x3reduce=96, ch3x3=128, ch5x5reduce=16, ch5x5=32, pool_proj=32, name='inception_3a')(x)\n", 108 | " #(None,28,28,256)\n", 109 | " x = Inception(ch1x1=128,ch3x3reduce=96,ch3x3=128,ch5x5reduce=16,ch5x5=32,pool_proj=64,name='inception_3b')(x)\n", 110 | " \n", 111 | " #(None,28,28,480)\n", 112 | " x = layers.MaxPool2D(pool_size=3, strides=2, padding='same', name='maxpool_3')(x)\n", 113 | " #(None,14,14,480)\n", 114 | " x = Inception(ch1x1=192, ch3x3reduce=96, ch3x3=208, ch5x5reduce=16, ch5x5=48, pool_proj=64, name='inception_4a')(x)\n", 115 | " if aux_logits:\n", 116 | " aux1 = InceptionAux(class_num,name='aux_1')(x)\n", 117 | " \n", 118 | " #(None,14,14,512)\n", 119 | " x = Inception(ch1x1=160, ch3x3reduce=112, ch3x3=224, ch5x5reduce=24, ch5x5=48, pool_proj=64, name='inception_4b')(x)\n", 120 | " # (None, 14, 14, 512)\n", 121 | " x = Inception(128, 128, 256, 24, 64, 64, name=\"inception_4c\")(x)\n", 122 | " # (None, 14, 14, 512)\n", 123 | " x = Inception(112, 144, 288, 32, 64, 64, name=\"inception_4d\")(x)\n", 124 | " if aux_logits:\n", 125 | " aux2 = InceptionAux(class_num, name=\"aux_2\")(x)\n", 126 | " \n", 127 | " # (None, 14, 14, 528)\n", 128 | " x = Inception(256, 160, 320, 32, 128, 128, name=\"inception_4e\")(x)\n", 129 | " # (None, 14, 14, 532)\n", 130 | " x = layers.MaxPool2D(pool_size=3, strides=2, padding=\"SAME\", name=\"maxpool_4\")(x)\n", 131 | "\n", 132 | " # (None, 7, 7, 832)\n", 133 | " x = Inception(256, 160, 320, 32, 128, 128, name=\"inception_5a\")(x)\n", 134 | " # (None, 7, 7, 832)\n", 135 | " x = Inception(384, 192, 384, 48, 128, 128, name=\"inception_5b\")(x)\n", 136 | " # (None, 7, 7, 1024)\n", 137 | " x = layers.AvgPool2D(pool_size=7, strides=1, name=\"avgpool_1\")(x)\n", 138 | " \n", 139 | " # (None, 1, 1, 1024)\n", 140 | " x = layers.Flatten(name=\"output_flatten\")(x)\n", 141 | " # (None, 1024)\n", 142 | " x = layers.Dropout(rate=0.4, name=\"output_dropout\")(x)\n", 143 | " x = layers.Dense(class_num, name=\"output_dense\")(x)\n", 144 | " # (None, class_num)\n", 145 | " aux3 = layers.Softmax(name=\"aux_3\")(x)\n", 146 | "\n", 147 | " if aux_logits:\n", 148 | " model = models.Model(inputs=input_image, outputs=[aux1, aux2, aux3])\n", 149 | " else:\n", 150 | " model = models.Model(inputs=input_image, outputs=aux3)\n", 151 | " return model" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": null, 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [] 167 | } 168 | ], 169 | "metadata": { 170 | "kernelspec": { 171 | "display_name": "Python 3", 172 | "language": "python", 173 | "name": "python3" 174 | }, 175 | "language_info": { 176 | "codemirror_mode": { 177 | "name": "ipython", 178 | "version": 3 179 | }, 180 | "file_extension": ".py", 181 | "mimetype": "text/x-python", 182 | "name": "python", 183 | "nbconvert_exporter": "python", 184 | "pygments_lexer": "ipython3", 185 | "version": "3.7.6" 186 | } 187 | }, 188 | "nbformat": 4, 189 | "nbformat_minor": 4 190 | } 191 | -------------------------------------------------------------------------------- /TF_GoogLeNet/.ipynb_checkpoints/Model_add_bn-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_GoogLeNet/.ipynb_checkpoints/predict-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_GoogLeNet/.ipynb_checkpoints/trainCPU-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "importing Jupyter notebook from Model.ipynb\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", 18 | "import matplotlib.pyplot as plt\n", 19 | "import tensorflow as tf\n", 20 | "import json\n", 21 | "import os\n", 22 | "import import_ipynb\n", 23 | "from Model import GoogLeNet" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 4, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "data_root = os.path.abspath(os.path.join(os.getcwd(),\"../../datasets\"))\n", 33 | "image_path = data_root+\"/flower_data/\"\n", 34 | "train_dir = image_path+'train'\n", 35 | "validation_dir = image_path+'val'" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 5, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "if not os.path.exists('save_weights'):\n", 45 | " os.mkdir('save_weights')\n", 46 | "im_height=224\n", 47 | "im_width =224\n", 48 | "batch_size=32\n", 49 | "epochs = 30" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 6, 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "def pre_function(img):\n", 59 | " img = img/255.\n", 60 | " img = (img-0.5)*2.0\n", 61 | " return img" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 7, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "name": "stdout", 71 | "output_type": "stream", 72 | "text": [ 73 | "Found 3306 images belonging to 5 classes.\n" 74 | ] 75 | } 76 | ], 77 | "source": [ 78 | "train_image_generator = ImageDataGenerator(preprocessing_function=pre_function,\n", 79 | " horizontal_flip=True)\n", 80 | "validation_image_generator = ImageDataGenerator(preprocessing_function=pre_function)\n", 81 | "\n", 82 | "\n", 83 | "train_data_gen = train_image_generator.flow_from_directory(directory=train_dir,\n", 84 | " batch_size=batch_size,\n", 85 | " shuffle=True,\n", 86 | " target_size=(im_height,im_width),\n", 87 | " class_mode='categorical')\n", 88 | "total_train = train_data_gen.n" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 8, 94 | "metadata": {}, 95 | "outputs": [ 96 | { 97 | "data": { 98 | "text/plain": [ 99 | "{'daisy': 0, 'dandelion': 1, 'roses': 2, 'sunflowers': 3, 'tulips': 4}" 100 | ] 101 | }, 102 | "execution_count": 8, 103 | "metadata": {}, 104 | "output_type": "execute_result" 105 | } 106 | ], 107 | "source": [ 108 | "class_indices = train_data_gen.class_indices\n", 109 | "class_indices" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 10, 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "inverse_dict = dict((val,key) for key,val in class_indices.items())" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": 11, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "json_str = json.dumps(inverse_dict,indent=4)\n", 128 | "with open('class_indices.json','w') as json_file:\n", 129 | " json_file.write(json_str)" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 12, 135 | "metadata": {}, 136 | "outputs": [ 137 | { 138 | "name": "stdout", 139 | "output_type": "stream", 140 | "text": [ 141 | "Found 364 images belonging to 5 classes.\n" 142 | ] 143 | } 144 | ], 145 | "source": [ 146 | "val_data_gen = validation_image_generator.flow_from_directory(directory=validation_dir,\n", 147 | " batch_size=batch_size,\n", 148 | " shuffle=False,\n", 149 | " target_size=(im_height,im_width),\n", 150 | " class_mode='categorical')\n", 151 | "total_val = val_data_gen.n" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 14, 157 | "metadata": {}, 158 | "outputs": [ 159 | { 160 | "ename": "AttributeError", 161 | "evalue": "'function' object has no attribute 'shape'", 162 | "output_type": "error", 163 | "traceback": [ 164 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 165 | "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", 166 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGoogLeNet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mim_height\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mim_height\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mim_width\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mim_width\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mclass_num\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maux_logits\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m224\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m224\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# subclass model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 167 | "\u001b[0;32m/home/zuoyuhui/paper/GoogLeNet/Model.ipynb\u001b[0m in \u001b[0;36mGoogLeNet\u001b[0;34m(im_height, im_width, class_num, aux_logits)\u001b[0m\n", 168 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m 885\u001b[0m \u001b[0;31m# Eager execution on data tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 886\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 887\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_build\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 888\u001b[0m \u001b[0mcast_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_inputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 889\u001b[0m with base_layer_utils.autocast_context_manager(\n", 169 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m_maybe_build\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2120\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2121\u001b[0m input_spec.assert_input_compatibility(\n\u001b[0;32m-> 2122\u001b[0;31m self.input_spec, inputs, self.name)\n\u001b[0m\u001b[1;32m 2123\u001b[0m \u001b[0minput_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2124\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0minput_list\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dtype_policy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_dtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 170 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/input_spec.py\u001b[0m in \u001b[0;36massert_input_compatibility\u001b[0;34m(input_spec, inputs, layer_name)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmin_ndim\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 162\u001b[0m spec.max_ndim is not None):\n\u001b[0;32m--> 163\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndims\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 164\u001b[0m raise ValueError('Input ' + str(input_index) + ' of layer ' +\n\u001b[1;32m 165\u001b[0m \u001b[0mlayer_name\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' is incompatible with the layer: '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 171 | "\u001b[0;31mAttributeError\u001b[0m: 'function' object has no attribute 'shape'" 172 | ] 173 | } 174 | ], 175 | "source": [ 176 | "model = GoogLeNet(im_height=im_height,im_width=im_width,class_num=5,aux_logits=True)\n", 177 | "# model.build((batch_size,224,224,3)) # subclass model\n", 178 | "model.summary()" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": null, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [] 187 | } 188 | ], 189 | "metadata": { 190 | "kernelspec": { 191 | "display_name": "Python 3", 192 | "language": "python", 193 | "name": "python3" 194 | }, 195 | "language_info": { 196 | "codemirror_mode": { 197 | "name": "ipython", 198 | "version": 3 199 | }, 200 | "file_extension": ".py", 201 | "mimetype": "text/x-python", 202 | "name": "python", 203 | "nbconvert_exporter": "python", 204 | "pygments_lexer": "ipython3", 205 | "version": "3.7.6" 206 | } 207 | }, 208 | "nbformat": 4, 209 | "nbformat_minor": 4 210 | } 211 | -------------------------------------------------------------------------------- /TF_GoogLeNet/.ipynb_checkpoints/trainGPU-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_GoogLeNet/Model.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import tensorflow as tf\n", 10 | "from tensorflow.keras import layers,models,Model,Sequential" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 2, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "class Inception(layers.Layer):\n", 20 | " def __init__(self, ch1x1, ch3x3reduce, ch3x3, ch5x5reduce, ch5x5, pool_proj, **kwargs):\n", 21 | " super(Inception, self).__init__(**kwargs)\n", 22 | " self.branch1 = layers.Conv2D(ch1x1, kernel_size=1, activation='relu')\n", 23 | " \n", 24 | " self.branch2 = Sequential([\n", 25 | " layers.Conv2D(ch3x3reduce,kernel_size=1,activation='relu'),\n", 26 | " layers.Conv2D(ch3x3,kernel_size=3,activation='relu')\n", 27 | " ])\n", 28 | " \n", 29 | " self.branch3 = Sequential([\n", 30 | " layers.Conv2D(ch5x5reduce,kernel_size=1,activation='relu'),\n", 31 | " layers.Conv2D(ch5x5,kernel_size=5,padding='same',activation='relu')\n", 32 | " ])\n", 33 | " \n", 34 | " self.branch4 = Sequential([\n", 35 | " layers.MaxPool2D(pool_size=3,strides=1,padding='same'),\n", 36 | " layers.Conv2D(pool_proj,kernel_size=1,activation='relu')\n", 37 | " ])\n", 38 | " \n", 39 | " def call(self,inputs,**kwargs):\n", 40 | " branch1 = self.branch1(inputs)\n", 41 | " branch2 = self.branch2(inputs)\n", 42 | " branch3 = self.branch3(inputs)\n", 43 | " branch4 = self.branch4(inputs)\n", 44 | " outputs = layers.concatenate([branch1,branch2,branch3,branch4])\n", 45 | " return outputs" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 3, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "class InceptionAux(layers.Layer):\n", 55 | " def __init__(self, num_classes, **kwargs):\n", 56 | " super(InceptionAux, self).__init__(**kwargs)\n", 57 | " self.averagePool = layers.AvgPool2D(pool_size=5,strides=3)\n", 58 | " self.conv = layers.Conv2D(128,kernel_size=1,activation='relu')\n", 59 | " \n", 60 | " self.fc1 = layers.Dense(1024,activation='relu')\n", 61 | " self.fc2 = layers.Dense(num_classes)\n", 62 | " self.softmax = layers.Softmax()\n", 63 | " \n", 64 | " def call(self,inputs,**kwargs):\n", 65 | " # aux1:N x 512 x 14 x 14, aux2:N x 528 x 14 x 14\n", 66 | " x = self.averagePool(inputs)\n", 67 | " # aux1:N x 512 x 4 x 4, aux2: N x 528 x 4 x 4\n", 68 | " x = self.conv(x)\n", 69 | " # N x 128 x 4 x 4\n", 70 | " x = layers.Flatten()(x)\n", 71 | " x = layers.Dropout(rate=0.5)(x)\n", 72 | " # N x 2048\n", 73 | " x = self.fc1(x)\n", 74 | " x = layers.Dropout(rate=0.5)(x)\n", 75 | " # N x 1024\n", 76 | " x = self.fc2(x)\n", 77 | " x = self.softmax(x)\n", 78 | " return x" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 4, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "def GoogLeNetV1(im_height=224, im_width=224, class_num=1000, aux_logits=False):\n", 88 | " # same卷积：卷积后的特征图大小跟卷积核大小无关，只跟卷积所用的步长有关，当卷积步长是1时，卷积前后特征图大小保持不变\n", 89 | " # W1 = math.ceil(W / s)\n", 90 | " # H1 = math.ceil(H / s)\n", 91 | " # valid卷积：VALID（根据图片实际大小执行卷积，不对图像边界填充，一般卷积后特征图变小）\n", 92 | " # W1 = math.ceil((W-k+1) / s)\n", 93 | " # H1 = math.ceil((H-k+1) / s)\n", 94 | " input_image = layers.Input(shape=(im_height,im_width,3),dtype='float32')\n", 95 | " #(None,224,224,3)\n", 96 | " x = layers.Conv2D(64,kernel_size=7,strides=2,padding='same',activation='relu',name='conv2d_1')(input)\n", 97 | " #(None,112,112,64)\n", 98 | " x = layers.MaxPool2D(pool_size=2,strides=2,name='maxpool_1')(x)\n", 99 | " #(None,56,56,64)\n", 100 | " x = layers.Conv2D(64, kernel_size=1,activation='relu',name=\"conv2d_2\")(x)\n", 101 | " #(None,56,56,64)\n", 102 | " x = layers.Conv2D(192,kernel_size=3,padding='same',activation='relu',name=\"conv2d_3\")(x)\n", 103 | " #(None,56,56,192)\n", 104 | " x = layers.MaxPool2D(pool_size=3,strides=2,padding='same',name='maxpool_2')(x)\n", 105 | " #(None,28,28,192)\n", 106 | " \n", 107 | " x = Inception(ch1x1=64, ch3x3reduce=96, ch3x3=128, ch5x5reduce=16, ch5x5=32, pool_proj=32, name='inception_3a')(x)\n", 108 | " #(None,28,28,256)\n", 109 | " x = Inception(ch1x1=128,ch3x3reduce=96,ch3x3=128,ch5x5reduce=16,ch5x5=32,pool_proj=64,name='inception_3b')(x)\n", 110 | " \n", 111 | " #(None,28,28,480)\n", 112 | " x = layers.MaxPool2D(pool_size=3, strides=2, padding='same', name='maxpool_3')(x)\n", 113 | " #(None,14,14,480)\n", 114 | " x = Inception(ch1x1=192, ch3x3reduce=96, ch3x3=208, ch5x5reduce=16, ch5x5=48, pool_proj=64, name='inception_4a')(x)\n", 115 | " if aux_logits:\n", 116 | " aux1 = InceptionAux(class_num,name='aux_1')(x)\n", 117 | " \n", 118 | " #(None,14,14,512)\n", 119 | " x = Inception(ch1x1=160, ch3x3reduce=112, ch3x3=224, ch5x5reduce=24, ch5x5=48, pool_proj=64, name='inception_4b')(x)\n", 120 | " # (None, 14, 14, 512)\n", 121 | " x = Inception(128, 128, 256, 24, 64, 64, name=\"inception_4c\")(x)\n", 122 | " # (None, 14, 14, 512)\n", 123 | " x = Inception(112, 144, 288, 32, 64, 64, name=\"inception_4d\")(x)\n", 124 | " if aux_logits:\n", 125 | " aux2 = InceptionAux(class_num, name=\"aux_2\")(x)\n", 126 | " \n", 127 | " # (None, 14, 14, 528)\n", 128 | " x = Inception(256, 160, 320, 32, 128, 128, name=\"inception_4e\")(x)\n", 129 | " # (None, 14, 14, 532)\n", 130 | " x = layers.MaxPool2D(pool_size=3, strides=2, padding=\"SAME\", name=\"maxpool_4\")(x)\n", 131 | "\n", 132 | " # (None, 7, 7, 832)\n", 133 | " x = Inception(256, 160, 320, 32, 128, 128, name=\"inception_5a\")(x)\n", 134 | " # (None, 7, 7, 832)\n", 135 | " x = Inception(384, 192, 384, 48, 128, 128, name=\"inception_5b\")(x)\n", 136 | " # (None, 7, 7, 1024)\n", 137 | " x = layers.AvgPool2D(pool_size=7, strides=1, name=\"avgpool_1\")(x)\n", 138 | " \n", 139 | " # (None, 1, 1, 1024)\n", 140 | " x = layers.Flatten(name=\"output_flatten\")(x)\n", 141 | " # (None, 1024)\n", 142 | " x = layers.Dropout(rate=0.4, name=\"output_dropout\")(x)\n", 143 | " x = layers.Dense(class_num, name=\"output_dense\")(x)\n", 144 | " # (None, class_num)\n", 145 | " aux3 = layers.Softmax(name=\"aux_3\")(x)\n", 146 | "\n", 147 | " if aux_logits:\n", 148 | " model = models.Model(inputs=input_image, outputs=[aux1, aux2, aux3])\n", 149 | " else:\n", 150 | " model = models.Model(inputs=input_image, outputs=aux3)\n", 151 | " return model" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": null, 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": null, 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [] 167 | }, 168 | { 169 | "cell_type": "code", 170 | "execution_count": null, 171 | "metadata": {}, 172 | "outputs": [], 173 | "source": [] 174 | } 175 | ], 176 | "metadata": { 177 | "kernelspec": { 178 | "display_name": "Python 3", 179 | "language": "python", 180 | "name": "python3" 181 | }, 182 | "language_info": { 183 | "codemirror_mode": { 184 | "name": "ipython", 185 | "version": 3 186 | }, 187 | "file_extension": ".py", 188 | "mimetype": "text/x-python", 189 | "name": "python", 190 | "nbconvert_exporter": "python", 191 | "pygments_lexer": "ipython3", 192 | "version": "3.7.6" 193 | } 194 | }, 195 | "nbformat": 4, 196 | "nbformat_minor": 4 197 | } 198 | -------------------------------------------------------------------------------- /TF_GoogLeNet/Model_add_bn.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from tensorflow.keras import layers,models,Model,Sequential" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "class Inception(layers.Layer):\n", 19 | " def __init__(self, ch1x1, ch3x3red, ch3x3, ch5x5red, ch5x5, pool_proj, **kwargs):\n", 20 | " super(Inception, self).__init__(**kwargs)\n", 21 | " self.branch1 = Sequential([\n", 22 | " layers.Conv2D(ch1x1, kernel_size=1, use_bias=False, name=\"conv\"),\n", 23 | " layers.BatchNormalization(momentum=0.9, epsilon=1e-5, name=\"bn\"),\n", 24 | " layers.ReLU()], name=\"branch1\")\n", 25 | "\n", 26 | " self.branch2 = Sequential([\n", 27 | " layers.Conv2D(ch3x3red, kernel_size=1, use_bias=False, name=\"0/conv\"),\n", 28 | " layers.BatchNormalization(momentum=0.9, epsilon=1e-5, name=\"0/bn\"),\n", 29 | " layers.ReLU(),\n", 30 | " layers.Conv2D(ch3x3, kernel_size=3, padding=\"SAME\", use_bias=False, name=\"1/conv\"),\n", 31 | " layers.BatchNormalization(momentum=0.9, epsilon=1e-5, name=\"1/bn\"),\n", 32 | " layers.ReLU()], name=\"branch2\") # output_size= input_size\n", 33 | "\n", 34 | " self.branch3 = Sequential([\n", 35 | " layers.Conv2D(ch5x5red, kernel_size=1, use_bias=False, name=\"0/conv\"),\n", 36 | " layers.BatchNormalization(momentum=0.9, epsilon=1e-5, name=\"0/bn\"),\n", 37 | " layers.ReLU(),\n", 38 | " layers.Conv2D(ch5x5, kernel_size=3, padding=\"SAME\", use_bias=False, name=\"1/conv\"),\n", 39 | " layers.BatchNormalization(momentum=0.9, epsilon=1e-5, name=\"1/bn\"),\n", 40 | " layers.ReLU()], name=\"branch3\") # output_size= input_size\n", 41 | "\n", 42 | " self.branch4 = Sequential([\n", 43 | " layers.MaxPool2D(pool_size=3, strides=1, padding=\"SAME\"), # caution: default strides==pool_size\n", 44 | " layers.Conv2D(pool_proj, kernel_size=1, use_bias=False, name=\"1/conv\"),\n", 45 | " layers.BatchNormalization(momentum=0.9, epsilon=1e-5, name=\"1/bn\"),\n", 46 | " layers.ReLU()], name=\"branch4\") # output_size= input_size\n", 47 | "\n", 48 | " def call(self, inputs, **kwargs):\n", 49 | " branch1 = self.branch1(inputs)\n", 50 | " branch2 = self.branch2(inputs)\n", 51 | " branch3 = self.branch3(inputs)\n", 52 | " branch4 = self.branch4(inputs)\n", 53 | " outputs = layers.concatenate([branch1, branch2, branch3, branch4])\n", 54 | " return outputs" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 3, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "class InceptionAux(layers.Layer):\n", 64 | " def __init__(self, num_classes, **kwargs):\n", 65 | " super(InceptionAux, self).__init__(**kwargs)\n", 66 | " self.averagePool = layers.AvgPool2D(pool_size=5, strides=3)\n", 67 | " self.conv = layers.Conv2D(128, kernel_size=1, use_bias=False, name=\"conv/conv\")\n", 68 | " self.bn1 = layers.BatchNormalization(momentum=0.9, epsilon=1e-5, name=\"conv/bn\")\n", 69 | " self.rule1 = layers.ReLU()\n", 70 | "\n", 71 | " self.fc1 = layers.Dense(1024, activation=\"relu\", name=\"fc1\")\n", 72 | " self.fc2 = layers.Dense(num_classes, name=\"fc2\")\n", 73 | " self.softmax = layers.Softmax()\n", 74 | "\n", 75 | " def call(self, inputs, **kwargs):\n", 76 | " # aux1: N x 512 x 14 x 14, aux2: N x 528 x 14 x 14\n", 77 | " x = self.averagePool(inputs)\n", 78 | " # aux1: N x 512 x 4 x 4, aux2: N x 528 x 4 x 4\n", 79 | " x = self.conv(x)\n", 80 | " x = self.bn1(x)\n", 81 | " x = self.rule1(x)\n", 82 | " # N x 128 x 4 x 4\n", 83 | " x = layers.Flatten()(x)\n", 84 | " x = layers.Dropout(rate=0.5)(x)\n", 85 | " # N x 2048\n", 86 | " x = self.fc1(x)\n", 87 | " x = layers.Dropout(rate=0.5)(x)\n", 88 | " # N x 1024\n", 89 | " x = self.fc2(x)\n", 90 | " # N x num_classes\n", 91 | " x = self.softmax(x)\n", 92 | "\n", 93 | " return x" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 4, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "def InceptionV1(im_height=224, im_width=224, class_num=1000, aux_logits=False):\n", 103 | " # tensorflow中的tensor通道排序是NHWC\n", 104 | " input_image = layers.Input(shape=(im_height, im_width, 3), dtype=\"float32\")\n", 105 | " # (None, 224, 224, 3)\n", 106 | " x = layers.Conv2D(64, kernel_size=7, strides=2, padding=\"SAME\", use_bias=False, name=\"conv1/conv\")(input_image)\n", 107 | " x = layers.BatchNormalization(momentum=0.9, epsilon=1e-5, name=\"conv1/bn\")(x)\n", 108 | " x = layers.ReLU()(x)\n", 109 | " # (None, 112, 112, 64)\n", 110 | " x = layers.MaxPool2D(pool_size=3, strides=2, padding=\"SAME\", name=\"maxpool_1\")(x)\n", 111 | " # (None, 56, 56, 64)\n", 112 | " x = layers.Conv2D(64, kernel_size=1, use_bias=False, name=\"conv2/conv\")(x)\n", 113 | " x = layers.BatchNormalization(momentum=0.9, epsilon=1e-5, name=\"conv2/bn\")(x)\n", 114 | " x = layers.ReLU()(x)\n", 115 | " # (None, 56, 56, 64)\n", 116 | " x = layers.Conv2D(192, kernel_size=3, padding=\"SAME\", use_bias=False, name=\"conv3/conv\")(x)\n", 117 | " x = layers.BatchNormalization(momentum=0.9, epsilon=1e-5, name=\"conv3/bn\")(x)\n", 118 | " x = layers.ReLU()(x)\n", 119 | " # (None, 56, 56, 192)\n", 120 | " x = layers.MaxPool2D(pool_size=3, strides=2, padding=\"SAME\", name=\"maxpool_2\")(x)\n", 121 | "\n", 122 | " # (None, 28, 28, 192)\n", 123 | " x = Inception(64, 96, 128, 16, 32, 32, name=\"inception3a\")(x)\n", 124 | " # (None, 28, 28, 256)\n", 125 | " x = Inception(128, 128, 192, 32, 96, 64, name=\"inception3b\")(x)\n", 126 | "\n", 127 | " # (None, 28, 28, 480)\n", 128 | " x = layers.MaxPool2D(pool_size=3, strides=2, padding=\"SAME\", name=\"maxpool_3\")(x)\n", 129 | " # (None, 14, 14, 480)\n", 130 | " x = Inception(192, 96, 208, 16, 48, 64, name=\"inception4a\")(x)\n", 131 | " if aux_logits:\n", 132 | " aux1 = InceptionAux(class_num, name=\"aux1\")(x)\n", 133 | "\n", 134 | " # (None, 14, 14, 512)\n", 135 | " x = Inception(160, 112, 224, 24, 64, 64, name=\"inception4b\")(x)\n", 136 | " # (None, 14, 14, 512)\n", 137 | " x = Inception(128, 128, 256, 24, 64, 64, name=\"inception4c\")(x)\n", 138 | " # (None, 14, 14, 512)\n", 139 | " x = Inception(112, 144, 288, 32, 64, 64, name=\"inception4d\")(x)\n", 140 | " if aux_logits:\n", 141 | " aux2 = InceptionAux(class_num, name=\"aux2\")(x)\n", 142 | "\n", 143 | " # (None, 14, 14, 528)\n", 144 | " x = Inception(256, 160, 320, 32, 128, 128, name=\"inception4e\")(x)\n", 145 | " # (None, 14, 14, 532)\n", 146 | " x = layers.MaxPool2D(pool_size=2, strides=2, padding=\"SAME\", name=\"maxpool_4\")(x)\n", 147 | "\n", 148 | " # (None, 7, 7, 832)\n", 149 | " x = Inception(256, 160, 320, 32, 128, 128, name=\"inception5a\")(x)\n", 150 | " # (None, 7, 7, 832)\n", 151 | " x = Inception(384, 192, 384, 48, 128, 128, name=\"inception5b\")(x)\n", 152 | " # (None, 7, 7, 1024)\n", 153 | " x = layers.AvgPool2D(pool_size=7, strides=1, name=\"avgpool_1\")(x)\n", 154 | "\n", 155 | " # (None, 1, 1, 1024)\n", 156 | " x = layers.Flatten(name=\"output_flatten\")(x)\n", 157 | " # (None, 1024)\n", 158 | " x = layers.Dropout(rate=0.4, name=\"output_dropout\")(x)\n", 159 | " x = layers.Dense(class_num, name=\"fc\")(x)\n", 160 | " # (None, class_num)\n", 161 | " aux3 = layers.Softmax()(x)\n", 162 | "\n", 163 | " if aux_logits:\n", 164 | " model = models.Model(inputs=input_image, outputs=[aux1, aux2, aux3])\n", 165 | " else:\n", 166 | " model = models.Model(inputs=input_image, outputs=aux3)\n", 167 | " return model" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": null, 173 | "metadata": {}, 174 | "outputs": [], 175 | "source": [] 176 | } 177 | ], 178 | "metadata": { 179 | "kernelspec": { 180 | "display_name": "Python 3", 181 | "language": "python", 182 | "name": "python3" 183 | }, 184 | "language_info": { 185 | "codemirror_mode": { 186 | "name": "ipython", 187 | "version": 3 188 | }, 189 | "file_extension": ".py", 190 | "mimetype": "text/x-python", 191 | "name": "python", 192 | "nbconvert_exporter": "python", 193 | "pygments_lexer": "ipython3", 194 | "version": "3.7.6" 195 | } 196 | }, 197 | "nbformat": 4, 198 | "nbformat_minor": 4 199 | } 200 | -------------------------------------------------------------------------------- /TF_GoogLeNet/class_indices.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": "dandelion", 3 | "1": "daisy", 4 | "2": "roses", 5 | "3": "tulips", 6 | "4": "sunflowers" 7 | } -------------------------------------------------------------------------------- /TF_GoogLeNet/model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Coding-Zuo/DL-paper/3015361b2ed710b4b49811ea0a0bb535426d9b09/TF_GoogLeNet/model.png -------------------------------------------------------------------------------- /TF_GoogLeNet/trainCPU.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "importing Jupyter notebook from Model.ipynb\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", 18 | "import matplotlib.pyplot as plt\n", 19 | "import tensorflow as tf\n", 20 | "import json\n", 21 | "import os\n", 22 | "import import_ipynb\n", 23 | "from Model import GoogLeNet" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 2, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "data_root = os.path.abspath(os.path.join(os.getcwd(),\"../../datasets\"))\n", 33 | "image_path = data_root+\"/flower_data/\"\n", 34 | "train_dir = image_path+'train'\n", 35 | "validation_dir = image_path+'val'" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 3, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "if not os.path.exists('save_weights'):\n", 45 | " os.mkdir('save_weights')\n", 46 | "im_height=224\n", 47 | "im_width =224\n", 48 | "batch_size=32\n", 49 | "epochs = 30" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 4, 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "def pre_function(img):\n", 59 | " img = img/255.\n", 60 | " img = (img-0.5)*2.0\n", 61 | " return img" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 5, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "name": "stdout", 71 | "output_type": "stream", 72 | "text": [ 73 | "Found 3306 images belonging to 5 classes.\n" 74 | ] 75 | } 76 | ], 77 | "source": [ 78 | "train_image_generator = ImageDataGenerator(preprocessing_function=pre_function,\n", 79 | " horizontal_flip=True)\n", 80 | "validation_image_generator = ImageDataGenerator(preprocessing_function=pre_function)\n", 81 | "\n", 82 | "\n", 83 | "train_data_gen = train_image_generator.flow_from_directory(directory=train_dir,\n", 84 | " batch_size=batch_size,\n", 85 | " shuffle=True,\n", 86 | " target_size=(im_height,im_width),\n", 87 | " class_mode='categorical')\n", 88 | "total_train = train_data_gen.n" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 6, 94 | "metadata": {}, 95 | "outputs": [ 96 | { 97 | "data": { 98 | "text/plain": [ 99 | "{'daisy': 0, 'dandelion': 1, 'roses': 2, 'sunflowers': 3, 'tulips': 4}" 100 | ] 101 | }, 102 | "execution_count": 6, 103 | "metadata": {}, 104 | "output_type": "execute_result" 105 | } 106 | ], 107 | "source": [ 108 | "class_indices = train_data_gen.class_indices\n", 109 | "class_indices" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 7, 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [ 118 | "inverse_dict = dict((val,key) for key,val in class_indices.items())" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": 8, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "json_str = json.dumps(inverse_dict,indent=4)\n", 128 | "with open('class_indices.json','w') as json_file:\n", 129 | " json_file.write(json_str)" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 9, 135 | "metadata": {}, 136 | "outputs": [ 137 | { 138 | "name": "stdout", 139 | "output_type": "stream", 140 | "text": [ 141 | "Found 364 images belonging to 5 classes.\n" 142 | ] 143 | } 144 | ], 145 | "source": [ 146 | "val_data_gen = validation_image_generator.flow_from_directory(directory=validation_dir,\n", 147 | " batch_size=batch_size,\n", 148 | " shuffle=False,\n", 149 | " target_size=(im_height,im_width),\n", 150 | " class_mode='categorical')\n", 151 | "total_val = val_data_gen.n" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 10, 157 | "metadata": {}, 158 | "outputs": [ 159 | { 160 | "ename": "AttributeError", 161 | "evalue": "'function' object has no attribute 'shape'", 162 | "output_type": "error", 163 | "traceback": [ 164 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 165 | "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", 166 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGoogLeNet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mim_height\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mim_height\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mim_width\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mim_width\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mclass_num\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maux_logits\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;31m# model.build((batch_size,224,224,3)) # subclass model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 167 | "\u001b[0;32m/home/zuoyuhui/paper/GoogLeNet/Model.ipynb\u001b[0m in \u001b[0;36mGoogLeNet\u001b[0;34m(im_height, im_width, class_num, aux_logits)\u001b[0m\n", 168 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[1;32m 885\u001b[0m \u001b[0;31m# Eager execution on data tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 886\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 887\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_build\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 888\u001b[0m \u001b[0mcast_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_inputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 889\u001b[0m with base_layer_utils.autocast_context_manager(\n", 169 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m_maybe_build\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2120\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2121\u001b[0m input_spec.assert_input_compatibility(\n\u001b[0;32m-> 2122\u001b[0;31m self.input_spec, inputs, self.name)\n\u001b[0m\u001b[1;32m 2123\u001b[0m \u001b[0minput_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2124\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0minput_list\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dtype_policy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_dtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 170 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/input_spec.py\u001b[0m in \u001b[0;36massert_input_compatibility\u001b[0;34m(input_spec, inputs, layer_name)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[0mspec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmin_ndim\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 162\u001b[0m spec.max_ndim is not None):\n\u001b[0;32m--> 163\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndims\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 164\u001b[0m raise ValueError('Input ' + str(input_index) + ' of layer ' +\n\u001b[1;32m 165\u001b[0m \u001b[0mlayer_name\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' is incompatible with the layer: '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 171 | "\u001b[0;31mAttributeError\u001b[0m: 'function' object has no attribute 'shape'" 172 | ] 173 | } 174 | ], 175 | "source": [ 176 | "model = GoogLeNet(im_height=im_height,im_width=im_width,class_num=5,aux_logits=True)\n", 177 | "# model.build((batch_size,224,224,3)) # subclass model\n", 178 | "model.summary()" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 21, 184 | "metadata": {}, 185 | "outputs": [], 186 | "source": [ 187 | "loss_object = tf.keras.losses.CategoricalCrossentropy(from_logits=False)\n", 188 | "optimizer = tf.keras.optimizers.Adam(learning_rate=0.0003)\n", 189 | "\n", 190 | "train_loss = tf.keras.metrics.Mean(name='train_loss')\n", 191 | "train_accuracy = tf.keras.metrics.CategoricalAccuracy(name='train_accuracy')\n", 192 | "\n", 193 | "test_loss = tf.keras.metrics.Mean(name='test_loss')\n", 194 | "test_accuracy = tf.keras.metrics.CategoricalAccuracy(name='test_accuracy')" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 22, 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [ 203 | "@tf.function\n", 204 | "def train_step(images,lables):\n", 205 | " with tf.GradientTape() as tape:\n", 206 | " aux1,aux2,output = model(images,training=True)\n", 207 | " loss1 = loss_object(lables,aux1)\n", 208 | " loss2 = loss_object(lables,aux2)\n", 209 | " loss3 = loss_object(lables,output)\n", 210 | " loss = loss1*0.3 + loss2*0.3 + loss3\n", 211 | " gradients = tape.gradient(loss,model.trainable_variables)\n", 212 | " optimizer.apply_gradients(zip(gradients,model.trainable_variables))\n", 213 | " \n", 214 | " train_loss(loss)\n", 215 | " train_accuracy(lables,output)" 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "execution_count": 17, 221 | "metadata": {}, 222 | "outputs": [], 223 | "source": [ 224 | "@tf.function\n", 225 | "def test_step(images,labels):\n", 226 | " _,_,output = model(images,training=False)\n", 227 | " t_loss = loss_object(labels,output)\n", 228 | " \n", 229 | " test_loss(t_loss)\n", 230 | " test_accuracy(labels,output)" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": 18, 236 | "metadata": {}, 237 | "outputs": [ 238 | { 239 | "name": "stdout", 240 | "output_type": "stream", 241 | "text": [ 242 | "Epoch 1, Loss:2.195668935775757, Accuracy:35.21685791015625, Test Loss:1.1382713317871094, Test Accuracy:46.306819915771484\n", 243 | "Epoch 2, Loss:1.7295641899108887, Accuracy:50.97087478637695, Test Loss:0.9794753789901733, Test Accuracy:62.95180892944336\n", 244 | "Epoch 3, Loss:1.4911433458328247, Accuracy:60.904090881347656, Test Loss:0.8690599799156189, Test Accuracy:64.45783233642578\n", 245 | "Epoch 4, Loss:1.3782355785369873, Accuracy:64.26390075683594, Test Loss:0.8264938592910767, Test Accuracy:67.77108001708984\n", 246 | "Epoch 5, Loss:1.2864623069763184, Accuracy:67.56261444091797, Test Loss:0.919053852558136, Test Accuracy:65.66265106201172\n", 247 | "Epoch 6, Loss:1.2512352466583252, Accuracy:68.87599182128906, Test Loss:0.7279787659645081, Test Accuracy:73.79518127441406\n", 248 | "Epoch 7, Loss:1.1951675415039062, Accuracy:72.08307647705078, Test Loss:0.7642440795898438, Test Accuracy:70.78313446044922\n", 249 | "Epoch 8, Loss:1.118859052658081, Accuracy:72.72449493408203, Test Loss:0.7976974248886108, Test Accuracy:70.78313446044922\n", 250 | "Epoch 9, Loss:1.0620782375335693, Accuracy:74.55712127685547, Test Loss:0.7536340951919556, Test Accuracy:70.78313446044922\n", 251 | "Epoch 10, Loss:1.0686018466949463, Accuracy:75.2596206665039, Test Loss:0.7879422903060913, Test Accuracy:70.78313446044922\n", 252 | "Epoch 11, Loss:1.0002797842025757, Accuracy:76.23701477050781, Test Loss:0.7664726376533508, Test Accuracy:75.90361022949219\n", 253 | "Epoch 12, Loss:0.9563718438148499, Accuracy:77.61148071289062, Test Loss:0.7247485518455505, Test Accuracy:75.60240936279297\n", 254 | "Epoch 13, Loss:0.8968386054039001, Accuracy:79.2608413696289, Test Loss:0.5625154972076416, Test Accuracy:79.82954406738281\n", 255 | "Epoch 14, Loss:0.870758593082428, Accuracy:78.92486572265625, Test Loss:0.5581256151199341, Test Accuracy:80.72289276123047\n", 256 | "Epoch 15, Loss:0.8147966861724854, Accuracy:81.24617767333984, Test Loss:0.6690281629562378, Test Accuracy:78.01204681396484\n", 257 | "Epoch 16, Loss:0.7983230352401733, Accuracy:81.4599838256836, Test Loss:0.580307126045227, Test Accuracy:80.1204833984375\n", 258 | "Epoch 17, Loss:0.771430253982544, Accuracy:82.10140228271484, Test Loss:0.5536425113677979, Test Accuracy:79.51807403564453\n", 259 | "Epoch 18, Loss:0.6873193383216858, Accuracy:84.20891571044922, Test Loss:0.5167847871780396, Test Accuracy:82.22891998291016\n", 260 | "Epoch 19, Loss:0.7409345507621765, Accuracy:82.55956268310547, Test Loss:0.4958334267139435, Test Accuracy:84.03614044189453\n", 261 | "Epoch 20, Loss:0.6013174653053284, Accuracy:86.4080581665039, Test Loss:0.7333269715309143, Test Accuracy:78.01204681396484\n", 262 | "Epoch 21, Loss:0.6686580777168274, Accuracy:84.78924560546875, Test Loss:0.5952565670013428, Test Accuracy:79.51807403564453\n", 263 | "Epoch 22, Loss:0.5491890907287598, Accuracy:87.35491943359375, Test Loss:0.6028401851654053, Test Accuracy:76.50602722167969\n", 264 | "Epoch 23, Loss:0.5468771457672119, Accuracy:87.1716537475586, Test Loss:0.6734090447425842, Test Accuracy:81.02410125732422\n", 265 | "Epoch 24, Loss:0.4604325294494629, Accuracy:89.76786804199219, Test Loss:0.5269966721534729, Test Accuracy:81.02410125732422\n", 266 | "Epoch 25, Loss:0.44093891978263855, Accuracy:90.31765747070312, Test Loss:0.7149350643157959, Test Accuracy:78.9772720336914\n", 267 | "Epoch 26, Loss:0.48439326882362366, Accuracy:89.27916717529297, Test Loss:0.6388185024261475, Test Accuracy:80.1204833984375\n", 268 | "Epoch 27, Loss:0.39715901017189026, Accuracy:91.69212341308594, Test Loss:0.6739597916603088, Test Accuracy:76.8072280883789\n", 269 | "Epoch 28, Loss:0.36852991580963135, Accuracy:91.35614013671875, Test Loss:0.6173710227012634, Test Accuracy:81.62650299072266\n", 270 | "Epoch 29, Loss:0.36978453397750854, Accuracy:91.44776916503906, Test Loss:0.6826313138008118, Test Accuracy:78.91566467285156\n", 271 | "Epoch 30, Loss:0.33965742588043213, Accuracy:92.60842895507812, Test Loss:0.5520007610321045, Test Accuracy:82.53012084960938\n" 272 | ] 273 | } 274 | ], 275 | "source": [ 276 | "best_test_loss = float('inf')\n", 277 | "for epoch in range(1,epochs+1):\n", 278 | " train_loss.reset_states()\n", 279 | " test_loss.reset_states()\n", 280 | " train_accuracy.reset_states()\n", 281 | " test_accuracy.reset_states()\n", 282 | " \n", 283 | " for step in range(total_train//batch_size):\n", 284 | " images,labels = next(train_data_gen)\n", 285 | " train_step(images,labels)\n", 286 | " \n", 287 | " for step in range(total_val//batch_size):\n", 288 | " test_images,test_labels = next(val_data_gen)\n", 289 | " test_step(test_images,test_labels)\n", 290 | " \n", 291 | " template ='Epoch {}, Loss:{}, Accuracy:{}, Test Loss:{}, Test Accuracy:{}'\n", 292 | " print(template.format(epoch,\n", 293 | " train_loss.result(),\n", 294 | " train_accuracy.result()*100,\n", 295 | " test_loss.result(),\n", 296 | " test_accuracy.result()*100))\n", 297 | " if test_loss.result() < best_test_loss:\n", 298 | " best_test_loss=test_loss.result()\n", 299 | " model.save_weights('./save_weights/myGoogleNetCPU.h5')" 300 | ] 301 | }, 302 | { 303 | "cell_type": "code", 304 | "execution_count": null, 305 | "metadata": {}, 306 | "outputs": [], 307 | "source": [] 308 | } 309 | ], 310 | "metadata": { 311 | "kernelspec": { 312 | "display_name": "Python 3", 313 | "language": "python", 314 | "name": "python3" 315 | }, 316 | "language_info": { 317 | "codemirror_mode": { 318 | "name": "ipython", 319 | "version": 3 320 | }, 321 | "file_extension": ".py", 322 | "mimetype": "text/x-python", 323 | "name": "python", 324 | "nbconvert_exporter": "python", 325 | "pygments_lexer": "ipython3", 326 | "version": "3.7.6" 327 | } 328 | }, 329 | "nbformat": 4, 330 | "nbformat_minor": 4 331 | } 332 | -------------------------------------------------------------------------------- /TF_VGG/.ipynb_checkpoints/VGGModel-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_VGG/.ipynb_checkpoints/class_indices-checkpoint.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": "daisy", 3 | "1": "dandelion", 4 | "2": "roses", 5 | "3": "sunflowers", 6 | "4": "tulips" 7 | } -------------------------------------------------------------------------------- /TF_VGG/.ipynb_checkpoints/fine_train_vgg16-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_VGG/.ipynb_checkpoints/predict-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_VGG/.ipynb_checkpoints/read_ckpt-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /TF_VGG/.ipynb_checkpoints/trainVGG-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", 10 | "import matplotlib.pyplot as plt\n", 11 | "import tensorflow as tf\n", 12 | "import json\n", 13 | "import os\n", 14 | "os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # 不使用GPU" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 4, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import import_ipynb\n", 24 | "from VGGModel import vgg" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 5, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "data_root = os.path.abspath(os.path.join(os.getcwd(),\"../../datasets\"))\n", 34 | "image_path = data_root+\"/flower_data/\"\n", 35 | "train_dir = image_path+'train'\n", 36 | "validation_dir = image_path+'val'" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 6, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "im_height=224\n", 46 | "im_width =224\n", 47 | "batch_size=32\n", 48 | "epochs = 10" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 10, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | "Found 3306 images belonging to 5 classes.\n" 61 | ] 62 | }, 63 | { 64 | "data": { 65 | "text/plain": [ 66 | "3306" 67 | ] 68 | }, 69 | "execution_count": 10, 70 | "metadata": {}, 71 | "output_type": "execute_result" 72 | } 73 | ], 74 | "source": [ 75 | "# 论文里有去均值的预处理，这里没有是因为从头开始训练的不是迁移学习\n", 76 | "# ImageNet RGB三通道均值是[123.68,116.78,103.94]\n", 77 | "train_image_generator = ImageDataGenerator(rescale=1./255.,horizontal_flip=True)\n", 78 | "validation_image_generator = ImageDataGenerator(rescale=1./255.)\n", 79 | "\n", 80 | "train_data_gen = train_image_generator.flow_from_directory(directory=train_dir,\n", 81 | " batch_size=batch_size,\n", 82 | " shuffle=True,\n", 83 | " target_size=(im_height,im_width),\n", 84 | " class_mode = 'categorical')\n", 85 | "train_data_gen.n" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [] 94 | } 95 | ], 96 | "metadata": { 97 | "kernelspec": { 98 | "display_name": "Python 3", 99 | "language": "python", 100 | "name": "python3" 101 | }, 102 | "language_info": { 103 | "codemirror_mode": { 104 | "name": "ipython", 105 | "version": 3 106 | }, 107 | "file_extension": ".py", 108 | "mimetype": "text/x-python", 109 | "name": "python", 110 | "nbconvert_exporter": "python", 111 | "pygments_lexer": "ipython3", 112 | "version": "3.7.6" 113 | } 114 | }, 115 | "nbformat": 4, 116 | "nbformat_minor": 4 117 | } 118 | -------------------------------------------------------------------------------- /TF_VGG/VGGModel.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from tensorflow.keras import layers,models,Model,Sequential" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "def VGG(feature,im_height=224,im_width=224,class_num=1000):\n", 19 | " input_image = layers.Input(shape=(im_height,im_width,3),dtype=\"float32\")\n", 20 | " x = feature(input_image)\n", 21 | " x = layers.Flatten()(x)\n", 22 | " x = layers.Dropout(rate=0.5)(x)\n", 23 | " x = layers.Dense(2048,activation='relu')(x)\n", 24 | " x = layers.Dropout(rate=0.5)(x)\n", 25 | " x = layers.Dense(2048,activation='relu')(x)\n", 26 | " x = layers.Dense(class_num)(x)\n", 27 | " output = layers.Softmax()(x)\n", 28 | " model = models.Model(inputs = input_image ,outputs = output)\n", 29 | " return model" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 3, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "cfgs = {\n", 39 | " 'vgg11': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],\n", 40 | " 'vgg13': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'],\n", 41 | " 'vgg16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'],\n", 42 | " 'vgg19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'],\n", 43 | "}" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 4, 49 | "metadata": {}, 50 | "outputs": [], 51 | "source": [ 52 | "def features(cfg):\n", 53 | " feature_layers = []\n", 54 | " for v in cfg:\n", 55 | " if v == 'M':\n", 56 | " feature_layers.append(layers.MaxPool2D(pool_size=2,strides=2))\n", 57 | " else:\n", 58 | " conv2d = layers.Conv2D(v,kernel_size=3,padding='same',activation='relu')\n", 59 | " feature_layers.append(conv2d)\n", 60 | " return Sequential(feature_layers,name='feature')" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 5, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [ 69 | "def vgg(model_name=\"vgg16\",im_height=224,im_width=224,class_num=1000):\n", 70 | " try:\n", 71 | " cfg=cfgs[model_name]\n", 72 | " except:\n", 73 | " print('Warning:model number{} not in cfgs dict!'.format(model_name))\n", 74 | " exit(-1)\n", 75 | " model = VGG(features(cfg),im_height=im_height,im_width=im_width,class_num=class_num)\n", 76 | " return model" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 6, 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "# model = vgg(model_name='vgg11')\n", 86 | "# model.summary()" 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "execution_count": null, 92 | "metadata": {}, 93 | "outputs": [], 94 | "source": [] 95 | } 96 | ], 97 | "metadata": { 98 | "kernelspec": { 99 | "display_name": "Python 3", 100 | "language": "python", 101 | "name": "python3" 102 | }, 103 | "language_info": { 104 | "codemirror_mode": { 105 | "name": "ipython", 106 | "version": 3 107 | }, 108 | "file_extension": ".py", 109 | "mimetype": "text/x-python", 110 | "name": "python", 111 | "nbconvert_exporter": "python", 112 | "pygments_lexer": "ipython3", 113 | "version": "3.7.6" 114 | } 115 | }, 116 | "nbformat": 4, 117 | "nbformat_minor": 4 118 | } 119 | -------------------------------------------------------------------------------- /TF_VGG/class_indices.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": "daisy", 3 | "1": "dandelion", 4 | "2": "roses", 5 | "3": "sunflowers", 6 | "4": "tulips" 7 | } -------------------------------------------------------------------------------- /TF_VGG/fine_train_vgg16.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "importing Jupyter notebook from VGGModel.ipynb\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", 18 | "import matplotlib.pyplot as plt\n", 19 | "import import_ipynb\n", 20 | "from VGGModel import vgg\n", 21 | "import tensorflow as tf\n", 22 | "import json\n", 23 | "import os" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": null, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "data_root = os.path.abspath(os.path.join(os.getcwd(), \"../..\")) # get data root path\n", 33 | "image_path = data_root + \"/data_set/flower_data/\" # flower data set path\n", 34 | "train_dir = image_path + \"train\"\n", 35 | "validation_dir = image_path + \"val\"\n", 36 | "\n", 37 | "# create direction for saving weights\n", 38 | "if not os.path.exists(\"save_weights\"):\n", 39 | " os.makedirs(\"save_weights\")\n", 40 | "\n", 41 | "im_height = 224\n", 42 | "im_width = 224\n", 43 | "batch_size = 32\n", 44 | "epochs = 10\n", 45 | "\n", 46 | "_R_MEAN = 123.68\n", 47 | "_G_MEAN = 116.78\n", 48 | "_B_MEAN = 103.94\n", 49 | "\n", 50 | "def pre_function(img):\n", 51 | " # img = im.open('test.jpg')\n", 52 | " # img = np.array(img).astype(np.float32)\n", 53 | " img = img - [_R_MEAN, _G_MEAN, _B_MEAN]\n", 54 | "\n", 55 | " return img\n", 56 | "\n", 57 | "# data generator with data augmentation\n", 58 | "train_image_generator = ImageDataGenerator(horizontal_flip=True,\n", 59 | " preprocessing_function=pre_function)\n", 60 | "validation_image_generator = ImageDataGenerator(preprocessing_function=pre_function)\n", 61 | "\n", 62 | "train_data_gen = train_image_generator.flow_from_directory(directory=train_dir,\n", 63 | " batch_size=batch_size,\n", 64 | " shuffle=True,\n", 65 | " target_size=(im_height, im_width),\n", 66 | " class_mode='categorical')\n", 67 | "total_train = train_data_gen.n\n", 68 | "\n", 69 | "# get class dict\n", 70 | "class_indices = train_data_gen.class_indices\n", 71 | "\n", 72 | "# transform value and key of dict\n", 73 | "inverse_dict = dict((val, key) for key, val in class_indices.items())\n", 74 | "# write dict into json file\n", 75 | "json_str = json.dumps(inverse_dict, indent=4)\n", 76 | "with open('class_indices.json', 'w') as json_file:\n", 77 | " json_file.write(json_str)\n", 78 | "\n", 79 | "val_data_gen = validation_image_generator.flow_from_directory(directory=validation_dir,\n", 80 | " batch_size=batch_size,\n", 81 | " shuffle=False,\n", 82 | " target_size=(im_height, im_width),\n", 83 | " class_mode='categorical')\n", 84 | "total_val = val_data_gen.n\n", 85 | "\n", 86 | "model = vgg(\"vgg16\", 224, 224, 5)\n", 87 | "model.load_weights('./pretrain_weights.ckpt')\n", 88 | "for layer_t in model.layers:\n", 89 | " if layer_t.name == 'feature':\n", 90 | " layer_t.trainable = False\n", 91 | " break\n", 92 | "\n", 93 | "model.summary()\n", 94 | "\n", 95 | "# using keras high level api for training\n", 96 | "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),\n", 97 | " loss=tf.keras.losses.CategoricalCrossentropy(from_logits=False),\n", 98 | " metrics=[\"accuracy\"])\n", 99 | "\n", 100 | "callbacks = [tf.keras.callbacks.ModelCheckpoint(filepath='./save_weights/myAlex_{epoch}.h5',\n", 101 | " save_best_only=True,\n", 102 | " save_weights_only=True,\n", 103 | " monitor='val_loss')]\n", 104 | "\n", 105 | " # tensorflow2.1 recommend to using fit\n", 106 | "history = model.fit(x=train_data_gen,\n", 107 | " steps_per_epoch=total_train // batch_size,\n", 108 | " epochs=epochs,\n", 109 | " validation_data=val_data_gen,\n", 110 | " validation_steps=total_val // batch_size,\n", 111 | " callbacks=callbacks)\n" 112 | ] 113 | } 114 | ], 115 | "metadata": { 116 | "kernelspec": { 117 | "display_name": "Python 3", 118 | "language": "python", 119 | "name": "python3" 120 | }, 121 | "language_info": { 122 | "codemirror_mode": { 123 | "name": "ipython", 124 | "version": 3 125 | }, 126 | "file_extension": ".py", 127 | "mimetype": "text/x-python", 128 | "name": "python", 129 | "nbconvert_exporter": "python", 130 | "pygments_lexer": "ipython3", 131 | "version": "3.7.6" 132 | } 133 | }, 134 | "nbformat": 4, 135 | "nbformat_minor": 4 136 | } 137 | -------------------------------------------------------------------------------- /TF_VGG/read_ckpt.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import tensorflow as tf" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "def rename_var(ckpt_path, new_ckpt_path, num_classes=5):\n", 19 | " with tf.Graph().as_default(), tf.compat.v1.Session().as_default() as sess:\n", 20 | " var_list = tf.train.list_variables(ckpt_path)\n", 21 | " new_var_list = []\n", 22 | "\n", 23 | " for var_name, shape in var_list:\n", 24 | " # print(var_name)\n", 25 | " if var_name in except_list:\n", 26 | " continue\n", 27 | "\n", 28 | " var = tf.train.load_variable(ckpt_path, var_name)\n", 29 | " new_var_name = var_name.replace('vgg_16', 'feature')\n", 30 | " new_var_name = new_var_name.replace(\"weights\", \"kernel\")\n", 31 | " new_var_name = new_var_name.replace(\"biases\", \"bias\")\n", 32 | "\n", 33 | " new_var_name = new_var_name.replace(\"conv1/conv1_1\", \"conv2d\")\n", 34 | " new_var_name = new_var_name.replace(\"conv1/conv1_2\", \"conv2d_1\")\n", 35 | "\n", 36 | " new_var_name = new_var_name.replace(\"conv2/conv2_1\", \"conv2d_2\")\n", 37 | " new_var_name = new_var_name.replace(\"conv2/conv2_2\", \"conv2d_3\")\n", 38 | "\n", 39 | " new_var_name = new_var_name.replace(\"conv3/conv3_1\", \"conv2d_4\")\n", 40 | " new_var_name = new_var_name.replace(\"conv3/conv3_2\", \"conv2d_5\")\n", 41 | " new_var_name = new_var_name.replace(\"conv3/conv3_3\", \"conv2d_6\")\n", 42 | "\n", 43 | " new_var_name = new_var_name.replace(\"conv4/conv4_1\", \"conv2d_7\")\n", 44 | " new_var_name = new_var_name.replace(\"conv4/conv4_2\", \"conv2d_8\")\n", 45 | " new_var_name = new_var_name.replace(\"conv4/conv4_3\", \"conv2d_9\")\n", 46 | "\n", 47 | " new_var_name = new_var_name.replace(\"conv5/conv5_1\", \"conv2d_10\")\n", 48 | " new_var_name = new_var_name.replace(\"conv5/conv5_2\", \"conv2d_11\")\n", 49 | " new_var_name = new_var_name.replace(\"conv5/conv5_3\", \"conv2d_12\")\n", 50 | "\n", 51 | " if 'fc' in new_var_name:\n", 52 | " # new_var_name = new_var_name.replace(\"feature/fc6\", \"dense\")\n", 53 | " # new_var_name = new_var_name.replace(\"feature/fc7\", \"dense_1\")\n", 54 | " # new_var_name = new_var_name.replace(\"fc8\", \"dense_2\")\n", 55 | " continue\n", 56 | "\n", 57 | " # print(new_var_name)\n", 58 | " re_var = tf.Variable(var, name=new_var_name)\n", 59 | " new_var_list.append(re_var)\n", 60 | "\n", 61 | " re_var = tf.Variable(tf.keras.initializers.he_uniform()([25088, 2048]), name=\"dense/kernel\")\n", 62 | " new_var_list.append(re_var)\n", 63 | " re_var = tf.Variable(tf.keras.initializers.he_uniform()([2048]), name=\"dense/bias\")\n", 64 | " new_var_list.append(re_var)\n", 65 | "\n", 66 | " re_var = tf.Variable(tf.keras.initializers.he_uniform()([2048, 2048]), name=\"dense_1/kernel\")\n", 67 | " new_var_list.append(re_var)\n", 68 | " re_var = tf.Variable(tf.keras.initializers.he_uniform()([2048]), name=\"dense_1/bias\")\n", 69 | " new_var_list.append(re_var)\n", 70 | "\n", 71 | " re_var = tf.Variable(tf.keras.initializers.he_uniform()([2048, num_classes]), name=\"dense_2/kernel\")\n", 72 | " new_var_list.append(re_var)\n", 73 | " re_var = tf.Variable(tf.keras.initializers.he_uniform()([num_classes]), name=\"dense_2/bias\")\n", 74 | " new_var_list.append(re_var)\n", 75 | "\n", 76 | " saver = tf.compat.v1.train.Saver(new_var_list)\n", 77 | " sess.run(tf.compat.v1.global_variables_initializer())\n", 78 | " saver.save(sess, save_path=new_ckpt_path, write_meta_graph=False, write_state=False)" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 3, 84 | "metadata": {}, 85 | "outputs": [ 86 | { 87 | "ename": "NotFoundError", 88 | "evalue": "Unsuccessful TensorSliceReader constructor: Failed to find any matching files for ./vgg_16.ckpt", 89 | "output_type": "error", 90 | "traceback": [ 91 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 92 | "\u001b[0;31mNotFoundError\u001b[0m Traceback (most recent call last)", 93 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mnew_ckpt_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'./pretrain_weights.ckpt'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mnum_classes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mrename_var\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mckpt_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_ckpt_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_classes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 94 | "\u001b[0;32m\u001b[0m in \u001b[0;36mrename_var\u001b[0;34m(ckpt_path, new_ckpt_path, num_classes)\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrename_var\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mckpt_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_ckpt_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_classes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mGraph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mvar_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlist_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mckpt_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mnew_var_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 95 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/training/checkpoint_utils.py\u001b[0m in \u001b[0;36mlist_variables\u001b[0;34m(ckpt_dir_or_file)\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0mList\u001b[0m \u001b[0mof\u001b[0m \u001b[0mtuples\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 96\u001b[0m \"\"\"\n\u001b[0;32m---> 97\u001b[0;31m \u001b[0mreader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mckpt_dir_or_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 98\u001b[0m \u001b[0mvariable_map\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_variable_to_shape_map\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[0mnames\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvariable_map\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 96 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/training/checkpoint_utils.py\u001b[0m in \u001b[0;36mload_checkpoint\u001b[0;34m(ckpt_dir_or_file)\u001b[0m\n\u001b[1;32m 64\u001b[0m raise ValueError(\"Couldn't find 'checkpoint' file or checkpoints in \"\n\u001b[1;32m 65\u001b[0m \"given directory %s\" % ckpt_dir_or_file)\n\u001b[0;32m---> 66\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mpywrap_tensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNewCheckpointReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 97 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/pywrap_tensorflow_internal.py\u001b[0m in \u001b[0;36mNewCheckpointReader\u001b[0;34m(filepattern)\u001b[0m\n\u001b[1;32m 871\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mNewCheckpointReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepattern\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 872\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 873\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mCheckpointReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepattern\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 874\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 875\u001b[0m \u001b[0mNewCheckpointReader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tf_api_names_v1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'train.NewCheckpointReader'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 98 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/pywrap_tensorflow_internal.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, filename)\u001b[0m\n\u001b[1;32m 883\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 884\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 885\u001b[0;31m \u001b[0mthis\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_pywrap_tensorflow_internal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnew_CheckpointReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 886\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 887\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mthis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 99 | "\u001b[0;31mNotFoundError\u001b[0m: Unsuccessful TensorSliceReader constructor: Failed to find any matching files for ./vgg_16.ckpt" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "except_list = ['global_step', 'vgg_16/mean_rgb', 'vgg_16/fc8/biases', 'vgg_16/fc8/weights']\n", 105 | "# http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz\n", 106 | "ckpt_path = './vgg_16.ckpt'\n", 107 | "new_ckpt_path = './pretrain_weights.ckpt'\n", 108 | "num_classes = 5\n", 109 | "rename_var(ckpt_path, new_ckpt_path, num_classes)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": null, 115 | "metadata": {}, 116 | "outputs": [], 117 | "source": [] 118 | } 119 | ], 120 | "metadata": { 121 | "kernelspec": { 122 | "display_name": "Python 3", 123 | "language": "python", 124 | "name": "python3" 125 | }, 126 | "language_info": { 127 | "codemirror_mode": { 128 | "name": "ipython", 129 | "version": 3 130 | }, 131 | "file_extension": ".py", 132 | "mimetype": "text/x-python", 133 | "name": "python", 134 | "nbconvert_exporter": "python", 135 | "pygments_lexer": "ipython3", 136 | "version": "3.7.6" 137 | } 138 | }, 139 | "nbformat": 4, 140 | "nbformat_minor": 4 141 | } 142 | -------------------------------------------------------------------------------- /TF_VGG/trainVGG.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", 10 | "import matplotlib.pyplot as plt\n", 11 | "import tensorflow as tf\n", 12 | "import json\n", 13 | "import os\n", 14 | "os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # 不使用GPU" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 4, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import import_ipynb\n", 24 | "from VGGModel import vgg" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 5, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "data_root = os.path.abspath(os.path.join(os.getcwd(),\"../../datasets\"))\n", 34 | "image_path = data_root+\"/flower_data/\"\n", 35 | "train_dir = image_path+'train'\n", 36 | "validation_dir = image_path+'val'" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 6, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "im_height=224\n", 46 | "im_width =224\n", 47 | "batch_size=32\n", 48 | "epochs = 10" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 18, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | "Found 3306 images belonging to 5 classes.\n" 61 | ] 62 | } 63 | ], 64 | "source": [ 65 | "# 论文里有去均值的预处理，这里没有是因为从头开始训练的不是迁移学习\n", 66 | "# ImageNet RGB三通道均值是[123.68,116.78,103.94]\n", 67 | "train_image_generator = ImageDataGenerator(rescale=1./255.,horizontal_flip=True)\n", 68 | "validation_image_generator = ImageDataGenerator(rescale=1./255.)\n", 69 | "\n", 70 | "train_data_gen = train_image_generator.flow_from_directory(directory=train_dir,\n", 71 | " batch_size=batch_size,\n", 72 | " shuffle=True,\n", 73 | " target_size=(im_height,im_width),\n", 74 | " class_mode = 'categorical')\n", 75 | "total_train=train_data_gen.n" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 11, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "data": { 85 | "text/plain": [ 86 | "{'daisy': 0, 'dandelion': 1, 'roses': 2, 'sunflowers': 3, 'tulips': 4}" 87 | ] 88 | }, 89 | "execution_count": 11, 90 | "metadata": {}, 91 | "output_type": "execute_result" 92 | } 93 | ], 94 | "source": [ 95 | "class_indices = train_data_gen.class_indices\n", 96 | "class_indices" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 12, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "inverse_dict = dict((val,key) for key,val in class_indices.items())\n", 106 | "\n", 107 | "json_str = json.dumps(inverse_dict,indent=4)\n", 108 | "with open('class_indices.json','w') as json_file:\n", 109 | " json_file.write(json_str)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 13, 115 | "metadata": {}, 116 | "outputs": [ 117 | { 118 | "name": "stdout", 119 | "output_type": "stream", 120 | "text": [ 121 | "Found 364 images belonging to 5 classes.\n" 122 | ] 123 | } 124 | ], 125 | "source": [ 126 | "val_data_gen = train_image_generator.flow_from_directory(directory=validation_dir,\n", 127 | " batch_size=batch_size,\n", 128 | " shuffle=True,\n", 129 | " target_size=(im_height,im_width),\n", 130 | " class_mode='categorical')\n", 131 | "total_val=val_data_gen.n" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": 14, 137 | "metadata": {}, 138 | "outputs": [ 139 | { 140 | "name": "stdout", 141 | "output_type": "stream", 142 | "text": [ 143 | "Model: \"model\"\n", 144 | "_________________________________________________________________\n", 145 | "Layer (type) Output Shape Param # \n", 146 | "=================================================================\n", 147 | "input_1 (InputLayer) [(None, 224, 224, 3)] 0 \n", 148 | "_________________________________________________________________\n", 149 | "feature (Sequential) (None, 7, 7, 512) 14714688 \n", 150 | "_________________________________________________________________\n", 151 | "flatten (Flatten) (None, 25088) 0 \n", 152 | "_________________________________________________________________\n", 153 | "dropout (Dropout) (None, 25088) 0 \n", 154 | "_________________________________________________________________\n", 155 | "dense (Dense) (None, 2048) 51382272 \n", 156 | "_________________________________________________________________\n", 157 | "dropout_1 (Dropout) (None, 2048) 0 \n", 158 | "_________________________________________________________________\n", 159 | "dense_1 (Dense) (None, 2048) 4196352 \n", 160 | "_________________________________________________________________\n", 161 | "dense_2 (Dense) (None, 5) 10245 \n", 162 | "_________________________________________________________________\n", 163 | "softmax (Softmax) (None, 5) 0 \n", 164 | "=================================================================\n", 165 | "Total params: 70,303,557\n", 166 | "Trainable params: 70,303,557\n", 167 | "Non-trainable params: 0\n", 168 | "_________________________________________________________________\n" 169 | ] 170 | } 171 | ], 172 | "source": [ 173 | "model = vgg('vgg16',224,224,5)\n", 174 | "model.summary()" 175 | ] 176 | }, 177 | { 178 | "cell_type": "code", 179 | "execution_count": 16, 180 | "metadata": {}, 181 | "outputs": [], 182 | "source": [ 183 | "model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n", 184 | " loss=tf.keras.losses.CategoricalCrossentropy(from_logits=False),\n", 185 | " metrics=['accuracy'])" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": 17, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "callbacks = [tf.keras.callbacks.ModelCheckpoint(filepath='./save_weights/myVGG_{epoch}.h5',\n", 195 | " save_best_only=True,\n", 196 | " save_weights_only=True,\n", 197 | " monitor='val_loss')]" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": null, 203 | "metadata": {}, 204 | "outputs": [], 205 | "source": [ 206 | "history = model.fit(x=train_data_gen,\n", 207 | " steps_per_epoch=total_train//batch_size,\n", 208 | " epochs=epochs,\n", 209 | " validation_data=val_data_gen)" 210 | ] 211 | } 212 | ], 213 | "metadata": { 214 | "kernelspec": { 215 | "display_name": "Python 3", 216 | "language": "python", 217 | "name": "python3" 218 | }, 219 | "language_info": { 220 | "codemirror_mode": { 221 | "name": "ipython", 222 | "version": 3 223 | }, 224 | "file_extension": ".py", 225 | "mimetype": "text/x-python", 226 | "name": "python", 227 | "nbconvert_exporter": "python", 228 | "pygments_lexer": "ipython3", 229 | "version": "3.7.6" 230 | } 231 | }, 232 | "nbformat": 4, 233 | "nbformat_minor": 4 234 | } 235 | -------------------------------------------------------------------------------- /Torch_ResNet/.gitignore: -------------------------------------------------------------------------------- 1 | res* 2 | -------------------------------------------------------------------------------- /Torch_ResNet/.ipynb_checkpoints/predict-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /Torch_ResNet/.ipynb_checkpoints/train-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /Torch_ResNet/class_indices.json: -------------------------------------------------------------------------------- 1 | { 2 | "0": "daisy", 3 | "1": "dandelion", 4 | "2": "roses", 5 | "3": "sunflowers", 6 | "4": "tulips" 7 | } -------------------------------------------------------------------------------- /Transformer/.ipynb_checkpoints/Transformer3-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import math\n", 10 | "import torch\n", 11 | "import numpy as np\n", 12 | "import torch.nn as nn\n", 13 | "import torch.optim as optim\n", 14 | "import torch.utils.data as Data" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 3, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "\"\"\"\n", 24 | "S:decoder的输入\n", 25 | "E:decoder的输出\n", 26 | "P:如果当前批处理数据大小小于序列长度，将填充空白序列的符号\n", 27 | "\"\"\"\n", 28 | "sentences = [\n", 29 | " # enc_input dec_input dec_output\n", 30 | " ['ich mochte ein bier P', 'S i want a beer .', 'i want a beer . E'],\n", 31 | " ['ich mochte ein cola P', 'S i want a coke .', 'i want a coke . E']\n", 32 | "]" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": 4, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "# Padding 是 0\n", 42 | "src_vocab = {'P' : 0, 'ich' : 1, 'mochte' : 2, 'ein' : 3, 'bier' : 4, 'cola' : 5} # 源词典\n", 43 | "src_vocab_size = len(src_vocab)\n", 44 | "\n", 45 | "tgt_vocab = {'P' : 0, 'i' : 1, 'want' : 2, 'a' : 3, 'beer' : 4, 'coke' : 5, 'S' : 6, 'E' : 7, '.' : 8} # 目标词词典\n", 46 | "tgt_vocab_size = len(tgt_vocab)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "execution_count": 6, 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [ 55 | "id2word = {i:w for i,w in enumerate(tgt_vocab)}" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 7, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "src_len = 5 # enc_input 最大序列长度\n", 65 | "tgt_len = 6 # dec_input 最大序列长度\n", 66 | "\n", 67 | "# Transformer 参数\n", 68 | "d_model = 512 # Embedding Size\n", 69 | "d_ff = 2048 # 前向传播维度\n", 70 | "d_k = d_v = 64 # K(=Q),V demension\n", 71 | "n_layers = 6 \n", 72 | "n_heads = 8" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 10, 78 | "metadata": {}, 79 | "outputs": [], 80 | "source": [ 81 | "def make_data(sentences):\n", 82 | " enc_inputs, dec_inputs, dec_outputs = [], [], []\n", 83 | " for i in range(len(sentences)):\n", 84 | " enc_input = [[src_vocab[n] for n in sentences[i][0].split()]] # [[1, 2, 3, 4, 0], [1, 2, 3, 5, 0]]\n", 85 | " dec_input = [[tgt_vocab[n] for n in sentences[i][1].split()]] # [[6, 1, 2, 3, 4, 8], [6, 1, 2, 3, 5, 8]]\n", 86 | " dec_output = [[tgt_vocab[n] for n in sentences[i][2].split()]] # [[1, 2, 3, 4, 8, 7], [1, 2, 3, 5, 8, 7]]\n", 87 | " \n", 88 | " enc_inputs.extend(enc_input)\n", 89 | " dec_inputs.extend(dec_input)\n", 90 | " dec_outputs.extend(dec_output)\n", 91 | " return torch.LongTensor(enc_inputs) , torch.LongTensor(dec_inputs) , torch.LongTensor(dec_outputs)\n", 92 | "\n", 93 | "enc_inputs, dec_inputs, dec_outputs = make_data(sentences)" 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "execution_count": 12, 99 | "metadata": {}, 100 | "outputs": [], 101 | "source": [ 102 | "class MyDataSet(Data.Dataset):\n", 103 | " def __init__(self, enc_inputs, dec_inputs, dec_outputs):\n", 104 | " super(MyDataSet, self).__init__()\n", 105 | " self.enc_inputs=enc_inputs\n", 106 | " self.dec_inputs=dec_inputs\n", 107 | " self.dec_outputs=dec_outputs\n", 108 | " \n", 109 | " def __len__(self):\n", 110 | " return self.enc_inputs.shape[0]\n", 111 | " \n", 112 | " def __getitem__(self,idx):\n", 113 | " return self.enc_inputs[idx], self.dec_inputs[idx], self.dec_outputs[idx]\n", 114 | " \n", 115 | "loader = Data.DataLoader(MyDataSet(enc_inputs, dec_inputs, dec_outputs), batch_size=2 , shuffle=True)" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": null, 128 | "metadata": {}, 129 | "outputs": [], 130 | "source": [] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": null, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "execution_count": null, 142 | "metadata": {}, 143 | "outputs": [], 144 | "source": [] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 14, 149 | "metadata": {}, 150 | "outputs": [ 151 | { 152 | "data": { 153 | "text/plain": [ 154 | "tensor([ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 13.,\n", 155 | " 14., 15., 16., 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27.,\n", 156 | " 28., 29., 30., 31., 32., 33., 34., 35., 36., 37., 38., 39., 40., 41.,\n", 157 | " 42., 43., 44., 45., 46., 47., 48., 49.])" 158 | ] 159 | }, 160 | "execution_count": 14, 161 | "metadata": {}, 162 | "output_type": "execute_result" 163 | } 164 | ], 165 | "source": [] 166 | }, 167 | { 168 | "cell_type": "code", 169 | "execution_count": 15, 170 | "metadata": {}, 171 | "outputs": [], 172 | "source": [ 173 | "class PositionEncoding(nn.Module):\n", 174 | " def __init__(self, d_model, dropout=0.1, max_len=5000):\n", 175 | " super(PositionalEncoding, self).__init__()\n", 176 | " self.dropout = nn.Dropout(p = dropout)\n", 177 | " \n", 178 | " pe = torch.zeros(max_len, d_model)\n", 179 | " position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n", 180 | " div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0)/ d_model))\n", 181 | " \n", 182 | " pe[:,0::2] = torch.sin(position * div_term)\n", 183 | " pe[:,1::2] = torch.cos(position * div_term)\n", 184 | " pe = pe.unsqueeze(0).transpose(0,1)\n", 185 | " self.register_buffer('pe', pe)\n", 186 | " \n", 187 | " def forward(self, x):\n", 188 | " # x:[seq_len, batch_size, d_model]\n", 189 | " x = x + self.pe[:x.size(0), :]\n", 190 | " return self.dropout(x)" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": null, 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [ 199 | "def get_attn_pad_mask(seq_q, seq_k):\n", 200 | " \"\"\"\n", 201 | " seq_q:[batch_size, seq_len]\n", 202 | " seq_k:[batch_size, seq_len]\n", 203 | " seq_len = src_len or tgt_len 不一定等长\n", 204 | " \"\"\"\n", 205 | " batch_size, len_q = seq_q.size()\n", 206 | " batch_size, len_k = seq_k,size()\n", 207 | " # eq(zero) is PAD token\n", 208 | " pad_attn_mask = seq_k.data.eq(0).unsqueeze(1) # [batch_size, 1, len_k] False is Masked\n", 209 | " return pad_attn_mask.expand(batch_size, len_q, len_k) # [batch_size, len_q, len_k]\n", 210 | " " 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": null, 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [ 219 | "class Encoder(nn.Module):\n", 220 | " def __init__(self):\n", 221 | " super(Encoder,self).__init__()\n", 222 | " self.src_emb = nn.Embedding(src_vocab_size, d_model)\n", 223 | " self.pos_emb = PositionEncoding(d_model)\n", 224 | " self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n", 225 | " \n", 226 | " def forward(self, enc_inputs):\n", 227 | " # enc_inputs : [batch_size, src_len]\n", 228 | " enc_outputs = self.src_emb(enc_inputs) # [batch_size,src_len, d_model]\n", 229 | " enc_outputs = self.pos_emb(enc_outputs.transpose(0,1)).transpose(0,1) # [batch_size, stc_len, d_model] \n", 230 | " \n", 231 | " enc_self_attn_mask = get_attn_pad_mask(enc_inputs, enc_inputs) # [batch_size, src_len, src_len] 要隐藏掉padding 0\n", 232 | " enc_self_attns = []\n", 233 | " for layer in self.layers:\n", 234 | " # enc_outputs:[batch_size, src_len, d_model]\n", 235 | " # enc_self_attn:[batch_size, n_heads, src_len, src_len]\n", 236 | " enc_outputs, enc_self_attn = layer(enc_outputs,enc_self_attn_mask) \n", 237 | " enc_self_attns.append(enc_self_attn)\n", 238 | " return enc_outputs, enc_self_attns\n", 239 | " " 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": null, 245 | "metadata": {}, 246 | "outputs": [], 247 | "source": [ 248 | "class Transformer(nn.Module):\n", 249 | " def __init__(self):\n", 250 | " super(Transformer, self).__init__()\n", 251 | " self.encoder = Encoder().cuda()\n", 252 | " self.decoder = Decoder().cuda()\n", 253 | " self.projection = nn.Linear(d_model, tgt_vocab_size, bias=False).cuda()\n", 254 | " \n", 255 | " def forward(self,enc_inputs, dec_inputs):\n", 256 | " \"\"\"\n", 257 | " enc_inputs:[batch_size, src_len]\n", 258 | " dec_inputs:[batch_size, tgt_len]\n", 259 | " \"\"\"\n", 260 | " # enc_outputs:[batch_size, src_len, d_model]\n", 261 | " # enc_self_attns:[n_layers, batch_size, n_heads, src_len, src_len]\n", 262 | " enc_outputs, enc_self_attns = self.encoder(enc_inputs)\n", 263 | " \n", 264 | " # dec_outputs:[batch_size, tgt_len, d_model]\n", 265 | " # dec_self_attns:[n_layers, batch_size, n_heads, tgt_len, tgt_len]\n", 266 | " # dec_enc_attns:[n_layers, batch_size, n_heads, tgt_len, src_len]\n", 267 | " dec_outputs, dec_self_attns, dec_enc_attns = self.decoder(dec_inputs, enc_inputs, enc_outputs)\n", 268 | " # dec_logits:[batch_size, tgt_len, tgt_vocab_size]\n", 269 | " dec_logits = self.projection(dec_outputs)\n", 270 | " \n", 271 | " return dec_logits.view(-1, dec_logits.size(-1)) , enc_self_attns, dec_self_attns, dec_enc_attns\n" 272 | ] 273 | } 274 | ], 275 | "metadata": { 276 | "kernelspec": { 277 | "display_name": "Python 3", 278 | "language": "python", 279 | "name": "python3" 280 | }, 281 | "language_info": { 282 | "codemirror_mode": { 283 | "name": "ipython", 284 | "version": 3 285 | }, 286 | "file_extension": ".py", 287 | "mimetype": "text/x-python", 288 | "name": "python", 289 | "nbconvert_exporter": "python", 290 | "pygments_lexer": "ipython3", 291 | "version": "3.7.6" 292 | } 293 | }, 294 | "nbformat": 4, 295 | "nbformat_minor": 4 296 | } 297 | -------------------------------------------------------------------------------- /Transformer/.ipynb_checkpoints/learn_spacy-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /Transformer/.ipynb_checkpoints/self_attention-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /Transformer/learn_spacy.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "ename": "OSError", 10 | "evalue": "[E050] Can't find model 'en'. It doesn't seem to be a shortcut link, a Python package or a valid path to a data directory.", 11 | "output_type": "error", 12 | "traceback": [ 13 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 14 | "\u001b[0;31mOSError\u001b[0m Traceback (most recent call last)", 15 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mspacy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mnlp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspacy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"en\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 16 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/spacy/__init__.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(name, **overrides)\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdepr_path\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0mwarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mWarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mW001\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepr_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDeprecationWarning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 17 | "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/spacy/util.py\u001b[0m in \u001b[0;36mload_model\u001b[0;34m(name, **overrides)\u001b[0m\n\u001b[1;32m 173\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"exists\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# Path or Path-like to model data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 174\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mload_model_from_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 175\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE050\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 176\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 177\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 18 | "\u001b[0;31mOSError\u001b[0m: [E050] Can't find model 'en'. It doesn't seem to be a shortcut link, a Python package or a valid path to a data directory." 19 | ] 20 | } 21 | ], 22 | "source": [ 23 | "import spacy\n", 24 | "nlp = spacy.load(\"en\")" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": null, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [] 33 | } 34 | ], 35 | "metadata": { 36 | "kernelspec": { 37 | "display_name": "Python 3", 38 | "language": "python", 39 | "name": "python3" 40 | }, 41 | "language_info": { 42 | "codemirror_mode": { 43 | "name": "ipython", 44 | "version": 3 45 | }, 46 | "file_extension": ".py", 47 | "mimetype": "text/x-python", 48 | "name": "python", 49 | "nbconvert_exporter": "python", 50 | "pygments_lexer": "ipython3", 51 | "version": "3.7.6" 52 | } 53 | }, 54 | "nbformat": 4, 55 | "nbformat_minor": 4 56 | } 57 | -------------------------------------------------------------------------------- /Transformer/self_attention.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 9, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "data": { 10 | "text/plain": [ 11 | "torch.Size([2, 3])" 12 | ] 13 | }, 14 | "execution_count": 9, 15 | "metadata": {}, 16 | "output_type": "execute_result" 17 | } 18 | ], 19 | "source": [ 20 | "import torch\n", 21 | "import numpy as np\n", 22 | "index = torch.from_numpy(np.array([[1,2,0], [2,0,1]])).type(torch.LongTensor)\n", 23 | "index.shape" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 10, 29 | "metadata": {}, 30 | "outputs": [ 31 | { 32 | "data": { 33 | "text/plain": [ 34 | "torch.Size([2, 3, 1])" 35 | ] 36 | }, 37 | "execution_count": 10, 38 | "metadata": {}, 39 | "output_type": "execute_result" 40 | } 41 | ], 42 | "source": [ 43 | "index = index[:,:,None]\n", 44 | "index.shape" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 11, 50 | "metadata": {}, 51 | "outputs": [ 52 | { 53 | "data": { 54 | "text/plain": [ 55 | "tensor([[[1],\n", 56 | " [2],\n", 57 | " [0]],\n", 58 | "\n", 59 | " [[2],\n", 60 | " [0],\n", 61 | " [1]]])" 62 | ] 63 | }, 64 | "execution_count": 11, 65 | "metadata": {}, 66 | "output_type": "execute_result" 67 | } 68 | ], 69 | "source": [ 70 | "index" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 14, 76 | "metadata": {}, 77 | "outputs": [ 78 | { 79 | "data": { 80 | "text/plain": [ 81 | "tensor([[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n", 82 | " [2, 2, 2, 2, 2, 2, 2, 2, 2, 2],\n", 83 | " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],\n", 84 | "\n", 85 | " [[2, 2, 2, 2, 2, 2, 2, 2, 2, 2],\n", 86 | " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n", 87 | " [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]])" 88 | ] 89 | }, 90 | "execution_count": 14, 91 | "metadata": {}, 92 | "output_type": "execute_result" 93 | } 94 | ], 95 | "source": [ 96 | "index = index.expand(-1,-1,10)\n", 97 | "index" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 15, 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "data": { 107 | "text/plain": [ 108 | "torch.Size([2, 3, 10])" 109 | ] 110 | }, 111 | "execution_count": 15, 112 | "metadata": {}, 113 | "output_type": "execute_result" 114 | } 115 | ], 116 | "source": [ 117 | "index.shape " 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 16, 123 | "metadata": {}, 124 | "outputs": [ 125 | { 126 | "data": { 127 | "text/plain": [ 128 | "tensor([[[0.8499, 0.5743, 0.1163, 0.7354, 0.7336, 0.0461, 0.5596, 0.5597,\n", 129 | " 0.8331, 0.8325],\n", 130 | " [0.3856, 0.3141, 0.2987, 0.2221, 0.7227, 0.9536, 0.9431, 0.8833,\n", 131 | " 0.6936, 0.0054],\n", 132 | " [0.4265, 0.8442, 0.0826, 0.3865, 0.4617, 0.3623, 0.0011, 0.3030,\n", 133 | " 0.3444, 0.3968]],\n", 134 | "\n", 135 | " [[0.2399, 0.8165, 0.5930, 0.0286, 0.9473, 0.1417, 0.3795, 0.2192,\n", 136 | " 0.5359, 0.6525],\n", 137 | " [0.2812, 0.9378, 0.3347, 0.7453, 0.1059, 0.8147, 0.4491, 0.3992,\n", 138 | " 0.0209, 0.1799],\n", 139 | " [0.1993, 0.1519, 0.2089, 0.3881, 0.1018, 0.3271, 0.7906, 0.7492,\n", 140 | " 0.9048, 0.2077]]])" 141 | ] 142 | }, 143 | "execution_count": 16, 144 | "metadata": {}, 145 | "output_type": "execute_result" 146 | } 147 | ], 148 | "source": [ 149 | "inputs = torch.rand(2,3,10)\n", 150 | "inputs # # 可以理解为 2个batch 每个batch 3句话， 每句话10个词 只不过不是索引值是连续值" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": 17, 156 | "metadata": {}, 157 | "outputs": [ 158 | { 159 | "data": { 160 | "text/plain": [ 161 | "tensor([[[0.3856, 0.3141, 0.2987, 0.2221, 0.7227, 0.9536, 0.9431, 0.8833,\n", 162 | " 0.6936, 0.0054],\n", 163 | " [0.4265, 0.8442, 0.0826, 0.3865, 0.4617, 0.3623, 0.0011, 0.3030,\n", 164 | " 0.3444, 0.3968],\n", 165 | " [0.8499, 0.5743, 0.1163, 0.7354, 0.7336, 0.0461, 0.5596, 0.5597,\n", 166 | " 0.8331, 0.8325]],\n", 167 | "\n", 168 | " [[0.1993, 0.1519, 0.2089, 0.3881, 0.1018, 0.3271, 0.7906, 0.7492,\n", 169 | " 0.9048, 0.2077],\n", 170 | " [0.2399, 0.8165, 0.5930, 0.0286, 0.9473, 0.1417, 0.3795, 0.2192,\n", 171 | " 0.5359, 0.6525],\n", 172 | " [0.2812, 0.9378, 0.3347, 0.7453, 0.1059, 0.8147, 0.4491, 0.3992,\n", 173 | " 0.0209, 0.1799]]])" 174 | ] 175 | }, 176 | "execution_count": 17, 177 | "metadata": {}, 178 | "output_type": "execute_result" 179 | } 180 | ], 181 | "source": [ 182 | "torch.gather(inputs, 1, index)" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": null, 195 | "metadata": {}, 196 | "outputs": [], 197 | "source": [] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": {}, 202 | "source": [ 203 | "# 过程\n", 204 | "- 准备输入\n", 205 | "- 初始化参数矩阵\n", 206 | "- 获取key、query和value\n", 207 | "- 计算softmax\n", 208 | "- 给value乘上score\n", 209 | "- 给value 加权求和获取output1\n", 210 | "- 重复4-7 获取output2、3" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": 1, 216 | "metadata": {}, 217 | "outputs": [ 218 | { 219 | "data": { 220 | "text/plain": [ 221 | "tensor([[1., 0., 1., 0.],\n", 222 | " [0., 2., 0., 2.],\n", 223 | " [1., 1., 1., 1.]])" 224 | ] 225 | }, 226 | "execution_count": 1, 227 | "metadata": {}, 228 | "output_type": "execute_result" 229 | } 230 | ], 231 | "source": [ 232 | "import torch\n", 233 | "x = [\n", 234 | " [1,0,1,0], # Input1\n", 235 | " [0,2,0,2], # Input1\n", 236 | " [1,1,1,1] # Input1\n", 237 | "]\n", 238 | "x = torch.tensor(x, dtype=torch.float32)\n", 239 | "x" 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "execution_count": 3, 245 | "metadata": {}, 246 | "outputs": [ 247 | { 248 | "name": "stdout", 249 | "output_type": "stream", 250 | "text": [ 251 | "Weights for key: \n", 252 | " tensor([[0., 0., 1.],\n", 253 | " [1., 1., 0.],\n", 254 | " [0., 1., 0.],\n", 255 | " [1., 1., 0.]])\n", 256 | "Weights for query: \n", 257 | " tensor([[1., 0., 1.],\n", 258 | " [1., 0., 0.],\n", 259 | " [0., 0., 1.],\n", 260 | " [0., 1., 1.]])\n", 261 | "Weights for value: \n", 262 | " tensor([[0., 2., 0.],\n", 263 | " [0., 3., 0.],\n", 264 | " [1., 0., 3.],\n", 265 | " [1., 1., 0.]])\n" 266 | ] 267 | } 268 | ], 269 | "source": [ 270 | "w_key = [\n", 271 | " [0, 0, 1],\n", 272 | " [1, 1, 0],\n", 273 | " [0, 1, 0],\n", 274 | " [1, 1, 0]\n", 275 | "]\n", 276 | "w_query = [\n", 277 | " [1, 0, 1],\n", 278 | " [1, 0, 0],\n", 279 | " [0, 0, 1],\n", 280 | " [0, 1, 1]\n", 281 | "]\n", 282 | "w_value = [\n", 283 | " [0, 2, 0],\n", 284 | " [0, 3, 0],\n", 285 | " [1, 0, 3],\n", 286 | " [1, 1, 0]\n", 287 | "]\n", 288 | "w_key = torch.tensor(w_key, dtype=torch.float32)\n", 289 | "w_query = torch.tensor(w_query, dtype=torch.float32)\n", 290 | "w_value = torch.tensor(w_value, dtype=torch.float32)\n", 291 | "\n", 292 | "print(\"Weights for key: \\n\", w_key)\n", 293 | "print(\"Weights for query: \\n\", w_query)\n", 294 | "print(\"Weights for value: \\n\", w_value)" 295 | ] 296 | }, 297 | { 298 | "cell_type": "code", 299 | "execution_count": 4, 300 | "metadata": {}, 301 | "outputs": [ 302 | { 303 | "name": "stdout", 304 | "output_type": "stream", 305 | "text": [ 306 | "Keys: \n", 307 | " tensor([[0., 1., 1.],\n", 308 | " [4., 4., 0.],\n", 309 | " [2., 3., 1.]])\n", 310 | "Querys: \n", 311 | " tensor([[1., 0., 2.],\n", 312 | " [2., 2., 2.],\n", 313 | " [2., 1., 3.]])\n", 314 | "Values: \n", 315 | " tensor([[1., 2., 3.],\n", 316 | " [2., 8., 0.],\n", 317 | " [2., 6., 3.]])\n" 318 | ] 319 | } 320 | ], 321 | "source": [ 322 | "\n", 323 | "keys = x @ w_key\n", 324 | "querys = x @ w_query\n", 325 | "values = x @ w_value\n", 326 | "\n", 327 | "print(\"Keys: \\n\", keys)\n", 328 | "print(\"Querys: \\n\", querys)\n", 329 | "print(\"Values: \\n\", values)" 330 | ] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "execution_count": 5, 335 | "metadata": {}, 336 | "outputs": [ 337 | { 338 | "data": { 339 | "text/plain": [ 340 | "tensor([[ 2., 4., 4.],\n", 341 | " [ 4., 16., 12.],\n", 342 | " [ 4., 12., 10.]])" 343 | ] 344 | }, 345 | "execution_count": 5, 346 | "metadata": {}, 347 | "output_type": "execute_result" 348 | } 349 | ], 350 | "source": [ 351 | "attn_scores = querys @ keys.T\n", 352 | "attn_scores" 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": 6, 358 | "metadata": {}, 359 | "outputs": [ 360 | { 361 | "data": { 362 | "text/plain": [ 363 | "tensor([[6.3379e-02, 4.6831e-01, 4.6831e-01],\n", 364 | " [6.0337e-06, 9.8201e-01, 1.7986e-02],\n", 365 | " [2.9539e-04, 8.8054e-01, 1.1917e-01]])" 366 | ] 367 | }, 368 | "execution_count": 6, 369 | "metadata": {}, 370 | "output_type": "execute_result" 371 | } 372 | ], 373 | "source": [ 374 | "from torch.nn.functional import softmax\n", 375 | "attn_scores_softmax = softmax(attn_scores, dim=-1)\n", 376 | "attn_scores_softmax" 377 | ] 378 | }, 379 | { 380 | "cell_type": "code", 381 | "execution_count": 7, 382 | "metadata": {}, 383 | "outputs": [ 384 | { 385 | "name": "stdout", 386 | "output_type": "stream", 387 | "text": [ 388 | "tensor([[0.0000, 0.5000, 0.5000],\n", 389 | " [0.0000, 1.0000, 0.0000],\n", 390 | " [0.0000, 0.9000, 0.1000]])\n" 391 | ] 392 | } 393 | ], 394 | "source": [ 395 | "# For readability, approximate the above as follows\n", 396 | "attn_scores_softmax = [\n", 397 | " [0.0, 0.5, 0.5],\n", 398 | " [0.0, 1.0, 0.0],\n", 399 | " [0.0, 0.9, 0.1]\n", 400 | "]\n", 401 | "attn_scores_softmax = torch.tensor(attn_scores_softmax)\n", 402 | "print(attn_scores_softmax)" 403 | ] 404 | }, 405 | { 406 | "cell_type": "code", 407 | "execution_count": 8, 408 | "metadata": {}, 409 | "outputs": [ 410 | { 411 | "name": "stdout", 412 | "output_type": "stream", 413 | "text": [ 414 | "tensor([[[0.0000, 0.0000, 0.0000],\n", 415 | " [0.0000, 0.0000, 0.0000],\n", 416 | " [0.0000, 0.0000, 0.0000]],\n", 417 | "\n", 418 | " [[1.0000, 4.0000, 0.0000],\n", 419 | " [2.0000, 8.0000, 0.0000],\n", 420 | " [1.8000, 7.2000, 0.0000]],\n", 421 | "\n", 422 | " [[1.0000, 3.0000, 1.5000],\n", 423 | " [0.0000, 0.0000, 0.0000],\n", 424 | " [0.2000, 0.6000, 0.3000]]])\n" 425 | ] 426 | } 427 | ], 428 | "source": [ 429 | "weighted_values = values[:,None] * attn_scores_softmax.T[:,:,None]\n", 430 | "print(weighted_values)" 431 | ] 432 | }, 433 | { 434 | "cell_type": "code", 435 | "execution_count": 9, 436 | "metadata": {}, 437 | "outputs": [ 438 | { 439 | "name": "stdout", 440 | "output_type": "stream", 441 | "text": [ 442 | "tensor([[[0.0000, 0.0000, 0.0000],\n", 443 | " [0.0000, 0.0000, 0.0000],\n", 444 | " [0.0000, 0.0000, 0.0000]],\n", 445 | "\n", 446 | " [[1.0000, 4.0000, 0.0000],\n", 447 | " [2.0000, 8.0000, 0.0000],\n", 448 | " [1.8000, 7.2000, 0.0000]],\n", 449 | "\n", 450 | " [[1.0000, 3.0000, 1.5000],\n", 451 | " [0.0000, 0.0000, 0.0000],\n", 452 | " [0.2000, 0.6000, 0.3000]]])\n", 453 | "tensor([[2.0000, 7.0000, 1.5000],\n", 454 | " [2.0000, 8.0000, 0.0000],\n", 455 | " [2.0000, 7.8000, 0.3000]])\n" 456 | ] 457 | } 458 | ], 459 | "source": [ 460 | "weighted_values = values[:,None] * attn_scores_softmax.T[:,:,None]\n", 461 | "print(weighted_values)\n", 462 | "outputs = weighted_values.sum(dim=0)\n", 463 | "print(outputs)\n" 464 | ] 465 | }, 466 | { 467 | "cell_type": "code", 468 | "execution_count": null, 469 | "metadata": {}, 470 | "outputs": [], 471 | "source": [] 472 | } 473 | ], 474 | "metadata": { 475 | "kernelspec": { 476 | "display_name": "Python 3", 477 | "language": "python", 478 | "name": "python3" 479 | }, 480 | "language_info": { 481 | "codemirror_mode": { 482 | "name": "ipython", 483 | "version": 3 484 | }, 485 | "file_extension": ".py", 486 | "mimetype": "text/x-python", 487 | "name": "python", 488 | "nbconvert_exporter": "python", 489 | "pygments_lexer": "ipython3", 490 | "version": "3.7.6" 491 | } 492 | }, 493 | "nbformat": 4, 494 | "nbformat_minor": 4 495 | } 496 | -------------------------------------------------------------------------------- /dgl_GCN/.ipynb_checkpoints/gcn-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch\n", 10 | "import torch.nn as nn\n", 11 | "from dgl.nn.pytorch import GraphConv" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 3, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "# GCN using DGL nn package\n", 21 | "class GCN(nn.Module):\n", 22 | " def __init__(self, g, in_feats, n_hidden, n_classes, n_layers, activation, dropout):\n", 23 | " super(GCN,self).__init__()\n", 24 | " self.g = g\n", 25 | " self.layers = nn.ModuleList()\n", 26 | " # input layer\n", 27 | " self.layers.append(GraphConv(in_feats, n_hidden, activation=activation))\n", 28 | " # hidden layers\n", 29 | " for i in range(n_layers - 1):\n", 30 | " self.layers.append(GraphConv(n_hidden, n_classes, activation=activation))\n", 31 | " # output_layers \n", 32 | " self.layers.append(GraphConv(n_hidden, n_classes))\n", 33 | " self.dropout = nn.Dropout(p = dropout)\n", 34 | " \n", 35 | " def forward(self, features):\n", 36 | " h = features\n", 37 | " for i,layer in enumerate(self.layers):\n", 38 | " if i != 0:\n", 39 | " h = self.dropout(h)\n", 40 | " h = layer(self.g, h)\n", 41 | " return h" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": null, 47 | "metadata": {}, 48 | "outputs": [], 49 | "source": [] 50 | } 51 | ], 52 | "metadata": { 53 | "kernelspec": { 54 | "display_name": "Python 3", 55 | "language": "python", 56 | "name": "python3" 57 | }, 58 | "language_info": { 59 | "codemirror_mode": { 60 | "name": "ipython", 61 | "version": 3 62 | }, 63 | "file_extension": ".py", 64 | "mimetype": "text/x-python", 65 | "name": "python", 66 | "nbconvert_exporter": "python", 67 | "pygments_lexer": "ipython3", 68 | "version": "3.7.6" 69 | } 70 | }, 71 | "nbformat": 4, 72 | "nbformat_minor": 4 73 | } 74 | -------------------------------------------------------------------------------- /dgl_GCN/.ipynb_checkpoints/gcn_mp-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import argparse, time, math\n", 10 | "import numpy as np\n", 11 | "import networkx as nx\n", 12 | "import torch\n", 13 | "import torch.nn as nn\n", 14 | "import torch.nn.functional as F\n", 15 | "import dgl\n", 16 | "from dgl.data import register_data_args\n", 17 | "from dgl.data import CoraGraphDataset, CiteseerGraphDataset, PubmedGraphDataset" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 3, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "def gcn_msg(edge):\n", 27 | " msg = edge.src['h'] * edge.src['norm']\n", 28 | " return {'m':msg}" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 4, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "def gcn_reduce(node):\n", 38 | " accum = torch.sum(node.mailbox['m'],1) * node.data['norm'] # 聚合邻居节点的m特征 在行维度下求和，再乘以norm属性得到accum \n", 39 | " return {'h':accum} # 定义为h" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 5, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "class NodeApplyMoudle(nn.Module):\n", 49 | " def __init__(self, out_feats, activation=None, bias=True):\n", 50 | " super(NodeApplyMoudle, self).__init__()\n", 51 | " if bias:\n", 52 | " self.bias = nn.Parameter(torch.Tensor(out_feats))\n", 53 | " else:\n", 54 | " self.bias = None\n", 55 | " self.activation = activation\n", 56 | " self.reset_parameters()\n", 57 | " \n", 58 | " def reset_parameters(self):\n", 59 | " if self.bias is not None:\n", 60 | " stdv = 1./ math.sqrt(self.bias.size(0))\n", 61 | " self.bias.data.uniform_(-stdv,stdv) # 用从连续均匀分布中采样的数字填充自身张量 1/(stdv+stdv)\n", 62 | " \n", 63 | " def forward(self,nodes):\n", 64 | " h = nodes.data['h']\n", 65 | " if self.bias is not None:\n", 66 | " h = h + self.bias\n", 67 | " if self.activation:\n", 68 | " h = self.activation(h)\n", 69 | " return {'h':h}" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 7, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "class GCNLayer(nn.Module):\n", 79 | " def __init__(self, g, in_feats, out_feats, activation, dropout, bias=True):\n", 80 | " super(GCNLayer, self).__init__()\n", 81 | " self.g = g\n", 82 | " self.weight = nn.Parameter(torch.Tensor(in_feats, out_feats))\n", 83 | " if dropout:\n", 84 | " self.dropout = nn.Dropout(p=dropout)\n", 85 | " else:\n", 86 | " self.dropout = 0.\n", 87 | " self.node_update = NodeApplyMoudle(out_feats, activation, bias)\n", 88 | " self.reset_patameters()\n", 89 | " \n", 90 | " def reset_parameters(self):\n", 91 | " stdv = 1./math.sqrt(self.weight.size(1))\n", 92 | " self.weight.data.uniform_(-stdv, stdv)\n", 93 | " \n", 94 | " def forward(self,h):\n", 95 | " if self.dropout:\n", 96 | " h = self.dropout(h)\n", 97 | " self.g.ndata['h'] = torch.mm(h, self.weight)\n", 98 | " self.g.update_all(gcn_msg, gcn_reduce, self.node_update)\n", 99 | " h = self.g.ndata.pop('h')\n", 100 | " return h" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 8, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "class GCN(nn.Module):\n", 110 | " def __init__(self, g, in_feats, n_hidden, n_classes, n_layers, activation, dropout):\n", 111 | " super(GCN, self).__init__()\n", 112 | " self.layers = nn.ModuleList()\n", 113 | " # input layer\n", 114 | " self.layers.append(GCNLayer(g, in_feats, n_hidden, activation, dropout))\n", 115 | " # hidden layers\n", 116 | " for i in range(n_layers-1):\n", 117 | " self.layers.append(GCNLayer(g, n_hidden, n_hidden, activation, dropout))\n", 118 | " # output layer\n", 119 | " self.layers.append(GCNLayer(g, n_hidden, n_classes, None, dropout))\n", 120 | " \n", 121 | " def forward(self,features):\n", 122 | " h = features\n", 123 | " for layer in self.layers:\n", 124 | " h = layer(h)\n", 125 | " return h \n", 126 | " " 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 9, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "def evaluate(model, features, labels, mask):\n", 136 | " model.eval()\n", 137 | " with torch.no_grad():\n", 138 | " logits = model(features)\n", 139 | " logits = logits[mask]\n", 140 | " labels = labels[mask]\n", 141 | " _, indices = torch.max(logits, dim=1)\n", 142 | " correct = torch.sum(indices == labels)\n", 143 | " return correct.item() * 1.0/len(labels)" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": null, 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "def main(args):\n", 153 | " # load and preprocess dataset\n", 154 | " if args.dataset == 'cora':\n", 155 | " data = CoraGraphDataset()\n", 156 | " elif args.dataset == 'citeseer':\n", 157 | " data = CiteseerGraphDataset()\n", 158 | " elif args.dataset == 'pubmed':\n", 159 | " data = PubmedGraphDataset()\n", 160 | " else:\n", 161 | " raise ValueError('Unknown dataset: {}'.format(args.dataset))\n", 162 | " \n", 163 | " g = data[0]\n", 164 | " if args.gpu < 0:\n", 165 | " cuda = False\n", 166 | " else:\n", 167 | " cuda = True\n", 168 | " g = g.to(args.gpu)\n", 169 | " \n", 170 | " features = g.ndata['feat']\n", 171 | " labels = g.ndata['label']\n", 172 | " train_mask = g.ndata['train_mask']\n", 173 | " val_mask = g.ndata['val_mask']\n", 174 | " test_mask = g.ndata['test_mask']\n", 175 | " in_feats = features.shape[1]\n", 176 | " n_classes = data.num_labels\n", 177 | " n_edges = data.graph.number_of_edges()\n", 178 | " print(\"\"\"----Data statistics------'\n", 179 | " #Edges %d\n", 180 | " #Classes %d\n", 181 | " #Train samples %d\n", 182 | " #Val samples %d\n", 183 | " #Test samples %d\"\"\" %\n", 184 | " (n_edges, n_classes,\n", 185 | " train_mask.int().sum().item(),\n", 186 | " val_mask.int().sum().item(),\n", 187 | " test_mask.int().sum().item()))\n", 188 | " \n", 189 | " # add self loop\n", 190 | " g = dgl.remove_self_loop(g)\n", 191 | " g = dgl.add_self_loop(g)\n", 192 | " n_edges = g.number_of_edges()\n", 193 | " \n", 194 | " # normalization\n", 195 | " degs = g.in_degrees().float()\n", 196 | " norm = torch.pow(degs,-0.5)\n", 197 | " norm[torch.isinf(norm)] = 0\n", 198 | " if cuda:\n", 199 | " norm = norm.cuda()\n", 200 | " g.ndata['norm'] = norm.unsqueeze(1)\n", 201 | " " 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": 12, 207 | "metadata": {}, 208 | "outputs": [ 209 | { 210 | "name": "stdout", 211 | "output_type": "stream", 212 | "text": [ 213 | "Loading from cache failed, re-processing.\n", 214 | "Finished data loading and preprocessing.\n", 215 | " NumNodes: 2708\n", 216 | " NumEdges: 10556\n", 217 | " NumFeats: 1433\n", 218 | " NumClasses: 7\n", 219 | " NumTrainingSamples: 140\n", 220 | " NumValidationSamples: 500\n", 221 | " NumTestSamples: 1000\n", 222 | "Done saving data into cached files.\n" 223 | ] 224 | }, 225 | { 226 | "data": { 227 | "text/plain": [ 228 | "Graph(num_nodes=2708, num_edges=10556,\n", 229 | " ndata_schemes={'train_mask': Scheme(shape=(), dtype=torch.bool), 'val_mask': Scheme(shape=(), dtype=torch.bool), 'test_mask': Scheme(shape=(), dtype=torch.bool), 'label': Scheme(shape=(), dtype=torch.int64), 'feat': Scheme(shape=(1433,), dtype=torch.float32)}\n", 230 | " edata_schemes={})" 231 | ] 232 | }, 233 | "execution_count": 12, 234 | "metadata": {}, 235 | "output_type": "execute_result" 236 | } 237 | ], 238 | "source": [ 239 | "data = CoraGraphDataset()\n", 240 | "g = data[0]\n", 241 | "g" 242 | ] 243 | }, 244 | { 245 | "cell_type": "code", 246 | "execution_count": 13, 247 | "metadata": {}, 248 | "outputs": [ 249 | { 250 | "data": { 251 | "text/plain": [ 252 | "tensor([[0., 0., 0., ..., 0., 0., 0.],\n", 253 | " [0., 0., 0., ..., 0., 0., 0.],\n", 254 | " [0., 0., 0., ..., 0., 0., 0.],\n", 255 | " ...,\n", 256 | " [0., 0., 0., ..., 0., 0., 0.],\n", 257 | " [0., 0., 0., ..., 0., 0., 0.],\n", 258 | " [0., 0., 0., ..., 0., 0., 0.]])" 259 | ] 260 | }, 261 | "execution_count": 13, 262 | "metadata": {}, 263 | "output_type": "execute_result" 264 | } 265 | ], 266 | "source": [ 267 | "g.ndata['feat']" 268 | ] 269 | }, 270 | { 271 | "cell_type": "code", 272 | "execution_count": 14, 273 | "metadata": {}, 274 | "outputs": [ 275 | { 276 | "data": { 277 | "text/plain": [ 278 | "tensor([3, 4, 4, ..., 3, 3, 3])" 279 | ] 280 | }, 281 | "execution_count": 14, 282 | "metadata": {}, 283 | "output_type": "execute_result" 284 | } 285 | ], 286 | "source": [ 287 | "g.ndata['label']" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": 15, 293 | "metadata": {}, 294 | "outputs": [ 295 | { 296 | "data": { 297 | "text/plain": [ 298 | "tensor([ True, True, True, ..., False, False, False])" 299 | ] 300 | }, 301 | "execution_count": 15, 302 | "metadata": {}, 303 | "output_type": "execute_result" 304 | } 305 | ], 306 | "source": [ 307 | "g.ndata['train_mask']" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": 16, 313 | "metadata": {}, 314 | "outputs": [ 315 | { 316 | "data": { 317 | "text/plain": [ 318 | "tensor([False, False, False, ..., False, False, False])" 319 | ] 320 | }, 321 | "execution_count": 16, 322 | "metadata": {}, 323 | "output_type": "execute_result" 324 | } 325 | ], 326 | "source": [ 327 | "g.ndata['val_mask']" 328 | ] 329 | }, 330 | { 331 | "cell_type": "code", 332 | "execution_count": 17, 333 | "metadata": {}, 334 | "outputs": [], 335 | "source": [ 336 | "g = dgl.graph((torch.tensor([0, 1, 1]), torch.tensor([1, 1, 0])))\n", 337 | "g.ndata['h'] = torch.ones(2, 1)" 338 | ] 339 | }, 340 | { 341 | "cell_type": "code", 342 | "execution_count": 21, 343 | "metadata": {}, 344 | "outputs": [ 345 | { 346 | "data": { 347 | "text/plain": [ 348 | "Graph(num_nodes=2, num_edges=3,\n", 349 | " ndata_schemes={'h': Scheme(shape=(1,), dtype=torch.float32)}\n", 350 | " edata_schemes={})" 351 | ] 352 | }, 353 | "execution_count": 21, 354 | "metadata": {}, 355 | "output_type": "execute_result" 356 | } 357 | ], 358 | "source": [ 359 | "g" 360 | ] 361 | }, 362 | { 363 | "cell_type": "code", 364 | "execution_count": 22, 365 | "metadata": {}, 366 | "outputs": [ 367 | { 368 | "data": { 369 | "text/plain": [ 370 | "tensor([[1.],\n", 371 | " [1.]])" 372 | ] 373 | }, 374 | "execution_count": 22, 375 | "metadata": {}, 376 | "output_type": "execute_result" 377 | } 378 | ], 379 | "source": [ 380 | "g.ndata['h']" 381 | ] 382 | }, 383 | { 384 | "cell_type": "code", 385 | "execution_count": null, 386 | "metadata": {}, 387 | "outputs": [], 388 | "source": [] 389 | } 390 | ], 391 | "metadata": { 392 | "kernelspec": { 393 | "display_name": "Python 3", 394 | "language": "python", 395 | "name": "python3" 396 | }, 397 | "language_info": { 398 | "codemirror_mode": { 399 | "name": "ipython", 400 | "version": 3 401 | }, 402 | "file_extension": ".py", 403 | "mimetype": "text/x-python", 404 | "name": "python", 405 | "nbconvert_exporter": "python", 406 | "pygments_lexer": "ipython3", 407 | "version": "3.7.6" 408 | } 409 | }, 410 | "nbformat": 4, 411 | "nbformat_minor": 4 412 | } 413 | -------------------------------------------------------------------------------- /dgl_GCN/gcn.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import torch\n", 10 | "import torch.nn as nn\n", 11 | "from dgl.nn.pytorch import GraphConv" 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 3, 17 | "metadata": {}, 18 | "outputs": [], 19 | "source": [ 20 | "# GCN using DGL nn package\n", 21 | "class GCN(nn.Module):\n", 22 | " def __init__(self, g, in_feats, n_hidden, n_classes, n_layers, activation, dropout):\n", 23 | " super(GCN,self).__init__()\n", 24 | " self.g = g\n", 25 | " self.layers = nn.ModuleList()\n", 26 | " # input layer\n", 27 | " self.layers.append(GraphConv(in_feats, n_hidden, activation=activation))\n", 28 | " # hidden layers\n", 29 | " for i in range(n_layers - 1):\n", 30 | " self.layers.append(GraphConv(n_hidden, n_classes, activation=activation))\n", 31 | " # output_layers \n", 32 | " self.layers.append(GraphConv(n_hidden, n_classes))\n", 33 | " self.dropout = nn.Dropout(p = dropout)\n", 34 | " \n", 35 | " def forward(self, features):\n", 36 | " h = features\n", 37 | " for i,layer in enumerate(self.layers):\n", 38 | " if i != 0:\n", 39 | " h = self.dropout(h)\n", 40 | " h = layer(self.g, h)\n", 41 | " return h" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 4, 47 | "metadata": {}, 48 | "outputs": [ 49 | { 50 | "ename": "NameError", 51 | "evalue": "name 'CoraGraphDataset' is not defined", 52 | "output_type": "error", 53 | "traceback": [ 54 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 55 | "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 56 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCoraGraphDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCiteseerGraphDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPubmedGraphDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 57 | "\u001b[0;31mNameError\u001b[0m: name 'CoraGraphDataset' is not defined" 58 | ] 59 | } 60 | ], 61 | "source": [] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": null, 66 | "metadata": {}, 67 | "outputs": [], 68 | "source": [] 69 | } 70 | ], 71 | "metadata": { 72 | "kernelspec": { 73 | "display_name": "Python 3", 74 | "language": "python", 75 | "name": "python3" 76 | }, 77 | "language_info": { 78 | "codemirror_mode": { 79 | "name": "ipython", 80 | "version": 3 81 | }, 82 | "file_extension": ".py", 83 | "mimetype": "text/x-python", 84 | "name": "python", 85 | "nbconvert_exporter": "python", 86 | "pygments_lexer": "ipython3", 87 | "version": "3.7.6" 88 | } 89 | }, 90 | "nbformat": 4, 91 | "nbformat_minor": 4 92 | } 93 | -------------------------------------------------------------------------------- /dgl_GCN/gcn_mp.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import argparse, time, math\n", 10 | "import numpy as np\n", 11 | "import networkx as nx\n", 12 | "import torch\n", 13 | "import torch.nn as nn\n", 14 | "import torch.nn.functional as F\n", 15 | "import dgl\n", 16 | "from dgl.data import register_data_args\n", 17 | "from dgl.data import CoraGraphDataset, CiteseerGraphDataset, PubmedGraphDataset" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 3, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "def gcn_msg(edge):\n", 27 | " msg = edge.src['h'] * edge.src['norm']\n", 28 | " return {'m':msg}" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 4, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "def gcn_reduce(node):\n", 38 | " accum = torch.sum(node.mailbox['m'],1) * node.data['norm'] # 聚合邻居节点的m特征 在行维度下求和，再乘以norm属性得到accum \n", 39 | " return {'h':accum} # 定义为h" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 5, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "class NodeApplyMoudle(nn.Module):\n", 49 | " def __init__(self, out_feats, activation=None, bias=True):\n", 50 | " super(NodeApplyMoudle, self).__init__()\n", 51 | " if bias:\n", 52 | " self.bias = nn.Parameter(torch.Tensor(out_feats))\n", 53 | " else:\n", 54 | " self.bias = None\n", 55 | " self.activation = activation\n", 56 | " self.reset_parameters()\n", 57 | " \n", 58 | " def reset_parameters(self):\n", 59 | " if self.bias is not None:\n", 60 | " stdv = 1./ math.sqrt(self.bias.size(0))\n", 61 | " self.bias.data.uniform_(-stdv,stdv) # 用从连续均匀分布中采样的数字填充自身张量 1/(stdv+stdv)\n", 62 | " \n", 63 | " def forward(self,nodes):\n", 64 | " h = nodes.data['h']\n", 65 | " if self.bias is not None:\n", 66 | " h = h + self.bias\n", 67 | " if self.activation:\n", 68 | " h = self.activation(h)\n", 69 | " return {'h':h}" 70 | ] 71 | }, 72 | { 73 | "cell_type": "code", 74 | "execution_count": 7, 75 | "metadata": {}, 76 | "outputs": [], 77 | "source": [ 78 | "class GCNLayer(nn.Module):\n", 79 | " def __init__(self, g, in_feats, out_feats, activation, dropout, bias=True):\n", 80 | " super(GCNLayer, self).__init__()\n", 81 | " self.g = g\n", 82 | " self.weight = nn.Parameter(torch.Tensor(in_feats, out_feats))\n", 83 | " if dropout:\n", 84 | " self.dropout = nn.Dropout(p=dropout)\n", 85 | " else:\n", 86 | " self.dropout = 0.\n", 87 | " self.node_update = NodeApplyMoudle(out_feats, activation, bias)\n", 88 | " self.reset_patameters()\n", 89 | " \n", 90 | " def reset_parameters(self):\n", 91 | " stdv = 1./math.sqrt(self.weight.size(1))\n", 92 | " self.weight.data.uniform_(-stdv, stdv)\n", 93 | " \n", 94 | " def forward(self,h):\n", 95 | " if self.dropout:\n", 96 | " h = self.dropout(h)\n", 97 | " self.g.ndata['h'] = torch.mm(h, self.weight)\n", 98 | " self.g.update_all(gcn_msg, gcn_reduce, self.node_update)\n", 99 | " h = self.g.ndata.pop('h')\n", 100 | " return h" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 8, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "class GCN(nn.Module):\n", 110 | " def __init__(self, g, in_feats, n_hidden, n_classes, n_layers, activation, dropout):\n", 111 | " super(GCN, self).__init__()\n", 112 | " self.layers = nn.ModuleList()\n", 113 | " # input layer\n", 114 | " self.layers.append(GCNLayer(g, in_feats, n_hidden, activation, dropout))\n", 115 | " # hidden layers\n", 116 | " for i in range(n_layers-1):\n", 117 | " self.layers.append(GCNLayer(g, n_hidden, n_hidden, activation, dropout))\n", 118 | " # output layer\n", 119 | " self.layers.append(GCNLayer(g, n_hidden, n_classes, None, dropout))\n", 120 | " \n", 121 | " def forward(self,features):\n", 122 | " h = features\n", 123 | " for layer in self.layers:\n", 124 | " h = layer(h)\n", 125 | " return h \n", 126 | " " 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 9, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "def evaluate(model, features, labels, mask):\n", 136 | " model.eval()\n", 137 | " with torch.no_grad():\n", 138 | " logits = model(features)\n", 139 | " logits = logits[mask]\n", 140 | " labels = labels[mask]\n", 141 | " _, indices = torch.max(logits, dim=1)\n", 142 | " correct = torch.sum(indices == labels)\n", 143 | " return correct.item() * 1.0/len(labels)" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": 23, 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "def main(args):\n", 153 | " # load and preprocess dataset\n", 154 | " if args.dataset == 'cora':\n", 155 | " data = CoraGraphDataset()\n", 156 | " elif args.dataset == 'citeseer':\n", 157 | " data = CiteseerGraphDataset()\n", 158 | " elif args.dataset == 'pubmed':\n", 159 | " data = PubmedGraphDataset()\n", 160 | " else:\n", 161 | " raise ValueError('Unknown dataset: {}'.format(args.dataset))\n", 162 | " \n", 163 | " g = data[0]\n", 164 | " if args.gpu < 0:\n", 165 | " cuda = False\n", 166 | " else:\n", 167 | " cuda = True\n", 168 | " g = g.to(args.gpu)\n", 169 | " \n", 170 | " features = g.ndata['feat']\n", 171 | " labels = g.ndata['label']\n", 172 | " train_mask = g.ndata['train_mask']\n", 173 | " val_mask = g.ndata['val_mask']\n", 174 | " test_mask = g.ndata['test_mask']\n", 175 | " in_feats = features.shape[1]\n", 176 | " n_classes = data.num_labels\n", 177 | " n_edges = data.graph.number_of_edges()\n", 178 | " print(\"\"\"----Data statistics------'\n", 179 | " #Edges %d\n", 180 | " #Classes %d\n", 181 | " #Train samples %d\n", 182 | " #Val samples %d\n", 183 | " #Test samples %d\"\"\" %\n", 184 | " (n_edges, n_classes,\n", 185 | " train_mask.int().sum().item(),\n", 186 | " val_mask.int().sum().item(),\n", 187 | " test_mask.int().sum().item()))\n", 188 | " \n", 189 | " # add self loop\n", 190 | " g = dgl.remove_self_loop(g)\n", 191 | " g = dgl.add_self_loop(g)\n", 192 | " n_edges = g.number_of_edges()\n", 193 | " \n", 194 | " # normalization\n", 195 | " degs = g.in_degrees().float()\n", 196 | " norm = torch.pow(degs,-0.5)\n", 197 | " norm[torch.isinf(norm)] = 0\n", 198 | " if cuda:\n", 199 | " norm = norm.cuda()\n", 200 | " g.ndata['norm'] = norm.unsqueeze(1)\n", 201 | " \n", 202 | " model = GCN(g, in_feats, args.n_hidden, n_classes, args.n_layers, F.relu, args.dropout)\n", 203 | " if cuda:\n", 204 | " model.cuda()\n", 205 | " loss_fcn = torch.nn.CrossEntropyLoss()\n", 206 | " optimizer = torch.optim.Adam(model.parameters(),lr=args.lr, weight_decay=args.weight_decay)\n", 207 | " \n", 208 | " # initalize graph\n", 209 | " dur = []\n", 210 | " for epoch in range(args.n_epochs):\n", 211 | " model.train()\n", 212 | " if epoch>=3:\n", 213 | " t0 = time.time()\n", 214 | " # forward\n", 215 | " logits = model(features)\n", 216 | " loss = loss_fcn(logits[train_mask], labels[train_mask])\n", 217 | " optimizer.zero_grad()\n", 218 | " loss.backward()\n", 219 | " optimizer.step()\n", 220 | " \n", 221 | " if epoch >= 3:\n", 222 | " dur.append(time.time() - t0)\n", 223 | " acc = evaluate(model, features, labels, val_mask)\n", 224 | " print(\"Epoch {:05f} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | ETputs(KTEPS) {:.2f}\".format(epoch, np.mead(dur),loss.item(),\n", 225 | " acc, n_edges/np.mean(dur)/1000))\n", 226 | " print()\n", 227 | " acc = evaluate(model, features, labels, test_mask)\n", 228 | " print('Test Accuracy {:.4f}'.format(acc))" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": {}, 235 | "outputs": [], 236 | "source": [ 237 | "if __name__ == '__main__':\n", 238 | " parser = argparse.ArgumentParser(description='GCN')\n", 239 | " register_data_args(parser)\n", 240 | " parser.add_argument(\"--dropout\", type=float, default=0.5,\n", 241 | " help=\"dropout probability\")\n", 242 | " parser.add_argument(\"--gpu\", type=int, default=-1,\n", 243 | " help=\"gpu\")\n", 244 | " parser.add_argument(\"--lr\", type=float, default=1e-2,\n", 245 | " help=\"learning rate\")\n", 246 | " parser.add_argument(\"--n-epochs\", type=int, default=200,\n", 247 | " help=\"number of training epochs\")\n", 248 | " parser.add_argument(\"--n-hidden\", type=int, default=16,\n", 249 | " help=\"number of hidden gcn units\")\n", 250 | " parser.add_argument(\"--n-layers\", type=int, default=1,\n", 251 | " help=\"number of hidden gcn layers\")\n", 252 | " parser.add_argument(\"--weight-decay\", type=float, default=5e-4,\n", 253 | " help=\"Weight for L2 loss\")\n", 254 | " args = parser.parse_args()\n", 255 | " print(args)\n", 256 | "\n", 257 | " main(args)\n" 258 | ] 259 | }, 260 | { 261 | "cell_type": "code", 262 | "execution_count": 12, 263 | "metadata": {}, 264 | "outputs": [ 265 | { 266 | "name": "stdout", 267 | "output_type": "stream", 268 | "text": [ 269 | "Loading from cache failed, re-processing.\n", 270 | "Finished data loading and preprocessing.\n", 271 | " NumNodes: 2708\n", 272 | " NumEdges: 10556\n", 273 | " NumFeats: 1433\n", 274 | " NumClasses: 7\n", 275 | " NumTrainingSamples: 140\n", 276 | " NumValidationSamples: 500\n", 277 | " NumTestSamples: 1000\n", 278 | "Done saving data into cached files.\n" 279 | ] 280 | }, 281 | { 282 | "data": { 283 | "text/plain": [ 284 | "Graph(num_nodes=2708, num_edges=10556,\n", 285 | " ndata_schemes={'train_mask': Scheme(shape=(), dtype=torch.bool), 'val_mask': Scheme(shape=(), dtype=torch.bool), 'test_mask': Scheme(shape=(), dtype=torch.bool), 'label': Scheme(shape=(), dtype=torch.int64), 'feat': Scheme(shape=(1433,), dtype=torch.float32)}\n", 286 | " edata_schemes={})" 287 | ] 288 | }, 289 | "execution_count": 12, 290 | "metadata": {}, 291 | "output_type": "execute_result" 292 | } 293 | ], 294 | "source": [ 295 | "data = CoraGraphDataset()\n", 296 | "g = data[0]\n", 297 | "g" 298 | ] 299 | }, 300 | { 301 | "cell_type": "code", 302 | "execution_count": 13, 303 | "metadata": {}, 304 | "outputs": [ 305 | { 306 | "data": { 307 | "text/plain": [ 308 | "tensor([[0., 0., 0., ..., 0., 0., 0.],\n", 309 | " [0., 0., 0., ..., 0., 0., 0.],\n", 310 | " [0., 0., 0., ..., 0., 0., 0.],\n", 311 | " ...,\n", 312 | " [0., 0., 0., ..., 0., 0., 0.],\n", 313 | " [0., 0., 0., ..., 0., 0., 0.],\n", 314 | " [0., 0., 0., ..., 0., 0., 0.]])" 315 | ] 316 | }, 317 | "execution_count": 13, 318 | "metadata": {}, 319 | "output_type": "execute_result" 320 | } 321 | ], 322 | "source": [ 323 | "g.ndata['feat']" 324 | ] 325 | }, 326 | { 327 | "cell_type": "code", 328 | "execution_count": 14, 329 | "metadata": {}, 330 | "outputs": [ 331 | { 332 | "data": { 333 | "text/plain": [ 334 | "tensor([3, 4, 4, ..., 3, 3, 3])" 335 | ] 336 | }, 337 | "execution_count": 14, 338 | "metadata": {}, 339 | "output_type": "execute_result" 340 | } 341 | ], 342 | "source": [ 343 | "g.ndata['label']" 344 | ] 345 | }, 346 | { 347 | "cell_type": "code", 348 | "execution_count": 15, 349 | "metadata": {}, 350 | "outputs": [ 351 | { 352 | "data": { 353 | "text/plain": [ 354 | "tensor([ True, True, True, ..., False, False, False])" 355 | ] 356 | }, 357 | "execution_count": 15, 358 | "metadata": {}, 359 | "output_type": "execute_result" 360 | } 361 | ], 362 | "source": [ 363 | "g.ndata['train_mask']" 364 | ] 365 | }, 366 | { 367 | "cell_type": "code", 368 | "execution_count": 16, 369 | "metadata": {}, 370 | "outputs": [ 371 | { 372 | "data": { 373 | "text/plain": [ 374 | "tensor([False, False, False, ..., False, False, False])" 375 | ] 376 | }, 377 | "execution_count": 16, 378 | "metadata": {}, 379 | "output_type": "execute_result" 380 | } 381 | ], 382 | "source": [ 383 | "g.ndata['val_mask']" 384 | ] 385 | }, 386 | { 387 | "cell_type": "code", 388 | "execution_count": 17, 389 | "metadata": {}, 390 | "outputs": [], 391 | "source": [ 392 | "g = dgl.graph((torch.tensor([0, 1, 1]), torch.tensor([1, 1, 0])))\n", 393 | "g.ndata['h'] = torch.ones(2, 1)" 394 | ] 395 | }, 396 | { 397 | "cell_type": "code", 398 | "execution_count": 21, 399 | "metadata": {}, 400 | "outputs": [ 401 | { 402 | "data": { 403 | "text/plain": [ 404 | "Graph(num_nodes=2, num_edges=3,\n", 405 | " ndata_schemes={'h': Scheme(shape=(1,), dtype=torch.float32)}\n", 406 | " edata_schemes={})" 407 | ] 408 | }, 409 | "execution_count": 21, 410 | "metadata": {}, 411 | "output_type": "execute_result" 412 | } 413 | ], 414 | "source": [ 415 | "g" 416 | ] 417 | }, 418 | { 419 | "cell_type": "code", 420 | "execution_count": 22, 421 | "metadata": {}, 422 | "outputs": [ 423 | { 424 | "data": { 425 | "text/plain": [ 426 | "tensor([[1.],\n", 427 | " [1.]])" 428 | ] 429 | }, 430 | "execution_count": 22, 431 | "metadata": {}, 432 | "output_type": "execute_result" 433 | } 434 | ], 435 | "source": [ 436 | "g.ndata['h']" 437 | ] 438 | }, 439 | { 440 | "cell_type": "code", 441 | "execution_count": null, 442 | "metadata": {}, 443 | "outputs": [], 444 | "source": [] 445 | } 446 | ], 447 | "metadata": { 448 | "kernelspec": { 449 | "display_name": "Python 3", 450 | "language": "python", 451 | "name": "python3" 452 | }, 453 | "language_info": { 454 | "codemirror_mode": { 455 | "name": "ipython", 456 | "version": 3 457 | }, 458 | "file_extension": ".py", 459 | "mimetype": "text/x-python", 460 | "name": "python", 461 | "nbconvert_exporter": "python", 462 | "pygments_lexer": "ipython3", 463 | "version": "3.7.6" 464 | } 465 | }, 466 | "nbformat": 4, 467 | "nbformat_minor": 4 468 | } 469 | -------------------------------------------------------------------------------- /dgl_torch_GNN/study_dgl/.ipynb_checkpoints/图生成模型-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /node2vec/.ipynb_checkpoints/learn_networkx-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /node2vec/.ipynb_checkpoints/main-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /node2vec/karate.edgelist: -------------------------------------------------------------------------------- 1 | 1 32 2 | 1 22 3 | 1 20 4 | 1 18 5 | 1 14 6 | 1 13 7 | 1 12 8 | 1 11 9 | 1 9 10 | 1 8 11 | 1 7 12 | 1 6 13 | 1 5 14 | 1 4 15 | 1 3 16 | 1 2 17 | 2 31 18 | 2 22 19 | 2 20 20 | 2 18 21 | 2 14 22 | 2 8 23 | 2 4 24 | 2 3 25 | 3 14 26 | 3 9 27 | 3 10 28 | 3 33 29 | 3 29 30 | 3 28 31 | 3 8 32 | 3 4 33 | 4 14 34 | 4 13 35 | 4 8 36 | 5 11 37 | 5 7 38 | 6 17 39 | 6 11 40 | 6 7 41 | 7 17 42 | 9 34 43 | 9 33 44 | 9 33 45 | 10 34 46 | 14 34 47 | 15 34 48 | 15 33 49 | 16 34 50 | 16 33 51 | 19 34 52 | 19 33 53 | 20 34 54 | 21 34 55 | 21 33 56 | 23 34 57 | 23 33 58 | 24 30 59 | 24 34 60 | 24 33 61 | 24 28 62 | 24 26 63 | 25 32 64 | 25 28 65 | 25 26 66 | 26 32 67 | 27 34 68 | 27 30 69 | 28 34 70 | 29 34 71 | 29 32 72 | 30 34 73 | 30 33 74 | 31 34 75 | 31 33 76 | 32 34 77 | 32 33 78 | 33 34 -------------------------------------------------------------------------------- /tensor_numpy_GCN_NN/.ipynb_checkpoints/gcn-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /tensor_numpy_GCN_NN/.ipynb_checkpoints/numpy_nn-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /tensor_numpy_GCN_NN/.ipynb_checkpoints/torch_tensor_nn-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [], 3 | "metadata": {}, 4 | "nbformat": 4, 5 | "nbformat_minor": 4 6 | } 7 | -------------------------------------------------------------------------------- /tensor_numpy_GCN_NN/gcn.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# GCN " 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import numpy as np" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 3, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "A = np.matrix([\n", 26 | " [0,1,0,0],\n", 27 | " [0,0,1,1],\n", 28 | " [0,1,0,0],\n", 29 | " [1,0,1,0],\n", 30 | "],dtype=float)" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 5, 36 | "metadata": {}, 37 | "outputs": [ 38 | { 39 | "data": { 40 | "text/plain": [ 41 | "matrix([[ 0., 0.],\n", 42 | " [ 1., -1.],\n", 43 | " [ 2., -2.],\n", 44 | " [ 3., -3.]])" 45 | ] 46 | }, 47 | "execution_count": 5, 48 | "metadata": {}, 49 | "output_type": "execute_result" 50 | } 51 | ], 52 | "source": [ 53 | "#接下来，我们需要抽取出特征！基于每个节点的索引为其生成两个整数特征，这简化了本文后面手动验证矩阵运算的过程。\n", 54 | "X = np.matrix([[i,-i] for i in range(A.shape[0])],dtype=float)\n", 55 | "X" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": 6, 61 | "metadata": {}, 62 | "outputs": [ 63 | { 64 | "data": { 65 | "text/plain": [ 66 | "matrix([[ 1., -1.],\n", 67 | " [ 5., -5.],\n", 68 | " [ 1., -1.],\n", 69 | " [ 2., -2.]])" 70 | ] 71 | }, 72 | "execution_count": 6, 73 | "metadata": {}, 74 | "output_type": "execute_result" 75 | } 76 | ], 77 | "source": [ 78 | "#应用传播规则\n", 79 | "# 邻接矩阵A 特征集合为X 应用传播规则后\n", 80 | "A*X # 每个节点的表征(每一行)现在是其邻节点特征的和，换句话说，图卷积层将每个节点表示为其相邻节点的聚合。" 81 | ] 82 | }, 83 | { 84 | "cell_type": "markdown", 85 | "metadata": {}, 86 | "source": [ 87 | "## 这里有个问题\n", 88 | "- 节点的聚合特征不包含他自己的特征，改表征是相邻节点的特征聚合，因此只有具有自环的节点才会在该聚合中包含自己的特征。\n", 89 | "- 度大的节点在其特征表述中具有较大的值，度小的节点将具有较小的值。这可能导致梯度小时和梯度爆炸。(随机梯度下降算法通常被用于信息这类网络，且对每个输入特征规模(或值的范围)都很敏感)" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 7, 95 | "metadata": {}, 96 | "outputs": [ 97 | { 98 | "data": { 99 | "text/plain": [ 100 | "matrix([[1., 0., 0., 0.],\n", 101 | " [0., 1., 0., 0.],\n", 102 | " [0., 0., 1., 0.],\n", 103 | " [0., 0., 0., 1.]])" 104 | ] 105 | }, 106 | "execution_count": 7, 107 | "metadata": {}, 108 | "output_type": "execute_result" 109 | } 110 | ], 111 | "source": [ 112 | "# 增加自环 与单位阵相加\n", 113 | "I = np.matrix(np.eye(A.shape[0]))\n", 114 | "I" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 8, 120 | "metadata": {}, 121 | "outputs": [ 122 | { 123 | "data": { 124 | "text/plain": [ 125 | "matrix([[ 1., -1.],\n", 126 | " [ 6., -6.],\n", 127 | " [ 3., -3.],\n", 128 | " [ 5., -5.]])" 129 | ] 130 | }, 131 | "execution_count": 8, 132 | "metadata": {}, 133 | "output_type": "execute_result" 134 | } 135 | ], 136 | "source": [ 137 | "A_hat = A+I\n", 138 | "A_hat*X #现在每个节点在对相邻节点的特征求和过程中也会囊括自己的特征" 139 | ] 140 | }, 141 | { 142 | "cell_type": "markdown", 143 | "metadata": {}, 144 | "source": [ 145 | "## 对特征表征进行归一化处理\n", 146 | "通过将邻接矩阵与度矩阵D的逆相乘，对其进行变换，从而通过节点的度对特征表示征进行归一化\n", 147 | "$ f(X,A)= D^{-1}AX$" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 9, 153 | "metadata": {}, 154 | "outputs": [ 155 | { 156 | "data": { 157 | "text/plain": [ 158 | "matrix([[1., 0., 0., 0.],\n", 159 | " [0., 2., 0., 0.],\n", 160 | " [0., 0., 2., 0.],\n", 161 | " [0., 0., 0., 1.]])" 162 | ] 163 | }, 164 | "execution_count": 9, 165 | "metadata": {}, 166 | "output_type": "execute_result" 167 | } 168 | ], 169 | "source": [ 170 | "D = np.array(np.sum(A,axis=0))[0]\n", 171 | "D = np.matrix(np.diag(D))\n", 172 | "D" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 10, 178 | "metadata": {}, 179 | "outputs": [ 180 | { 181 | "data": { 182 | "text/plain": [ 183 | "matrix([[0. , 1. , 0. , 0. ],\n", 184 | " [0. , 0. , 0.5, 0.5],\n", 185 | " [0. , 0.5, 0. , 0. ],\n", 186 | " [1. , 0. , 1. , 0. ]])" 187 | ] 188 | }, 189 | "execution_count": 10, 190 | "metadata": {}, 191 | "output_type": "execute_result" 192 | } 193 | ], 194 | "source": [ 195 | "D**-1*A" 196 | ] 197 | }, 198 | { 199 | "cell_type": "code", 200 | "execution_count": 11, 201 | "metadata": {}, 202 | "outputs": [ 203 | { 204 | "data": { 205 | "text/plain": [ 206 | "matrix([[ 1. , -1. ],\n", 207 | " [ 2.5, -2.5],\n", 208 | " [ 0.5, -0.5],\n", 209 | " [ 2. , -2. ]])" 210 | ] 211 | }, 212 | "execution_count": 11, 213 | "metadata": {}, 214 | "output_type": "execute_result" 215 | } 216 | ], 217 | "source": [ 218 | "D**-1*A*X" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": 13, 224 | "metadata": {}, 225 | "outputs": [ 226 | { 227 | "data": { 228 | "text/plain": [ 229 | "matrix([[2., 0., 0., 0.],\n", 230 | " [0., 3., 0., 0.],\n", 231 | " [0., 0., 3., 0.],\n", 232 | " [0., 0., 0., 2.]])" 233 | ] 234 | }, 235 | "execution_count": 13, 236 | "metadata": {}, 237 | "output_type": "execute_result" 238 | } 239 | ], 240 | "source": [ 241 | "W = np.matrix([\n", 242 | " [1,-1],\n", 243 | " [-1,1]\n", 244 | "])\n", 245 | "\n", 246 | "D_hat= D+I\n", 247 | "D_hat" 248 | ] 249 | }, 250 | { 251 | "cell_type": "code", 252 | "execution_count": 14, 253 | "metadata": {}, 254 | "outputs": [ 255 | { 256 | "data": { 257 | "text/plain": [ 258 | "matrix([[ 1., -1.],\n", 259 | " [ 4., -4.],\n", 260 | " [ 2., -2.],\n", 261 | " [ 5., -5.]])" 262 | ] 263 | }, 264 | "execution_count": 14, 265 | "metadata": {}, 266 | "output_type": "execute_result" 267 | } 268 | ], 269 | "source": [ 270 | "D_hat**-1 *A_hat*X*W" 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "execution_count": 15, 276 | "metadata": {}, 277 | "outputs": [ 278 | { 279 | "data": { 280 | "text/plain": [ 281 | "matrix([[1.],\n", 282 | " [4.],\n", 283 | " [2.],\n", 284 | " [5.]])" 285 | ] 286 | }, 287 | "execution_count": 15, 288 | "metadata": {}, 289 | "output_type": "execute_result" 290 | } 291 | ], 292 | "source": [ 293 | "# 如果想减小输出特征表征维度，可以减小权重矩阵W的规模\n", 294 | "W = np.matrix([\n", 295 | " [1],\n", 296 | " [-1]\n", 297 | "])\n", 298 | "D_hat**-1 *A_hat*X*W" 299 | ] 300 | }, 301 | { 302 | "cell_type": "code", 303 | "execution_count": 17, 304 | "metadata": {}, 305 | "outputs": [], 306 | "source": [ 307 | "W = np.matrix([\n", 308 | " [1,-1],\n", 309 | " [-1,1]\n", 310 | "])\n", 311 | "H = D_hat**-1 *A_hat*X*W" 312 | ] 313 | }, 314 | { 315 | "cell_type": "code", 316 | "execution_count": 18, 317 | "metadata": {}, 318 | "outputs": [ 319 | { 320 | "data": { 321 | "text/plain": [ 322 | "matrix([[1, 0],\n", 323 | " [1, 0],\n", 324 | " [1, 0],\n", 325 | " [1, 0]])" 326 | ] 327 | }, 328 | "execution_count": 18, 329 | "metadata": {}, 330 | "output_type": "execute_result" 331 | } 332 | ], 333 | "source": [ 334 | "(H > 0) * 1" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": null, 340 | "metadata": {}, 341 | "outputs": [], 342 | "source": [] 343 | } 344 | ], 345 | "metadata": { 346 | "kernelspec": { 347 | "display_name": "Python 3", 348 | "language": "python", 349 | "name": "python3" 350 | }, 351 | "language_info": { 352 | "codemirror_mode": { 353 | "name": "ipython", 354 | "version": 3 355 | }, 356 | "file_extension": ".py", 357 | "mimetype": "text/x-python", 358 | "name": "python", 359 | "nbconvert_exporter": "python", 360 | "pygments_lexer": "ipython3", 361 | "version": "3.7.6" 362 | } 363 | }, 364 | "nbformat": 4, 365 | "nbformat_minor": 4 366 | } 367 | -------------------------------------------------------------------------------- /tulip.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Coding-Zuo/DL-paper/3015361b2ed710b4b49811ea0a0bb535426d9b09/tulip.jpg --------------------------------------------------------------------------------