├── README.md ├── ImagenetDensenet3.ipynb ├── .ipynb_checkpoints ├── ImagenetDensenet3-checkpoint.ipynb ├── ImagenetDensenet2-checkpoint.ipynb └── ImagenetDensenet1-checkpoint.ipynb ├── ImagenetDensenet2.ipynb └── ImagenetDensenet1.ipynb /README.md: -------------------------------------------------------------------------------- 1 | # TinyImagenet-EIP3 2 | 3 | Tried resnet50, heavily overfitted 4 | 5 | Tried resnet18, better than resnet50 but still overfitting 6 | 7 | Increased image size to 112x112 since receptive field of resnet18 is high, got max 56% validation accuracy 8 | 9 | Scrapped resnet and created custom densenet architecure and reached max of 60.51 accuracy after training on 32x32 20x20 and 64x64 10 | 11 | To-do :- Class-based training based on which classes are underperforming 12 | 13 | Key Helper in accuracy jump :- Lr tuning 14 | -------------------------------------------------------------------------------- /ImagenetDensenet3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 0, 6 | "metadata": { 7 | "colab": { 8 | "base_uri": "https://localhost:8080/", 9 | "height": 136 10 | }, 11 | "colab_type": "code", 12 | "id": "NHAZPhA39Tli", 13 | "outputId": "b8ff0352-fdc8-4732-f39f-bf9c4d7552e8" 14 | }, 15 | "outputs": [ 16 | { 17 | "name": "stdout", 18 | "output_type": "stream", 19 | "text": [ 20 | "Epoch 38/38\n", 21 | "4166/4166 [==============================] - 9052s 2s/step - loss: 0.9444 - acc: 0.8071 - val_loss: 1.6717 - val_acc: 0.6051\n", 22 | "Current learning rate 4e-05 84949\n", 23 | "Saving to id 12O9biUN8KrdZybj_0ZU6p-Rad8yznYBw\n", 24 | "\n", 25 | "Epoch 00038: saving model to Imagenetvdbig.hdf5\n" 26 | ] 27 | }, 28 | { 29 | "data": { 30 | "text/plain": [ 31 | "" 32 | ] 33 | }, 34 | "execution_count": 18, 35 | "metadata": { 36 | "tags": [] 37 | }, 38 | "output_type": "execute_result" 39 | } 40 | ], 41 | "source": [ 42 | "#Moderate augmentation with learning rate reduced to 4x10^-6 on 64x64\n", 43 | "model.fit_generator(initial_epoch=37, \n", 44 | " generator=train_generator,\n", 45 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 46 | " validation_data=val_generator,\n", 47 | " validation_steps=STEP_SIZE_VALID,\n", 48 | " epochs=38, verbose=1,\n", 49 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 50 | ] 51 | } 52 | ], 53 | "metadata": { 54 | "accelerator": "TPU", 55 | "colab": { 56 | "name": "Big2.ipynb", 57 | "provenance": [], 58 | "version": "0.3.2" 59 | }, 60 | "kernelspec": { 61 | "display_name": "Python 3", 62 | "language": "python", 63 | "name": "python3" 64 | }, 65 | "language_info": { 66 | "codemirror_mode": { 67 | "name": "ipython", 68 | "version": 3 69 | }, 70 | "file_extension": ".py", 71 | "mimetype": "text/x-python", 72 | "name": "python", 73 | "nbconvert_exporter": "python", 74 | "pygments_lexer": "ipython3", 75 | "version": "3.7.1" 76 | } 77 | }, 78 | "nbformat": 4, 79 | "nbformat_minor": 1 80 | } 81 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/ImagenetDensenet3-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 0, 6 | "metadata": { 7 | "colab": { 8 | "base_uri": "https://localhost:8080/", 9 | "height": 136 10 | }, 11 | "colab_type": "code", 12 | "id": "NHAZPhA39Tli", 13 | "outputId": "b8ff0352-fdc8-4732-f39f-bf9c4d7552e8" 14 | }, 15 | "outputs": [ 16 | { 17 | "name": "stdout", 18 | "output_type": "stream", 19 | "text": [ 20 | "Epoch 38/38\n", 21 | "4166/4166 [==============================] - 9052s 2s/step - loss: 0.9444 - acc: 0.8071 - val_loss: 1.6717 - val_acc: 0.6051\n", 22 | "Current learning rate 4e-05 84949\n", 23 | "Saving to id 12O9biUN8KrdZybj_0ZU6p-Rad8yznYBw\n", 24 | "\n", 25 | "Epoch 00038: saving model to Imagenetvdbig.hdf5\n" 26 | ] 27 | }, 28 | { 29 | "data": { 30 | "text/plain": [ 31 | "" 32 | ] 33 | }, 34 | "execution_count": 18, 35 | "metadata": { 36 | "tags": [] 37 | }, 38 | "output_type": "execute_result" 39 | } 40 | ], 41 | "source": [ 42 | "#Moderate augmentation with learning rate reduced to 4x10^-6 on 64x64\n", 43 | "model.fit_generator(initial_epoch=37, \n", 44 | " generator=train_generator,\n", 45 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 46 | " validation_data=val_generator,\n", 47 | " validation_steps=STEP_SIZE_VALID,\n", 48 | " epochs=38, verbose=1,\n", 49 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 50 | ] 51 | } 52 | ], 53 | "metadata": { 54 | "accelerator": "TPU", 55 | "colab": { 56 | "name": "Big2.ipynb", 57 | "provenance": [], 58 | "version": "0.3.2" 59 | }, 60 | "kernelspec": { 61 | "display_name": "Python 3", 62 | "language": "python", 63 | "name": "python3" 64 | }, 65 | "language_info": { 66 | "codemirror_mode": { 67 | "name": "ipython", 68 | "version": 3 69 | }, 70 | "file_extension": ".py", 71 | "mimetype": "text/x-python", 72 | "name": "python", 73 | "nbconvert_exporter": "python", 74 | "pygments_lexer": "ipython3", 75 | "version": "3.7.1" 76 | } 77 | }, 78 | "nbformat": 4, 79 | "nbformat_minor": 1 80 | } 81 | -------------------------------------------------------------------------------- /ImagenetDensenet2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 0, 6 | "metadata": { 7 | "colab": { 8 | "base_uri": "https://localhost:8080/", 9 | "height": 136 10 | }, 11 | "colab_type": "code", 12 | "id": "pndS1tLQsWBu", 13 | "outputId": "442da859-e9c6-4bb2-83d5-7002d3291678" 14 | }, 15 | "outputs": [ 16 | { 17 | "name": "stdout", 18 | "output_type": "stream", 19 | "text": [ 20 | "Epoch 30/50\n", 21 | "4166/4166 [==============================] - 8900s 2s/step - loss: 2.0637 - acc: 0.5313 - val_loss: 2.0083 - val_acc: 0.5229\n", 22 | "Current learning rate 0.001 51523\n", 23 | "\n", 24 | "Epoch 00030: saving model to Imagenetvdbig.hdf5\n", 25 | "Epoch 31/50\n", 26 | " 3/4166 [..............................] - ETA: 2:26:33 - loss: 1.8809 - acc: 0.6250" 27 | ] 28 | } 29 | ], 30 | "source": [ 31 | "#With moderate augmentation and 64x64\n", 32 | "model.fit_generator(initial_epoch=29, \n", 33 | " generator=train_generator,\n", 34 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 35 | " validation_data=val_generator,\n", 36 | " validation_steps=STEP_SIZE_VALID,\n", 37 | " epochs=NUM_EPOCHS, verbose=1,\n", 38 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 0, 44 | "metadata": { 45 | "colab": { 46 | "base_uri": "https://localhost:8080/", 47 | "height": 34 48 | }, 49 | "colab_type": "code", 50 | "id": "bXr2bEbabdlA", 51 | "outputId": "1a313ac8-dc1f-46f0-90a0-89b520a160aa" 52 | }, 53 | "outputs": [ 54 | { 55 | "data": { 56 | "text/plain": [ 57 | "0.0002" 58 | ] 59 | }, 60 | "execution_count": 10, 61 | "metadata": { 62 | "tags": [] 63 | }, 64 | "output_type": "execute_result" 65 | } 66 | ], 67 | "source": [ 68 | "k.set_value(model.optimizer.lr, 0.0002)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 0, 74 | "metadata": { 75 | "colab": { 76 | "base_uri": "https://localhost:8080/", 77 | "height": 255 78 | }, 79 | "colab_type": "code", 80 | "id": "N5pQoopOtnHr", 81 | "outputId": "9708bb04-0811-4b26-85a8-b34e038e70e8" 82 | }, 83 | "outputs": [ 84 | { 85 | "name": "stdout", 86 | "output_type": "stream", 87 | "text": [ 88 | "Epoch 32/50\n", 89 | "4166/4166 [==============================] - 8921s 2s/step - loss: 2.5294 - acc: 0.4344 - val_loss: 1.8326 - val_acc: 0.5652\n", 90 | "Current learning rate 0.0002 59953\n", 91 | "Saving to id 1wBERqhpP07YuWlpkpYAMQTi5GMSRRkIu\n", 92 | "\n", 93 | "Epoch 00032: saving model to Imagenetvdbig.hdf5\n", 94 | "Epoch 33/50\n", 95 | "4166/4166 [==============================] - 8916s 2s/step - loss: 2.3288 - acc: 0.4751 - val_loss: 1.8353 - val_acc: 0.5692\n", 96 | "Current learning rate 0.0002 64119\n", 97 | "Saving to id 1wrrQOECuO3IM0KySYpqye70UBqOfffDL\n", 98 | "\n", 99 | "Epoch 00033: saving model to Imagenetvdbig.hdf5\n", 100 | "Epoch 34/50\n", 101 | "2885/4166 [===================>..........] - ETA: 44:28 - loss: 2.3016 - acc: 0.4789" 102 | ] 103 | } 104 | ], 105 | "source": [ 106 | "#With heavy augmentation and reduced learning rate\n", 107 | "model.fit_generator(initial_epoch=31, \n", 108 | " generator=train_generator,\n", 109 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 110 | " validation_data=val_generator,\n", 111 | " validation_steps=STEP_SIZE_VALID,\n", 112 | " epochs=NUM_EPOCHS, verbose=1,\n", 113 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 0, 119 | "metadata": { 120 | "colab": { 121 | "base_uri": "https://localhost:8080/", 122 | "height": 202 123 | }, 124 | "colab_type": "code", 125 | "id": "OaYppiN4Ltc6", 126 | "outputId": "08a41199-fe9c-4e64-b228-c66e7806c662" 127 | }, 128 | "outputs": [ 129 | { 130 | "name": "stdout", 131 | "output_type": "stream", 132 | "text": [ 133 | "Epoch 34/35\n", 134 | "4166/4166 [==============================] - 8740s 2s/step - loss: 1.2309 - acc: 0.7307 - val_loss: 1.6902 - val_acc: 0.5957\n", 135 | "Current learning rate 0.0002 68285\n", 136 | "\n", 137 | "Epoch 00034: saving model to Imagenetvdbig.hdf5\n", 138 | "Epoch 35/35\n", 139 | "4166/4166 [==============================] - 8684s 2s/step - loss: 1.0157 - acc: 0.8002 - val_loss: 1.7079 - val_acc: 0.5998\n", 140 | "Current learning rate 0.0002 72451\n", 141 | "\n", 142 | "Epoch 00035: saving model to Imagenetvdbig.hdf5\n" 143 | ] 144 | }, 145 | { 146 | "data": { 147 | "text/plain": [ 148 | "" 149 | ] 150 | }, 151 | "execution_count": 16, 152 | "metadata": { 153 | "tags": [] 154 | }, 155 | "output_type": "execute_result" 156 | } 157 | ], 158 | "source": [ 159 | "#Removed augmentation\n", 160 | "model.fit_generator(initial_epoch=33, \n", 161 | " generator=train_generator,\n", 162 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 163 | " validation_data=val_generator,\n", 164 | " validation_steps=STEP_SIZE_VALID,\n", 165 | " epochs=35, verbose=1,\n", 166 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 0, 172 | "metadata": { 173 | "colab": {}, 174 | "colab_type": "code", 175 | "id": "t8IJE9-R7Fao" 176 | }, 177 | "outputs": [], 178 | "source": [] 179 | } 180 | ], 181 | "metadata": { 182 | "accelerator": "TPU", 183 | "colab": { 184 | "name": "Big.ipynb", 185 | "provenance": [], 186 | "version": "0.3.2" 187 | }, 188 | "kernelspec": { 189 | "display_name": "Python 3", 190 | "language": "python", 191 | "name": "python3" 192 | }, 193 | "language_info": { 194 | "codemirror_mode": { 195 | "name": "ipython", 196 | "version": 3 197 | }, 198 | "file_extension": ".py", 199 | "mimetype": "text/x-python", 200 | "name": "python", 201 | "nbconvert_exporter": "python", 202 | "pygments_lexer": "ipython3", 203 | "version": "3.7.1" 204 | } 205 | }, 206 | "nbformat": 4, 207 | "nbformat_minor": 1 208 | } 209 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/ImagenetDensenet2-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 0, 6 | "metadata": { 7 | "colab": { 8 | "base_uri": "https://localhost:8080/", 9 | "height": 136 10 | }, 11 | "colab_type": "code", 12 | "id": "pndS1tLQsWBu", 13 | "outputId": "442da859-e9c6-4bb2-83d5-7002d3291678" 14 | }, 15 | "outputs": [ 16 | { 17 | "name": "stdout", 18 | "output_type": "stream", 19 | "text": [ 20 | "Epoch 30/50\n", 21 | "4166/4166 [==============================] - 8900s 2s/step - loss: 2.0637 - acc: 0.5313 - val_loss: 2.0083 - val_acc: 0.5229\n", 22 | "Current learning rate 0.001 51523\n", 23 | "\n", 24 | "Epoch 00030: saving model to Imagenetvdbig.hdf5\n", 25 | "Epoch 31/50\n", 26 | " 3/4166 [..............................] - ETA: 2:26:33 - loss: 1.8809 - acc: 0.6250" 27 | ] 28 | } 29 | ], 30 | "source": [ 31 | "#With moderate augmentation and 64x64\n", 32 | "model.fit_generator(initial_epoch=29, \n", 33 | " generator=train_generator,\n", 34 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 35 | " validation_data=val_generator,\n", 36 | " validation_steps=STEP_SIZE_VALID,\n", 37 | " epochs=NUM_EPOCHS, verbose=1,\n", 38 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 0, 44 | "metadata": { 45 | "colab": { 46 | "base_uri": "https://localhost:8080/", 47 | "height": 34 48 | }, 49 | "colab_type": "code", 50 | "id": "bXr2bEbabdlA", 51 | "outputId": "1a313ac8-dc1f-46f0-90a0-89b520a160aa" 52 | }, 53 | "outputs": [ 54 | { 55 | "data": { 56 | "text/plain": [ 57 | "0.0002" 58 | ] 59 | }, 60 | "execution_count": 10, 61 | "metadata": { 62 | "tags": [] 63 | }, 64 | "output_type": "execute_result" 65 | } 66 | ], 67 | "source": [ 68 | "k.set_value(model.optimizer.lr, 0.0002)" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 0, 74 | "metadata": { 75 | "colab": { 76 | "base_uri": "https://localhost:8080/", 77 | "height": 255 78 | }, 79 | "colab_type": "code", 80 | "id": "N5pQoopOtnHr", 81 | "outputId": "9708bb04-0811-4b26-85a8-b34e038e70e8" 82 | }, 83 | "outputs": [ 84 | { 85 | "name": "stdout", 86 | "output_type": "stream", 87 | "text": [ 88 | "Epoch 32/50\n", 89 | "4166/4166 [==============================] - 8921s 2s/step - loss: 2.5294 - acc: 0.4344 - val_loss: 1.8326 - val_acc: 0.5652\n", 90 | "Current learning rate 0.0002 59953\n", 91 | "Saving to id 1wBERqhpP07YuWlpkpYAMQTi5GMSRRkIu\n", 92 | "\n", 93 | "Epoch 00032: saving model to Imagenetvdbig.hdf5\n", 94 | "Epoch 33/50\n", 95 | "4166/4166 [==============================] - 8916s 2s/step - loss: 2.3288 - acc: 0.4751 - val_loss: 1.8353 - val_acc: 0.5692\n", 96 | "Current learning rate 0.0002 64119\n", 97 | "Saving to id 1wrrQOECuO3IM0KySYpqye70UBqOfffDL\n", 98 | "\n", 99 | "Epoch 00033: saving model to Imagenetvdbig.hdf5\n", 100 | "Epoch 34/50\n", 101 | "2885/4166 [===================>..........] - ETA: 44:28 - loss: 2.3016 - acc: 0.4789" 102 | ] 103 | } 104 | ], 105 | "source": [ 106 | "#With heavy augmentation and reduced learning rate\n", 107 | "model.fit_generator(initial_epoch=31, \n", 108 | " generator=train_generator,\n", 109 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 110 | " validation_data=val_generator,\n", 111 | " validation_steps=STEP_SIZE_VALID,\n", 112 | " epochs=NUM_EPOCHS, verbose=1,\n", 113 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 0, 119 | "metadata": { 120 | "colab": { 121 | "base_uri": "https://localhost:8080/", 122 | "height": 202 123 | }, 124 | "colab_type": "code", 125 | "id": "OaYppiN4Ltc6", 126 | "outputId": "08a41199-fe9c-4e64-b228-c66e7806c662" 127 | }, 128 | "outputs": [ 129 | { 130 | "name": "stdout", 131 | "output_type": "stream", 132 | "text": [ 133 | "Epoch 34/35\n", 134 | "4166/4166 [==============================] - 8740s 2s/step - loss: 1.2309 - acc: 0.7307 - val_loss: 1.6902 - val_acc: 0.5957\n", 135 | "Current learning rate 0.0002 68285\n", 136 | "\n", 137 | "Epoch 00034: saving model to Imagenetvdbig.hdf5\n", 138 | "Epoch 35/35\n", 139 | "4166/4166 [==============================] - 8684s 2s/step - loss: 1.0157 - acc: 0.8002 - val_loss: 1.7079 - val_acc: 0.5998\n", 140 | "Current learning rate 0.0002 72451\n", 141 | "\n", 142 | "Epoch 00035: saving model to Imagenetvdbig.hdf5\n" 143 | ] 144 | }, 145 | { 146 | "data": { 147 | "text/plain": [ 148 | "" 149 | ] 150 | }, 151 | "execution_count": 16, 152 | "metadata": { 153 | "tags": [] 154 | }, 155 | "output_type": "execute_result" 156 | } 157 | ], 158 | "source": [ 159 | "#Removed augmentation\n", 160 | "model.fit_generator(initial_epoch=33, \n", 161 | " generator=train_generator,\n", 162 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 163 | " validation_data=val_generator,\n", 164 | " validation_steps=STEP_SIZE_VALID,\n", 165 | " epochs=35, verbose=1,\n", 166 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 0, 172 | "metadata": { 173 | "colab": {}, 174 | "colab_type": "code", 175 | "id": "t8IJE9-R7Fao" 176 | }, 177 | "outputs": [], 178 | "source": [] 179 | } 180 | ], 181 | "metadata": { 182 | "accelerator": "TPU", 183 | "colab": { 184 | "name": "Big.ipynb", 185 | "provenance": [], 186 | "version": "0.3.2" 187 | }, 188 | "kernelspec": { 189 | "display_name": "Python 3", 190 | "language": "python", 191 | "name": "python3" 192 | }, 193 | "language_info": { 194 | "codemirror_mode": { 195 | "name": "ipython", 196 | "version": 3 197 | }, 198 | "file_extension": ".py", 199 | "mimetype": "text/x-python", 200 | "name": "python", 201 | "nbconvert_exporter": "python", 202 | "pygments_lexer": "ipython3", 203 | "version": "3.7.1" 204 | } 205 | }, 206 | "nbformat": 4, 207 | "nbformat_minor": 1 208 | } 209 | -------------------------------------------------------------------------------- /ImagenetDensenet1.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 0, 6 | "metadata": { 7 | "colab": { 8 | "base_uri": "https://localhost:8080/", 9 | "height": 513 10 | }, 11 | "colab_type": "code", 12 | "id": "0lhx1XzJ3CIy", 13 | "outputId": "2f24eaba-605f-43f2-ad14-64e17dcfefd4" 14 | }, 15 | "outputs": [ 16 | { 17 | "name": "stdout", 18 | "output_type": "stream", 19 | "text": [ 20 | "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n", 21 | "\n", 22 | "Enter your authorization code:\n", 23 | "··········\n", 24 | "Mounted at /content/drive\n", 25 | "Cloning into 'keras-resnet'...\n", 26 | "remote: Enumerating objects: 126, done.\u001b[K\n", 27 | "remote: Total 126 (delta 0), reused 0 (delta 0), pack-reused 126\u001b[K\n", 28 | "Receiving objects: 100% (126/126), 1.65 MiB | 1.53 MiB/s, done.\n", 29 | "Resolving deltas: 100% (58/58), done.\n", 30 | "Collecting PyDrive\n", 31 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/52/e0/0e64788e5dd58ce2d6934549676243dc69d982f198524be9b99e9c2a4fd5/PyDrive-1.3.1.tar.gz (987kB)\n", 32 | "\u001b[K 100% |████████████████████████████████| 993kB 19.3MB/s \n", 33 | "\u001b[?25hRequirement already satisfied: google-api-python-client>=1.2 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (1.6.7)\n", 34 | "Requirement already satisfied: oauth2client>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (4.1.3)\n", 35 | "Requirement already satisfied: PyYAML>=3.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (3.13)\n", 36 | "Requirement already satisfied: httplib2<1dev,>=0.9.2 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (0.11.3)\n", 37 | "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (3.0.0)\n", 38 | "Requirement already satisfied: six<2dev,>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (1.11.0)\n", 39 | "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.4.5)\n", 40 | "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.2.4)\n", 41 | "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (4.0)\n", 42 | "Building wheels for collected packages: PyDrive\n", 43 | " Building wheel for PyDrive (setup.py) ... \u001b[?25ldone\n", 44 | "\u001b[?25h Stored in directory: /root/.cache/pip/wheels/fa/d2/9a/d3b6b506c2da98289e5d417215ce34b696db856643bad779f4\n", 45 | "Successfully built PyDrive\n", 46 | "Installing collected packages: PyDrive\n", 47 | "Successfully installed PyDrive-1.3.1\n" 48 | ] 49 | } 50 | ], 51 | "source": [ 52 | "from google.colab import drive\n", 53 | "drive.mount('/content/drive', force_remount=True)\n", 54 | "!rm -rf tiny-imagenet-200.zip tiny-imagenet-200\n", 55 | "!git clone https://github.com/raghakot/keras-resnet.git\n", 56 | "!cp drive/'My Drive'/tiny-imagenet-200_flow.zip .\n", 57 | "!unzip -qq tiny-imagenet-200_flow.zip\n", 58 | "!rm -rf tiny-imagenet-200/val/images tiny-imagenet-200/val/val_annotations.txt\n", 59 | "!pip install PyDrive\n", 60 | "from pydrive.auth import GoogleAuth\n", 61 | "from pydrive.drive import GoogleDrive\n", 62 | "from google.colab import auth\n", 63 | "from oauth2client.client import GoogleCredentials\n", 64 | "\n", 65 | "# 1. Authenticate and create the PyDrive client.\n", 66 | "\n", 67 | "import tensorflow as tf\n", 68 | "from keras import backend as k\n", 69 | "config = tf.ConfigProto()\n", 70 | "config.gpu_options.allow_growth = True\n", 71 | "# Create a session with the above options specified.\n", 72 | "k.tensorflow_backend.set_session(tf.Session(config=config))" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 0, 78 | "metadata": { 79 | "colab": {}, 80 | "colab_type": "code", 81 | "id": "0zxrHu6fpFPI" 82 | }, 83 | "outputs": [], 84 | "source": [ 85 | "def drive_authenticate():\n", 86 | " auth.authenticate_user()\n", 87 | " gauth = GoogleAuth()\n", 88 | " gauth.credentials = GoogleCredentials.get_application_default()\n", 89 | " drive = GoogleDrive(gauth)\n", 90 | " return drive" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 0, 96 | "metadata": { 97 | "colab": {}, 98 | "colab_type": "code", 99 | "id": "HYefiC_L3LYf" 100 | }, 101 | "outputs": [], 102 | "source": [ 103 | "IMAGE_SIZE, TRAINING_IMAGES_DIR, VAL_IMAGES_DIR = None, None, None\n", 104 | "model_file=\"Imagenetvd.hdf5\" \n", 105 | "def set_image_size(image_size):\n", 106 | " global IMAGE_SIZE, TRAINING_IMAGES_DIR, VAL_IMAGES_DIR\n", 107 | " IMAGE_SIZE = image_size\n", 108 | " if IMAGE_SIZE == 32:\n", 109 | " TRAINING_IMAGES_DIR = 'tiny-imagenet-200-32/train/'\n", 110 | " VAL_IMAGES_DIR = 'tiny-imagenet-200-32/val/'\n", 111 | " else:\n", 112 | " TRAINING_IMAGES_DIR = 'tiny-imagenet-200/train/'\n", 113 | " VAL_IMAGES_DIR = 'tiny-imagenet-200/val/'\n", 114 | "set_image_size(64) " 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 0, 120 | "metadata": { 121 | "colab": { 122 | "base_uri": "https://localhost:8080/", 123 | "height": 51 124 | }, 125 | "colab_type": "code", 126 | "id": "uY7EnC8b3otl", 127 | "outputId": "eb640b9a-a3ee-43ce-e46d-5aa545cf7f6b" 128 | }, 129 | "outputs": [ 130 | { 131 | "name": "stdout", 132 | "output_type": "stream", 133 | "text": [ 134 | "Found 100000 images belonging to 200 classes.\n", 135 | "Found 10000 images belonging to 200 classes.\n" 136 | ] 137 | } 138 | ], 139 | "source": [ 140 | "import shutil\n", 141 | "import numpy as np\n", 142 | "import pandas as pd\n", 143 | "import os\n", 144 | "from sklearn import preprocessing\n", 145 | "import matplotlib.image as mpimg\n", 146 | "from keras.preprocessing.image import ImageDataGenerator\n", 147 | "from keras import backend as k\n", 148 | "from keras.utils import np_utils\n", 149 | "from keras.callbacks import ReduceLROnPlateau, CSVLogger, EarlyStopping, Callback, ModelCheckpoint\n", 150 | "from keras.utils import Sequence\n", 151 | "from keras.preprocessing.image import load_img\n", 152 | "from keras.preprocessing.image import img_to_array\n", 153 | "import imgaug as ia\n", 154 | "from imgaug import augmenters as iaa\n", 155 | "from keras.layers import Concatenate\n", 156 | "\n", 157 | "BATCH_SIZE = 32\n", 158 | "NUM_CLASSES = 200\n", 159 | "NUM_IMAGES_PER_CLASS = 500\n", 160 | "NUM_IMAGES = NUM_CLASSES * NUM_IMAGES_PER_CLASS\n", 161 | "TRAIN_SIZE = NUM_IMAGES\n", 162 | "NUM_CHANNELS = 3\n", 163 | "NUM_VAL_IMAGES = 10000\n", 164 | "NUM_EPOCHS = 50\n", 165 | "IMAGE_SIZE_T = 48\n", 166 | "\n", 167 | "ia.seed(1)\n", 168 | "sometimes = lambda aug: iaa.Sometimes(0.5, aug)\n", 169 | "\n", 170 | "# Define our sequence of augmentation steps that will be applied to every image.\n", 171 | "\n", 172 | "seq20 = iaa.Sequential(\n", 173 | " [\n", 174 | " #\n", 175 | " # Apply the following augmenters to most images.\n", 176 | " #\n", 177 | " iaa.Fliplr(0.5), # horizontally flip 50% of all images\n", 178 | "\n", 179 | " # crop some of the images by 0-10% of their height/width\n", 180 | " sometimes(iaa.Crop(percent=(0, 0.1))),\n", 181 | "\n", 182 | " # Apply affine transformations to some of the images\n", 183 | " # - scale to 80-120% of image height/width (each axis independently)\n", 184 | " # - translate by -20 to +20 relative to height/width (per axis)\n", 185 | " # - rotate by -45 to +45 degrees\n", 186 | " # - shear by -16 to +16 degrees\n", 187 | " # - order: use nearest neighbour or bilinear interpolation (fast)\n", 188 | " # - mode: use any available mode to fill newly created pixels\n", 189 | " # see API or scikit-image for which modes are available\n", 190 | " # - cval: if the mode is constant, then use a random brightness\n", 191 | " # for the newly created pixels (e.g. sometimes black,\n", 192 | " # sometimes white)\n", 193 | " sometimes(iaa.Affine(\n", 194 | " scale={\"x\": (0.9, 1.1), \"y\": (0.9, 1.1)},\n", 195 | " translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n", 196 | " rotate=(-10, 10),\n", 197 | " shear=(-4, 4),\n", 198 | " order=[0, 1],\n", 199 | " cval=(0, 255),\n", 200 | " mode=ia.ALL\n", 201 | " )),\n", 202 | "\n", 203 | " #\n", 204 | " # Execute 0 to 5 of the following (less important) augmenters per\n", 205 | " # image. Don't execute all of them, as that would often be way too\n", 206 | " # strong.\n", 207 | " #\n", 208 | " iaa.SomeOf((0, 2),\n", 209 | " [\n", 210 | " iaa.OneOf([\n", 211 | " iaa.GaussianBlur((0, 3.0)),\n", 212 | " iaa.AverageBlur(k=(2, 7)),\n", 213 | " ]),\n", 214 | " iaa.AdditiveGaussianNoise(\n", 215 | " loc=0, scale=(0.0, 0.05*255), per_channel=0.5\n", 216 | " ),\n", 217 | " iaa.OneOf([\n", 218 | " iaa.Dropout((0.01, 0.1), per_channel=0.5),\n", 219 | " iaa.CoarseDropout(\n", 220 | " (0.03, 0.15), size_percent=(0.02, 0.05),\n", 221 | " per_channel=0.2\n", 222 | " ),\n", 223 | " ]),\n", 224 | " sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) ],\n", 225 | " # do all of the above augmentations in random order\n", 226 | " random_order=True\n", 227 | " )\n", 228 | " ],\n", 229 | " # do all of the above augmentations in random order\n", 230 | " random_order=True\n", 231 | ")\n", 232 | "\n", 233 | "seq32 = iaa.Sequential(\n", 234 | " [\n", 235 | " #\n", 236 | " # Apply the following augmenters to most images.\n", 237 | " #\n", 238 | " iaa.Fliplr(0.5), # horizontally flip 50% of all images\n", 239 | "\n", 240 | " # crop some of the images by 0-10% of their height/width\n", 241 | " sometimes(iaa.Crop(percent=(0, 0.1))),\n", 242 | "\n", 243 | " # Apply affine transformations to some of the images\n", 244 | " # - scale to 80-120% of image height/width (each axis independently)\n", 245 | " # - translate by -20 to +20 relative to height/width (per axis)\n", 246 | " # - rotate by -45 to +45 degrees\n", 247 | " # - shear by -16 to +16 degrees\n", 248 | " # - order: use nearest neighbour or bilinear interpolation (fast)\n", 249 | " # - mode: use any available mode to fill newly created pixels\n", 250 | " # see API or scikit-image for which modes are available\n", 251 | " # - cval: if the mode is constant, then use a random brightness\n", 252 | " # for the newly created pixels (e.g. sometimes black,\n", 253 | " # sometimes white)\n", 254 | " sometimes(iaa.Affine(\n", 255 | " scale={\"x\": (0.625, 1), \"y\": (0.625, 1)},\n", 256 | " translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n", 257 | " rotate=(-20, 20),\n", 258 | " shear=(-4, 4),\n", 259 | " order=[0, 1],\n", 260 | " cval=(0, 255),\n", 261 | " mode=ia.ALL\n", 262 | " )),\n", 263 | "\n", 264 | " #\n", 265 | " # Execute 0 to 5 of the following (less important) augmenters per\n", 266 | " # image. Don't execute all of them, as that would often be way too\n", 267 | " # strong.\n", 268 | " #\n", 269 | " iaa.SomeOf((0, 4),\n", 270 | " [\n", 271 | " iaa.OneOf([\n", 272 | " iaa.GaussianBlur((0, 3.0)),\n", 273 | " iaa.AverageBlur(k=(2, 7)),\n", 274 | " ]),\n", 275 | " iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),\n", 276 | " iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),\n", 277 | " sometimes(iaa.OneOf([\n", 278 | " iaa.EdgeDetect(alpha=(0, 0.7)),\n", 279 | " iaa.DirectedEdgeDetect(\n", 280 | " alpha=(0, 0.7), direction=(0.0, 1.0)\n", 281 | " ),\n", 282 | " ])),\n", 283 | " iaa.AdditiveGaussianNoise(\n", 284 | " loc=0, scale=(0.0, 0.05*255), per_channel=0.5\n", 285 | " ),\n", 286 | " iaa.OneOf([\n", 287 | " iaa.Dropout((0.01, 0.1), per_channel=0.5),\n", 288 | " iaa.CoarseDropout(\n", 289 | " (0.03, 0.15), size_percent=(0.02, 0.05),\n", 290 | " per_channel=0.2\n", 291 | " ),\n", 292 | " ]),\n", 293 | " iaa.Invert(0.05, per_channel=True), \n", 294 | " iaa.Add((-10, 10), per_channel=0.5),\n", 295 | " iaa.Multiply((0.5, 1.5), per_channel=0.5),\n", 296 | " iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5),\n", 297 | " sometimes(\n", 298 | " iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)\n", 299 | " ),\n", 300 | " sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) ],\n", 301 | " # do all of the above augmentations in random order\n", 302 | " random_order=True\n", 303 | " )\n", 304 | " ],\n", 305 | " # do all of the above augmentations in random order\n", 306 | " random_order=True\n", 307 | ")\n", 308 | "\n", 309 | "\n", 310 | "seq = iaa.Sequential(\n", 311 | " [\n", 312 | " #\n", 313 | " # Apply the following augmenters to most images.\n", 314 | " #\n", 315 | " iaa.Fliplr(0.5), # horizontally flip 50% of all images\n", 316 | "\n", 317 | " # crop some of the images by 0-10% of their height/width\n", 318 | " sometimes(iaa.Crop(percent=(0, 0.1))),\n", 319 | "\n", 320 | " # Apply affine transformations to some of the images\n", 321 | " # - scale to 80-120% of image height/width (each axis independently)\n", 322 | " # - translate by -20 to +20 relative to height/width (per axis)\n", 323 | " # - rotate by -45 to +45 degrees\n", 324 | " # - shear by -16 to +16 degrees\n", 325 | " # - order: use nearest neighbour or bilinear interpolation (fast)\n", 326 | " # - mode: use any available mode to fill newly created pixels\n", 327 | " # see API or scikit-image for which modes are available\n", 328 | " # - cval: if the mode is constant, then use a random brightness\n", 329 | " # for the newly created pixels (e.g. sometimes black,\n", 330 | " # sometimes white)\n", 331 | " sometimes(iaa.Affine(\n", 332 | " scale={\"x\": (0.6, 1), \"y\": (0.6, 1)},\n", 333 | " translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)},\n", 334 | " rotate=(-20, 20),\n", 335 | " shear=(-8, 8),\n", 336 | " order=[0, 1],\n", 337 | " cval=(0, 255),\n", 338 | " mode=ia.ALL\n", 339 | " )),\n", 340 | "\n", 341 | " #\n", 342 | " # Execute 0 to 5 of the following (less important) augmenters per\n", 343 | " # image. Don't execute all of them, as that would often be way too\n", 344 | " # strong.\n", 345 | " #\n", 346 | " iaa.SomeOf((0, 5),\n", 347 | " [\n", 348 | " iaa.OneOf([\n", 349 | " iaa.GaussianBlur((0, 3.0)),\n", 350 | " iaa.AverageBlur(k=(2, 7)),\n", 351 | " ]),\n", 352 | " iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),\n", 353 | " iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),\n", 354 | " sometimes(iaa.OneOf([\n", 355 | " iaa.EdgeDetect(alpha=(0, 0.7)),\n", 356 | " iaa.DirectedEdgeDetect(\n", 357 | " alpha=(0, 0.7), direction=(0.0, 1.0)\n", 358 | " ),\n", 359 | " ])),\n", 360 | " iaa.AdditiveGaussianNoise(\n", 361 | " loc=0, scale=(0.0, 0.05*255), per_channel=0.5\n", 362 | " ),\n", 363 | " iaa.OneOf([\n", 364 | " iaa.Dropout((0.01, 0.1), per_channel=0.5),\n", 365 | " iaa.CoarseDropout(\n", 366 | " (0.03, 0.15), size_percent=(0.02, 0.05),\n", 367 | " per_channel=0.2\n", 368 | " ),\n", 369 | " ]),\n", 370 | " iaa.Invert(0.05, per_channel=True), \n", 371 | " iaa.Add((-10, 10), per_channel=0.5),\n", 372 | " iaa.Multiply((0.5, 1.5), per_channel=0.5),\n", 373 | " iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5),\n", 374 | " sometimes(\n", 375 | " iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)\n", 376 | " ),\n", 377 | " sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) ],\n", 378 | " # do all of the above augmentations in random order\n", 379 | " random_order=True\n", 380 | " )\n", 381 | " ],\n", 382 | " # do all of the above augmentations in random order\n", 383 | " random_order=True\n", 384 | ")\n", 385 | "\n", 386 | "def save_to_drive(drive, epoch, filename): \n", 387 | " title = 'Imagenetvd' + str(epoch) + '.hdf5'\n", 388 | " model_file = drive.CreateFile({'title' : title})\n", 389 | " model_file.SetContentFile(filename)\n", 390 | " model_file.Upload()\n", 391 | " print(\"Saving to id\", model_file.get('id'))\n", 392 | " # download to google drive\n", 393 | " drive.CreateFile({'id': model_file.get('id')}) \n", 394 | " \n", 395 | " \n", 396 | "def get_from_drive(idv):\n", 397 | " global model_file\n", 398 | " gauth = GoogleAuth()\n", 399 | " gauth.credentials = GoogleCredentials.get_application_default()\n", 400 | " drive = GoogleDrive(gauth)\n", 401 | " last_weight_file = drive.CreateFile({'id': idv}) \n", 402 | " last_weight_file.GetContentFile(model_file) \n", 403 | " \n", 404 | "class EarlyStoppingByAccuracy(Callback):\n", 405 | " def __init__(self, monitor='val_acc', mode='max', value=0.98, verbose=0):\n", 406 | " super(Callback, self).__init__()\n", 407 | " self.monitor = monitor\n", 408 | " self.value = value\n", 409 | " self.verbose = verbose\n", 410 | "\n", 411 | " def on_epoch_end(self, epoch, logs={}):\n", 412 | " global model_file\n", 413 | " current = logs.get(self.monitor)\n", 414 | " if current is None:\n", 415 | " warnings.warn(\"Early stopping requires %s available!\" % self.monitor, RuntimeWarning)\n", 416 | " print(\"Current learning rate\", k.get_value(self.model.optimizer.lr), k.get_value(self.model.optimizer.iterations)) \n", 417 | " self.model.save(model_file, overwrite=True)\n", 418 | " epoch = int((k.get_value(self.model.optimizer.iterations))/(int(NUM_IMAGES/BATCH_SIZE)))\n", 419 | " #if(epoch%5==0):\n", 420 | " drive = drive_authenticate()\n", 421 | " save_to_drive(drive, epoch, model_file)\n", 422 | " if current >= self.value:\n", 423 | " if self.verbose > 0:\n", 424 | " print(\"Epoch %05d: early stopping THR\" % epoch)\n", 425 | " self.model.stop_training = True\n", 426 | " \n", 427 | "train_datagen = ImageDataGenerator(\n", 428 | " rescale=1./255,\n", 429 | " featurewise_center=True, featurewise_std_normalization=True,\n", 430 | " preprocessing_function=seq32.augment_image) \n", 431 | "train_datagen.mean = np.array([0.480337294172018, 0.44825522891986064, 0.39783786137898763], dtype=np.float32).reshape((1,1,3))\n", 432 | "train_datagen.std = np.array([0.27676311280482857, 0.26885390556775607, 0.28181860401080205], dtype=np.float32).reshape((1,1,3))\n", 433 | "valid_datagen = ImageDataGenerator(\n", 434 | " rescale=1./255,\n", 435 | " featurewise_center=True, featurewise_std_normalization=True,\n", 436 | " shear_range = 0, # random application of shearing\n", 437 | " zoom_range = 0,\n", 438 | " horizontal_flip = False) \n", 439 | "valid_datagen.mean = np.array([0.480337294172018, 0.44825522891986064, 0.39783786137898763], dtype=np.float32).reshape((1,1,3))\n", 440 | "valid_datagen.std = np.array([0.27676311280482857, 0.26885390556775607, 0.28181860401080205], dtype=np.float32).reshape((1,1,3))\n", 441 | "train_generator = train_datagen.flow_from_directory(\n", 442 | " TRAINING_IMAGES_DIR,\n", 443 | " batch_size=BATCH_SIZE,\n", 444 | " target_size=(IMAGE_SIZE_T, IMAGE_SIZE_T),\n", 445 | " color_mode=\"rgb\",\n", 446 | " class_mode='categorical',\n", 447 | " shuffle=True,\n", 448 | " seed=42)\n", 449 | "STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size\n", 450 | "val_generator = valid_datagen.flow_from_directory(\n", 451 | " directory=VAL_IMAGES_DIR,\n", 452 | " target_size=(IMAGE_SIZE_T, IMAGE_SIZE_T),\n", 453 | " batch_size=BATCH_SIZE,\n", 454 | " color_mode=\"rgb\",\n", 455 | " class_mode=\"categorical\",\n", 456 | " shuffle=True,\n", 457 | " seed=42\n", 458 | ")\n", 459 | "STEP_SIZE_VALID=val_generator.n//val_generator.batch_size \n", 460 | "\n", 461 | "lr_reducer = ReduceLROnPlateau(monitor='val_acc', factor=0.2,\n", 462 | " cooldown=0, patience=5, min_lr=0.0000001)\n", 463 | "model_checkpoint= ModelCheckpoint(model_file, monitor=\"val_acc\", save_best_only=False,\n", 464 | " verbose=1)\n", 465 | "early_stopping = EarlyStoppingByAccuracy(monitor='val_acc', value=0.70, verbose=1)\n", 466 | "csv_logger = CSVLogger('resnet50_imagenet200.csv') " 467 | ] 468 | }, 469 | { 470 | "cell_type": "code", 471 | "execution_count": 0, 472 | "metadata": { 473 | "colab": {}, 474 | "colab_type": "code", 475 | "id": "dryNSKqP3qRv" 476 | }, 477 | "outputs": [], 478 | "source": [ 479 | "def bn_relu_conv(block, num_filter, conv_size):\n", 480 | " block = BatchNormalization(epsilon=1.1e-5)(block)\n", 481 | " block = Activation('relu')(block)\n", 482 | " block = SeparableConv2D(num_filter, conv_size, use_bias=False, padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(0.0001))(block) \n", 483 | " return block\n", 484 | "\n", 485 | "def add_denseblock(input, num_filter, out_filter):\n", 486 | " temp = input\n", 487 | " for _ in range(3): \n", 488 | " block = bn_relu_conv(temp, num_filter, (3, 3))\n", 489 | " concat = Concatenate(axis=-1)([temp,block])\n", 490 | " temp = concat\n", 491 | " out_filter += num_filter\n", 492 | "\n", 493 | " return temp, out_filter\n", 494 | "\n", 495 | "def add_transition(input, num_filter, pool=False):\n", 496 | " block = bn_relu_conv(input, num_filter, (1, 1))\n", 497 | " if pool:\n", 498 | " block = AveragePooling2D(pool_size=(2,2))(block) \n", 499 | " return block, num_filter\n", 500 | "\n", 501 | "def output_layer(input):\n", 502 | " block = bn_relu_conv(input, NUM_CLASSES, (1, 1))\n", 503 | " block = BatchNormalization()(block)\n", 504 | " block = Activation('relu')(block)\n", 505 | " block = GlobalAveragePooling2D()(block)\n", 506 | " output = Softmax(NUM_CLASSES)(block)\n", 507 | " return output" 508 | ] 509 | }, 510 | { 511 | "cell_type": "code", 512 | "execution_count": 0, 513 | "metadata": { 514 | "colab": {}, 515 | "colab_type": "code", 516 | "id": "oQND6issBo-t" 517 | }, 518 | "outputs": [], 519 | "source": [ 520 | "import tensorflow as tf\n", 521 | "def space_to_depth_2(x):\n", 522 | " return tf.space_to_depth(x, block_size=2)\n", 523 | " \n", 524 | "def space_to_depth_4(x):\n", 525 | " return tf.space_to_depth(x, block_size=4) " 526 | ] 527 | }, 528 | { 529 | "cell_type": "code", 530 | "execution_count": 0, 531 | "metadata": { 532 | "colab": {}, 533 | "colab_type": "code", 534 | "id": "DVtjAdI63sC1" 535 | }, 536 | "outputs": [], 537 | "source": [ 538 | "def densenet(num_filter=32):\n", 539 | " block = Conv2D(int(num_filter/2), (3,3), use_bias=False ,padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(0.0001))(input)\n", 540 | " block1, out1 = add_denseblock(block, num_filter, num_filter*2)\n", 541 | " block1, out1 = add_transition(block1, out1)\n", 542 | " block2, out2 = add_denseblock(block1, num_filter, out1)\n", 543 | " block2, out2 = add_transition(block2, out2)\n", 544 | " block3, out3 = add_denseblock(block2, num_filter, out2)\n", 545 | " block3, out3 = add_transition(block3, out3, pool=True)\n", 546 | " block4, out4 = add_denseblock(block3, num_filter, out3)\n", 547 | " block4, out4 = add_transition(block4, out4, pool=True)\n", 548 | " c1 = bn_relu_conv(block2, 64, (1,1))\n", 549 | " c1 = Lambda(space_to_depth_4)(c1)\n", 550 | " c2 = bn_relu_conv(block3, 256, (1,1))\n", 551 | " c2 = Lambda(space_to_depth_2)(c2)\n", 552 | " block = Concatenate(axis=-1)([c1,c2,block4])\n", 553 | " block = bn_relu_conv(block, NUM_CLASSES*8, (1,1))\n", 554 | " block = bn_relu_conv(block, NUM_CLASSES*4, (3,3)) \n", 555 | " block = bn_relu_conv(block, NUM_CLASSES*2, (3,3)) \n", 556 | " output = output_layer(block) \n", 557 | " model = Model(inputs=[input], outputs=[output])\n", 558 | " return model" 559 | ] 560 | }, 561 | { 562 | "cell_type": "code", 563 | "execution_count": 0, 564 | "metadata": { 565 | "colab": { 566 | "base_uri": "https://localhost:8080/", 567 | "height": 3360 568 | }, 569 | "colab_type": "code", 570 | "id": "AX5-NpZC3vRB", 571 | "outputId": "80393c89-b2c5-4b2a-a08f-db8c8d85036d" 572 | }, 573 | "outputs": [ 574 | { 575 | "name": "stdout", 576 | "output_type": "stream", 577 | "text": [ 578 | "__________________________________________________________________________________________________\n", 579 | "Layer (type) Output Shape Param # Connected to \n", 580 | "==================================================================================================\n", 581 | "input_2 (InputLayer) (None, None, None, 3 0 \n", 582 | "__________________________________________________________________________________________________\n", 583 | "conv2d_2 (Conv2D) (None, None, None, 3 864 input_2[0][0] \n", 584 | "__________________________________________________________________________________________________\n", 585 | "batch_normalization_24 (BatchNo (None, None, None, 3 128 conv2d_2[0][0] \n", 586 | "__________________________________________________________________________________________________\n", 587 | "activation_24 (Activation) (None, None, None, 3 0 batch_normalization_24[0][0] \n", 588 | "__________________________________________________________________________________________________\n", 589 | "separable_conv2d_23 (SeparableC (None, None, None, 6 2336 activation_24[0][0] \n", 590 | "__________________________________________________________________________________________________\n", 591 | "concatenate_14 (Concatenate) (None, None, None, 9 0 conv2d_2[0][0] \n", 592 | " separable_conv2d_23[0][0] \n", 593 | "__________________________________________________________________________________________________\n", 594 | "batch_normalization_25 (BatchNo (None, None, None, 9 384 concatenate_14[0][0] \n", 595 | "__________________________________________________________________________________________________\n", 596 | "activation_25 (Activation) (None, None, None, 9 0 batch_normalization_25[0][0] \n", 597 | "__________________________________________________________________________________________________\n", 598 | "separable_conv2d_24 (SeparableC (None, None, None, 6 7008 activation_25[0][0] \n", 599 | "__________________________________________________________________________________________________\n", 600 | "concatenate_15 (Concatenate) (None, None, None, 1 0 concatenate_14[0][0] \n", 601 | " separable_conv2d_24[0][0] \n", 602 | "__________________________________________________________________________________________________\n", 603 | "batch_normalization_26 (BatchNo (None, None, None, 1 640 concatenate_15[0][0] \n", 604 | "__________________________________________________________________________________________________\n", 605 | "activation_26 (Activation) (None, None, None, 1 0 batch_normalization_26[0][0] \n", 606 | "__________________________________________________________________________________________________\n", 607 | "separable_conv2d_25 (SeparableC (None, None, None, 6 11680 activation_26[0][0] \n", 608 | "__________________________________________________________________________________________________\n", 609 | "concatenate_16 (Concatenate) (None, None, None, 2 0 concatenate_15[0][0] \n", 610 | " separable_conv2d_25[0][0] \n", 611 | "__________________________________________________________________________________________________\n", 612 | "batch_normalization_27 (BatchNo (None, None, None, 2 896 concatenate_16[0][0] \n", 613 | "__________________________________________________________________________________________________\n", 614 | "activation_27 (Activation) (None, None, None, 2 0 batch_normalization_27[0][0] \n", 615 | "__________________________________________________________________________________________________\n", 616 | "separable_conv2d_26 (SeparableC (None, None, None, 3 71904 activation_27[0][0] \n", 617 | "__________________________________________________________________________________________________\n", 618 | "batch_normalization_28 (BatchNo (None, None, None, 3 1280 separable_conv2d_26[0][0] \n", 619 | "__________________________________________________________________________________________________\n", 620 | "activation_28 (Activation) (None, None, None, 3 0 batch_normalization_28[0][0] \n", 621 | "__________________________________________________________________________________________________\n", 622 | "separable_conv2d_27 (SeparableC (None, None, None, 6 23360 activation_28[0][0] \n", 623 | "__________________________________________________________________________________________________\n", 624 | "concatenate_17 (Concatenate) (None, None, None, 3 0 separable_conv2d_26[0][0] \n", 625 | " separable_conv2d_27[0][0] \n", 626 | "__________________________________________________________________________________________________\n", 627 | "batch_normalization_29 (BatchNo (None, None, None, 3 1536 concatenate_17[0][0] \n", 628 | "__________________________________________________________________________________________________\n", 629 | "activation_29 (Activation) (None, None, None, 3 0 batch_normalization_29[0][0] \n", 630 | "__________________________________________________________________________________________________\n", 631 | "separable_conv2d_28 (SeparableC (None, None, None, 6 28032 activation_29[0][0] \n", 632 | "__________________________________________________________________________________________________\n", 633 | "concatenate_18 (Concatenate) (None, None, None, 4 0 concatenate_17[0][0] \n", 634 | " separable_conv2d_28[0][0] \n", 635 | "__________________________________________________________________________________________________\n", 636 | "batch_normalization_30 (BatchNo (None, None, None, 4 1792 concatenate_18[0][0] \n", 637 | "__________________________________________________________________________________________________\n", 638 | "activation_30 (Activation) (None, None, None, 4 0 batch_normalization_30[0][0] \n", 639 | "__________________________________________________________________________________________________\n", 640 | "separable_conv2d_29 (SeparableC (None, None, None, 6 32704 activation_30[0][0] \n", 641 | "__________________________________________________________________________________________________\n", 642 | "concatenate_19 (Concatenate) (None, None, None, 5 0 concatenate_18[0][0] \n", 643 | " separable_conv2d_29[0][0] \n", 644 | "__________________________________________________________________________________________________\n", 645 | "batch_normalization_31 (BatchNo (None, None, None, 5 2048 concatenate_19[0][0] \n", 646 | "__________________________________________________________________________________________________\n", 647 | "activation_31 (Activation) (None, None, None, 5 0 batch_normalization_31[0][0] \n", 648 | "__________________________________________________________________________________________________\n", 649 | "separable_conv2d_30 (SeparableC (None, None, None, 5 262656 activation_31[0][0] \n", 650 | "__________________________________________________________________________________________________\n", 651 | "batch_normalization_32 (BatchNo (None, None, None, 5 2048 separable_conv2d_30[0][0] \n", 652 | "__________________________________________________________________________________________________\n", 653 | "activation_32 (Activation) (None, None, None, 5 0 batch_normalization_32[0][0] \n", 654 | "__________________________________________________________________________________________________\n", 655 | "separable_conv2d_31 (SeparableC (None, None, None, 6 37376 activation_32[0][0] \n", 656 | "__________________________________________________________________________________________________\n", 657 | "concatenate_20 (Concatenate) (None, None, None, 5 0 separable_conv2d_30[0][0] \n", 658 | " separable_conv2d_31[0][0] \n", 659 | "__________________________________________________________________________________________________\n", 660 | "batch_normalization_33 (BatchNo (None, None, None, 5 2304 concatenate_20[0][0] \n", 661 | "__________________________________________________________________________________________________\n", 662 | "activation_33 (Activation) (None, None, None, 5 0 batch_normalization_33[0][0] \n", 663 | "__________________________________________________________________________________________________\n", 664 | "separable_conv2d_32 (SeparableC (None, None, None, 6 42048 activation_33[0][0] \n", 665 | "__________________________________________________________________________________________________\n", 666 | "concatenate_21 (Concatenate) (None, None, None, 6 0 concatenate_20[0][0] \n", 667 | " separable_conv2d_32[0][0] \n", 668 | "__________________________________________________________________________________________________\n", 669 | "batch_normalization_34 (BatchNo (None, None, None, 6 2560 concatenate_21[0][0] \n", 670 | "__________________________________________________________________________________________________\n", 671 | "activation_34 (Activation) (None, None, None, 6 0 batch_normalization_34[0][0] \n", 672 | "__________________________________________________________________________________________________\n", 673 | "separable_conv2d_33 (SeparableC (None, None, None, 6 46720 activation_34[0][0] \n", 674 | "__________________________________________________________________________________________________\n", 675 | "concatenate_22 (Concatenate) (None, None, None, 7 0 concatenate_21[0][0] \n", 676 | " separable_conv2d_33[0][0] \n", 677 | "__________________________________________________________________________________________________\n", 678 | "batch_normalization_35 (BatchNo (None, None, None, 7 2816 concatenate_22[0][0] \n", 679 | "__________________________________________________________________________________________________\n", 680 | "activation_35 (Activation) (None, None, None, 7 0 batch_normalization_35[0][0] \n", 681 | "__________________________________________________________________________________________________\n", 682 | "separable_conv2d_34 (SeparableC (None, None, None, 7 496320 activation_35[0][0] \n", 683 | "__________________________________________________________________________________________________\n", 684 | "average_pooling2d_3 (AveragePoo (None, None, None, 7 0 separable_conv2d_34[0][0] \n", 685 | "__________________________________________________________________________________________________\n", 686 | "batch_normalization_36 (BatchNo (None, None, None, 7 2816 average_pooling2d_3[0][0] \n", 687 | "__________________________________________________________________________________________________\n", 688 | "activation_36 (Activation) (None, None, None, 7 0 batch_normalization_36[0][0] \n", 689 | "__________________________________________________________________________________________________\n", 690 | "separable_conv2d_35 (SeparableC (None, None, None, 6 51392 activation_36[0][0] \n", 691 | "__________________________________________________________________________________________________\n", 692 | "concatenate_23 (Concatenate) (None, None, None, 7 0 average_pooling2d_3[0][0] \n", 693 | " separable_conv2d_35[0][0] \n", 694 | "__________________________________________________________________________________________________\n", 695 | "batch_normalization_37 (BatchNo (None, None, None, 7 3072 concatenate_23[0][0] \n", 696 | "__________________________________________________________________________________________________\n", 697 | "activation_37 (Activation) (None, None, None, 7 0 batch_normalization_37[0][0] \n", 698 | "__________________________________________________________________________________________________\n", 699 | "separable_conv2d_36 (SeparableC (None, None, None, 6 56064 activation_37[0][0] \n", 700 | "__________________________________________________________________________________________________\n", 701 | "concatenate_24 (Concatenate) (None, None, None, 8 0 concatenate_23[0][0] \n", 702 | " separable_conv2d_36[0][0] \n", 703 | "__________________________________________________________________________________________________\n", 704 | "batch_normalization_38 (BatchNo (None, None, None, 8 3328 concatenate_24[0][0] \n", 705 | "__________________________________________________________________________________________________\n", 706 | "activation_38 (Activation) (None, None, None, 8 0 batch_normalization_38[0][0] \n", 707 | "__________________________________________________________________________________________________\n", 708 | "separable_conv2d_37 (SeparableC (None, None, None, 6 60736 activation_38[0][0] \n", 709 | "__________________________________________________________________________________________________\n", 710 | "concatenate_25 (Concatenate) (None, None, None, 8 0 concatenate_24[0][0] \n", 711 | " separable_conv2d_37[0][0] \n", 712 | "__________________________________________________________________________________________________\n", 713 | "batch_normalization_40 (BatchNo (None, None, None, 5 2048 separable_conv2d_30[0][0] \n", 714 | "__________________________________________________________________________________________________\n", 715 | "batch_normalization_41 (BatchNo (None, None, None, 7 2816 average_pooling2d_3[0][0] \n", 716 | "__________________________________________________________________________________________________\n", 717 | "batch_normalization_39 (BatchNo (None, None, None, 8 3584 concatenate_25[0][0] \n", 718 | "__________________________________________________________________________________________________\n", 719 | "activation_40 (Activation) (None, None, None, 5 0 batch_normalization_40[0][0] \n", 720 | "__________________________________________________________________________________________________\n", 721 | "activation_41 (Activation) (None, None, None, 7 0 batch_normalization_41[0][0] \n", 722 | "__________________________________________________________________________________________________\n", 723 | "activation_39 (Activation) (None, None, None, 8 0 batch_normalization_39[0][0] \n", 724 | "__________________________________________________________________________________________________\n", 725 | "separable_conv2d_39 (SeparableC (None, None, None, 6 33280 activation_40[0][0] \n", 726 | "__________________________________________________________________________________________________\n", 727 | "separable_conv2d_40 (SeparableC (None, None, None, 2 180928 activation_41[0][0] \n", 728 | "__________________________________________________________________________________________________\n", 729 | "separable_conv2d_38 (SeparableC (None, None, None, 8 803712 activation_39[0][0] \n", 730 | "__________________________________________________________________________________________________\n", 731 | "lambda_3 (Lambda) (None, None, None, 1 0 separable_conv2d_39[0][0] \n", 732 | "__________________________________________________________________________________________________\n", 733 | "lambda_4 (Lambda) (None, None, None, 1 0 separable_conv2d_40[0][0] \n", 734 | "__________________________________________________________________________________________________\n", 735 | "average_pooling2d_4 (AveragePoo (None, None, None, 8 0 separable_conv2d_38[0][0] \n", 736 | "__________________________________________________________________________________________________\n", 737 | "concatenate_26 (Concatenate) (None, None, None, 2 0 lambda_3[0][0] \n", 738 | " lambda_4[0][0] \n", 739 | " average_pooling2d_4[0][0] \n", 740 | "__________________________________________________________________________________________________\n", 741 | "batch_normalization_42 (BatchNo (None, None, None, 2 11776 concatenate_26[0][0] \n", 742 | "__________________________________________________________________________________________________\n", 743 | "activation_42 (Activation) (None, None, None, 2 0 batch_normalization_42[0][0] \n", 744 | "__________________________________________________________________________________________________\n", 745 | "separable_conv2d_41 (SeparableC (None, None, None, 1 4713344 activation_42[0][0] \n", 746 | "__________________________________________________________________________________________________\n", 747 | "batch_normalization_43 (BatchNo (None, None, None, 1 6400 separable_conv2d_41[0][0] \n", 748 | "__________________________________________________________________________________________________\n", 749 | "activation_43 (Activation) (None, None, None, 1 0 batch_normalization_43[0][0] \n", 750 | "__________________________________________________________________________________________________\n", 751 | "separable_conv2d_42 (SeparableC (None, None, None, 8 1294400 activation_43[0][0] \n", 752 | "__________________________________________________________________________________________________\n", 753 | "batch_normalization_44 (BatchNo (None, None, None, 8 3200 separable_conv2d_42[0][0] \n", 754 | "__________________________________________________________________________________________________\n", 755 | "activation_44 (Activation) (None, None, None, 8 0 batch_normalization_44[0][0] \n", 756 | "__________________________________________________________________________________________________\n", 757 | "separable_conv2d_43 (SeparableC (None, None, None, 4 327200 activation_44[0][0] \n", 758 | "__________________________________________________________________________________________________\n", 759 | "batch_normalization_45 (BatchNo (None, None, None, 4 1600 separable_conv2d_43[0][0] \n", 760 | "__________________________________________________________________________________________________\n", 761 | "activation_45 (Activation) (None, None, None, 4 0 batch_normalization_45[0][0] \n", 762 | "__________________________________________________________________________________________________\n", 763 | "separable_conv2d_44 (SeparableC (None, None, None, 2 80400 activation_45[0][0] \n", 764 | "__________________________________________________________________________________________________\n", 765 | "batch_normalization_46 (BatchNo (None, None, None, 2 800 separable_conv2d_44[0][0] \n", 766 | "__________________________________________________________________________________________________\n", 767 | "activation_46 (Activation) (None, None, None, 2 0 batch_normalization_46[0][0] \n", 768 | "__________________________________________________________________________________________________\n", 769 | "global_average_pooling2d_2 (Glo (None, 200) 0 activation_46[0][0] \n", 770 | "__________________________________________________________________________________________________\n", 771 | "softmax_2 (Softmax) (None, 200) 0 global_average_pooling2d_2[0][0] \n", 772 | "==================================================================================================\n", 773 | "Total params: 8,724,336\n", 774 | "Trainable params: 8,694,400\n", 775 | "Non-trainable params: 29,936\n", 776 | "__________________________________________________________________________________________________\n" 777 | ] 778 | } 779 | ], 780 | "source": [ 781 | "from keras.layers.advanced_activations import Softmax\n", 782 | "from keras.layers import Dense, Dropout, Flatten, Input, AveragePooling2D, merge, Activation, GlobalAveragePooling2D, SeparableConv2D\n", 783 | "from keras.layers import Reshape, Activation, Conv2D, Input, MaxPooling2D, BatchNormalization, Flatten, Dense, Lambda\n", 784 | "from keras.models import Model\n", 785 | "from keras.optimizers import Adam\n", 786 | "from keras.regularizers import l2\n", 787 | "input = Input(shape=(None, None, NUM_CHANNELS,))\n", 788 | "model = densenet(num_filter=64)\n", 789 | "model.summary()\n", 790 | "model.compile(loss='categorical_crossentropy',\n", 791 | " optimizer=Adam(lr=0.001),\n", 792 | " metrics=['accuracy'])" 793 | ] 794 | }, 795 | { 796 | "cell_type": "code", 797 | "execution_count": 0, 798 | "metadata": { 799 | "colab": { 800 | "base_uri": "https://localhost:8080/", 801 | "height": 1254 802 | }, 803 | "colab_type": "code", 804 | "id": "egp1Echg3xUA", 805 | "outputId": "4e7b9a8a-b66c-45e2-ebfe-f721ac7ebe33" 806 | }, 807 | "outputs": [ 808 | { 809 | "name": "stdout", 810 | "output_type": "stream", 811 | "text": [ 812 | "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", 813 | "Instructions for updating:\n", 814 | "Use tf.cast instead.\n", 815 | "Epoch 1/50\n", 816 | "1562/1562 [==============================] - 1592s 1s/step - loss: 4.8105 - acc: 0.0605 - val_loss: 4.3155 - val_acc: 0.1087\n", 817 | "Current learning rate 0.001 1562\n", 818 | "\n", 819 | "Epoch 00001: saving model to Imagenetvd.hdf5\n", 820 | "Epoch 2/50\n", 821 | "1562/1562 [==============================] - 1580s 1s/step - loss: 4.2235 - acc: 0.1411 - val_loss: 3.8027 - val_acc: 0.1886\n", 822 | "Current learning rate 0.001 3124\n", 823 | "\n", 824 | "Epoch 00002: saving model to Imagenetvd.hdf5\n", 825 | "Epoch 3/50\n", 826 | "1562/1562 [==============================] - 1576s 1s/step - loss: 3.8336 - acc: 0.1946 - val_loss: 3.4333 - val_acc: 0.2470\n", 827 | "Current learning rate 0.001 4686\n", 828 | "\n", 829 | "Epoch 00003: saving model to Imagenetvd.hdf5\n", 830 | "Epoch 4/50\n", 831 | "1562/1562 [==============================] - 1577s 1s/step - loss: 3.5559 - acc: 0.2360 - val_loss: 3.2340 - val_acc: 0.2766\n", 832 | "Current learning rate 0.001 6248\n", 833 | "\n", 834 | "Epoch 00004: saving model to Imagenetvd.hdf5\n", 835 | "Epoch 5/50\n", 836 | "1562/1562 [==============================] - 1578s 1s/step - loss: 3.3554 - acc: 0.2684 - val_loss: 3.1185 - val_acc: 0.3032\n", 837 | "Current learning rate 0.001 7810\n" 838 | ] 839 | }, 840 | { 841 | "ename": "NameError", 842 | "evalue": "ignored", 843 | "output_type": "error", 844 | "traceback": [ 845 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 846 | "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 847 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSTEP_SIZE_VALID\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNUM_EPOCHS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])\n\u001b[0m", 848 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 89\u001b[0m warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[1;32m 90\u001b[0m 'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 92\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 849 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 1416\u001b[0m \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1417\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1418\u001b[0;31m initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m 1419\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1420\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 850 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 249\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 250\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 251\u001b[0;31m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 252\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 253\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallback_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 851 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_epoch_end\u001b[0;34m(self, epoch, logs)\u001b[0m\n\u001b[1;32m 77\u001b[0m \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlogs\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 78\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 79\u001b[0;31m \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 80\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 852 | "\u001b[0;32m\u001b[0m in \u001b[0;36mon_epoch_end\u001b[0;34m(self, epoch, logs)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mNUM_IMAGES\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0mdrive\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdrive_authenticate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0msave_to_drive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdrive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcurrent\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 853 | "\u001b[0;31mNameError\u001b[0m: name 'drive_authenticate' is not defined" 854 | ] 855 | } 856 | ], 857 | "source": [ 858 | "model.fit_generator(generator=train_generator,\n", 859 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 860 | " validation_data=val_generator,\n", 861 | " validation_steps=STEP_SIZE_VALID,\n", 862 | " epochs=NUM_EPOCHS, verbose=1,\n", 863 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 864 | ] 865 | }, 866 | { 867 | "cell_type": "code", 868 | "execution_count": 0, 869 | "metadata": { 870 | "colab": { 871 | "base_uri": "https://localhost:8080/", 872 | "height": 1260 873 | }, 874 | "colab_type": "code", 875 | "id": "Pz6okkTLogxI", 876 | "outputId": "4f4aad10-ad25-4635-bfca-469dada172e2" 877 | }, 878 | "outputs": [ 879 | { 880 | "name": "stdout", 881 | "output_type": "stream", 882 | "text": [ 883 | "Epoch 6/50\n", 884 | "1562/1562 [==============================] - 1587s 1s/step - loss: 3.1948 - acc: 0.2952 - val_loss: 2.9732 - val_acc: 0.3278\n", 885 | "Current learning rate 0.001 9372\n", 886 | "\n", 887 | "Epoch 00006: saving model to Imagenetvd.hdf5\n", 888 | "Epoch 7/50\n", 889 | "1562/1562 [==============================] - 1585s 1s/step - loss: 3.0575 - acc: 0.3168 - val_loss: 2.7520 - val_acc: 0.3686\n", 890 | "Current learning rate 0.001 10934\n", 891 | "\n", 892 | "Epoch 00007: saving model to Imagenetvd.hdf5\n", 893 | "Epoch 8/50\n", 894 | "1562/1562 [==============================] - 1581s 1s/step - loss: 2.9428 - acc: 0.3387 - val_loss: 2.6633 - val_acc: 0.3871\n", 895 | "Current learning rate 0.001 12496\n", 896 | "\n", 897 | "Epoch 00008: saving model to Imagenetvd.hdf5\n", 898 | "Epoch 9/50\n", 899 | "1562/1562 [==============================] - 1579s 1s/step - loss: 2.8363 - acc: 0.3583 - val_loss: 2.7351 - val_acc: 0.3737\n", 900 | "Current learning rate 0.001 14058\n", 901 | "\n", 902 | "Epoch 00009: saving model to Imagenetvd.hdf5\n", 903 | "Epoch 10/50\n", 904 | "1562/1562 [==============================] - 1575s 1s/step - loss: 2.7361 - acc: 0.3767 - val_loss: 2.6234 - val_acc: 0.3937\n", 905 | "Current learning rate 0.001 15620\n", 906 | "Saving to id 1RcSoWtoolBgk-U8z7upzFUeP5KT8v3d5\n", 907 | "\n", 908 | "Epoch 00010: saving model to Imagenetvd.hdf5\n", 909 | "Epoch 11/50\n", 910 | "1562/1562 [==============================] - 1577s 1s/step - loss: 2.6561 - acc: 0.3922 - val_loss: 2.5113 - val_acc: 0.4150\n", 911 | "Current learning rate 0.001 17182\n", 912 | "\n", 913 | "Epoch 00011: saving model to Imagenetvd.hdf5\n", 914 | "Epoch 12/50\n", 915 | "1562/1562 [==============================] - 1574s 1s/step - loss: 2.5775 - acc: 0.4067 - val_loss: 2.4428 - val_acc: 0.4308\n", 916 | "Current learning rate 0.001 18744\n", 917 | "\n", 918 | "Epoch 00012: saving model to Imagenetvd.hdf5\n", 919 | "Epoch 13/50\n", 920 | "1562/1562 [==============================] - 1575s 1s/step - loss: 2.5041 - acc: 0.4196 - val_loss: 2.4277 - val_acc: 0.4355\n", 921 | "Current learning rate 0.001 20306\n", 922 | "\n", 923 | "Epoch 00013: saving model to Imagenetvd.hdf5\n", 924 | "Epoch 14/50\n", 925 | "1562/1562 [==============================] - 1573s 1s/step - loss: 2.4260 - acc: 0.4355 - val_loss: 2.4518 - val_acc: 0.4330\n", 926 | "Current learning rate 0.001 21868\n", 927 | "\n", 928 | "Epoch 00014: saving model to Imagenetvd.hdf5\n", 929 | "Epoch 15/50\n", 930 | "1562/1562 [==============================] - 1564s 1s/step - loss: 2.3595 - acc: 0.4479 - val_loss: 2.3932 - val_acc: 0.4416\n", 931 | "Current learning rate 0.001 23430\n", 932 | "Saving to id 1xd_cUTGNRavpjYFrmin52dEXyO23UQIm\n", 933 | "\n", 934 | "Epoch 00015: saving model to Imagenetvd.hdf5\n", 935 | "Epoch 16/50\n", 936 | "1562/1562 [==============================] - 1560s 999ms/step - loss: 2.2924 - acc: 0.4622 - val_loss: 2.3120 - val_acc: 0.4578\n", 937 | "Current learning rate 0.001 24992\n", 938 | "\n", 939 | "Epoch 00016: saving model to Imagenetvd.hdf5\n", 940 | "Epoch 17/50\n", 941 | "1562/1562 [==============================] - 1559s 998ms/step - loss: 2.2273 - acc: 0.4755 - val_loss: 2.3026 - val_acc: 0.4611\n", 942 | "Current learning rate 0.001 26554\n", 943 | "\n", 944 | "Epoch 00017: saving model to Imagenetvd.hdf5\n", 945 | "Epoch 18/50\n", 946 | "1562/1562 [==============================] - 1559s 998ms/step - loss: 2.1666 - acc: 0.4878 - val_loss: 2.3460 - val_acc: 0.4541\n", 947 | "Current learning rate 0.001 28116\n", 948 | "\n", 949 | "Epoch 00018: saving model to Imagenetvd.hdf5\n", 950 | "Epoch 19/50\n", 951 | "1562/1562 [==============================] - 1559s 998ms/step - loss: 2.1177 - acc: 0.4952 - val_loss: 2.3441 - val_acc: 0.4584\n", 952 | "Current learning rate 0.001 29678\n", 953 | "\n", 954 | "Epoch 00019: saving model to Imagenetvd.hdf5\n", 955 | "Epoch 20/50\n", 956 | "1500/1562 [===========================>..] - ETA: 1:00 - loss: 2.0555 - acc: 0.5098" 957 | ] 958 | } 959 | ], 960 | "source": [ 961 | "model.fit_generator(initial_epoch=5,\n", 962 | " generator=train_generator,\n", 963 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 964 | " validation_data=val_generator,\n", 965 | " validation_steps=STEP_SIZE_VALID,\n", 966 | " epochs=NUM_EPOCHS, verbose=1,\n", 967 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 968 | ] 969 | }, 970 | { 971 | "cell_type": "code", 972 | "execution_count": 0, 973 | "metadata": { 974 | "colab": { 975 | "base_uri": "https://localhost:8080/", 976 | "height": 87 977 | }, 978 | "colab_type": "code", 979 | "id": "lyXQliKeqWUw", 980 | "outputId": "da035c7b-9b09-461c-9c79-7fbbeb38a26a" 981 | }, 982 | "outputs": [ 983 | { 984 | "name": "stdout", 985 | "output_type": "stream", 986 | "text": [ 987 | "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", 988 | "Instructions for updating:\n", 989 | "Use tf.cast instead.\n" 990 | ] 991 | } 992 | ], 993 | "source": [ 994 | "from keras.models import load_model\n", 995 | "!cp drive/'My Drive'/Imagenetvd15.hdf5 .\n", 996 | "model = load_model('Imagenetvd15.hdf5', custom_objects={\"tf\": tf})" 997 | ] 998 | }, 999 | { 1000 | "cell_type": "code", 1001 | "execution_count": 0, 1002 | "metadata": { 1003 | "colab": { 1004 | "base_uri": "https://localhost:8080/", 1005 | "height": 470 1006 | }, 1007 | "colab_type": "code", 1008 | "id": "XVS_GVIRI7Dg", 1009 | "outputId": "8883860d-b4e3-4bd2-bc5c-aaee209c1ad4" 1010 | }, 1011 | "outputs": [ 1012 | { 1013 | "name": "stdout", 1014 | "output_type": "stream", 1015 | "text": [ 1016 | "Epoch 16/20\n", 1017 | "781/781 [==============================] - 1001s 1s/step - loss: 2.2861 - acc: 0.4626 - val_loss: 2.8079 - val_acc: 0.3857\n", 1018 | "Current learning rate 0.001 25370\n", 1019 | "\n", 1020 | "Epoch 00016: saving model to Imagenetvd.hdf5\n", 1021 | "Epoch 17/20\n", 1022 | "781/781 [==============================] - 1007s 1s/step - loss: 2.5634 - acc: 0.4003 - val_loss: 2.6938 - val_acc: 0.3931\n", 1023 | "Current learning rate 0.001 26151\n", 1024 | "\n", 1025 | "Epoch 00017: saving model to Imagenetvd.hdf5\n", 1026 | "Epoch 18/20\n", 1027 | "781/781 [==============================] - 998s 1s/step - loss: 2.4797 - acc: 0.4150 - val_loss: 2.6603 - val_acc: 0.3956\n", 1028 | "Current learning rate 0.001 26932\n", 1029 | "\n", 1030 | "Epoch 00018: saving model to Imagenetvd.hdf5\n", 1031 | "Epoch 19/20\n", 1032 | "781/781 [==============================] - 1003s 1s/step - loss: 2.4064 - acc: 0.4307 - val_loss: 2.6281 - val_acc: 0.4009\n", 1033 | "Current learning rate 0.001 27713\n", 1034 | "Saving to id 1IGJWcVW-iiEjZzKaz4Y0rfK_2o9gW4Dt\n", 1035 | "\n", 1036 | "Epoch 00019: saving model to Imagenetvd.hdf5\n", 1037 | "Epoch 20/20\n", 1038 | "781/781 [==============================] - 989s 1s/step - loss: 2.3469 - acc: 0.4427 - val_loss: 2.6095 - val_acc: 0.4119\n", 1039 | "Current learning rate 0.001 28494\n", 1040 | "\n", 1041 | "Epoch 00020: saving model to Imagenetvd.hdf5\n" 1042 | ] 1043 | }, 1044 | { 1045 | "data": { 1046 | "text/plain": [ 1047 | "" 1048 | ] 1049 | }, 1050 | "execution_count": 30, 1051 | "metadata": { 1052 | "tags": [] 1053 | }, 1054 | "output_type": "execute_result" 1055 | } 1056 | ], 1057 | "source": [ 1058 | "#Loaded checkpoint from epoch 15, Training with 20x20\n", 1059 | "model.fit_generator(initial_epoch=15,\n", 1060 | " generator=train_generator,\n", 1061 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 1062 | " validation_data=val_generator,\n", 1063 | " validation_steps=STEP_SIZE_VALID,\n", 1064 | " epochs=20, verbose=1,\n", 1065 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 1066 | ] 1067 | }, 1068 | { 1069 | "cell_type": "code", 1070 | "execution_count": 0, 1071 | "metadata": { 1072 | "colab": {}, 1073 | "colab_type": "code", 1074 | "id": "UayVzRr2mwP3" 1075 | }, 1076 | "outputs": [], 1077 | "source": [ 1078 | "from keras.models import load_model\n", 1079 | "!cp drive/'My Drive'/Imagenetvd20.hdf5 .\n", 1080 | "model = load_model('Imagenetvd20.hdf5', custom_objects={\"tf\": tf})" 1081 | ] 1082 | }, 1083 | { 1084 | "cell_type": "code", 1085 | "execution_count": 0, 1086 | "metadata": { 1087 | "colab": { 1088 | "base_uri": "https://localhost:8080/", 1089 | "height": 218 1090 | }, 1091 | "colab_type": "code", 1092 | "id": "lnDLHABGJ2Rf", 1093 | "outputId": "c9c8745e-ef41-4070-ecae-ccdcf9e44cb8" 1094 | }, 1095 | "outputs": [ 1096 | { 1097 | "name": "stdout", 1098 | "output_type": "stream", 1099 | "text": [ 1100 | "Epoch 21/22\n", 1101 | "1562/1562 [==============================] - 1553s 994ms/step - loss: 2.7223 - acc: 0.3797 - val_loss: 2.3829 - val_acc: 0.4442\n", 1102 | "Current learning rate 0.001 30140\n", 1103 | "\n", 1104 | "Epoch 00021: saving model to Imagenetvd.hdf5\n", 1105 | "Epoch 22/22\n", 1106 | "1562/1562 [==============================] - 1563s 1s/step - loss: 2.6052 - acc: 0.4011 - val_loss: 2.3326 - val_acc: 0.4622\n", 1107 | "Current learning rate 0.001 31702\n", 1108 | "Saving to id 1Jn3aYMi7JlFesaJ01RqDcNRhcmhf6Mph\n", 1109 | "\n", 1110 | "Epoch 00022: saving model to Imagenetvd.hdf5\n" 1111 | ] 1112 | }, 1113 | { 1114 | "data": { 1115 | "text/plain": [ 1116 | "" 1117 | ] 1118 | }, 1119 | "execution_count": 32, 1120 | "metadata": { 1121 | "tags": [] 1122 | }, 1123 | "output_type": "execute_result" 1124 | } 1125 | ], 1126 | "source": [ 1127 | "#Training on 32x32\n", 1128 | "model.fit_generator(initial_epoch=20,\n", 1129 | " generator=train_generator,\n", 1130 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 1131 | " validation_data=val_generator,\n", 1132 | " validation_steps=STEP_SIZE_VALID,\n", 1133 | " epochs=22, verbose=1,\n", 1134 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 1135 | ] 1136 | }, 1137 | { 1138 | "cell_type": "code", 1139 | "execution_count": 0, 1140 | "metadata": { 1141 | "colab": { 1142 | "base_uri": "https://localhost:8080/", 1143 | "height": 286 1144 | }, 1145 | "colab_type": "code", 1146 | "id": "1KdJi7v4mPqf", 1147 | "outputId": "a4aca2d6-0f22-431d-861f-154b27d229a7" 1148 | }, 1149 | "outputs": [ 1150 | { 1151 | "name": "stdout", 1152 | "output_type": "stream", 1153 | "text": [ 1154 | "Epoch 23/25\n", 1155 | "1562/1562 [==============================] - 1561s 1000ms/step - loss: 2.5498 - acc: 0.4108 - val_loss: 2.3393 - val_acc: 0.4516\n", 1156 | "Current learning rate 0.001 33264\n", 1157 | "\n", 1158 | "Epoch 00023: saving model to Imagenetvd.hdf5\n", 1159 | "Epoch 24/25\n", 1160 | "1562/1562 [==============================] - 1555s 996ms/step - loss: 2.5113 - acc: 0.4195 - val_loss: 2.3512 - val_acc: 0.4597\n", 1161 | "Current learning rate 0.001 34826\n", 1162 | "\n", 1163 | "Epoch 00024: saving model to Imagenetvd.hdf5\n", 1164 | "Epoch 25/25\n", 1165 | "1562/1562 [==============================] - 1555s 996ms/step - loss: 2.4594 - acc: 0.4290 - val_loss: 2.3073 - val_acc: 0.4688\n", 1166 | "Current learning rate 0.001 36388\n", 1167 | "\n", 1168 | "Epoch 00025: saving model to Imagenetvd.hdf5\n" 1169 | ] 1170 | }, 1171 | { 1172 | "data": { 1173 | "text/plain": [ 1174 | "" 1175 | ] 1176 | }, 1177 | "execution_count": 33, 1178 | "metadata": { 1179 | "tags": [] 1180 | }, 1181 | "output_type": "execute_result" 1182 | } 1183 | ], 1184 | "source": [ 1185 | "#Continue training on 32x32\n", 1186 | "model.fit_generator(initial_epoch=22,\n", 1187 | " generator=train_generator,\n", 1188 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 1189 | " validation_data=val_generator,\n", 1190 | " validation_steps=STEP_SIZE_VALID,\n", 1191 | " epochs=25, verbose=1,\n", 1192 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 1193 | ] 1194 | }, 1195 | { 1196 | "cell_type": "code", 1197 | "execution_count": 0, 1198 | "metadata": { 1199 | "colab": { 1200 | "base_uri": "https://localhost:8080/", 1201 | "height": 1425 1202 | }, 1203 | "colab_type": "code", 1204 | "id": "ZNoGbVk-ZZ3O", 1205 | "outputId": "27acc8f4-6256-43e5-9200-01e1f1a704ce" 1206 | }, 1207 | "outputs": [ 1208 | { 1209 | "name": "stdout", 1210 | "output_type": "stream", 1211 | "text": [ 1212 | "Epoch 26/30\n", 1213 | "1562/1562 [==============================] - 1562s 1000ms/step - loss: 2.4247 - acc: 0.4346 - val_loss: 2.2871 - val_acc: 0.4694\n", 1214 | "Current learning rate 0.001 37950\n", 1215 | "\n", 1216 | "Epoch 00026: saving model to Imagenetvd.hdf5\n", 1217 | "Epoch 27/30\n", 1218 | "1562/1562 [==============================] - 1561s 999ms/step - loss: 2.3984 - acc: 0.4414 - val_loss: 2.3543 - val_acc: 0.4573\n", 1219 | "Current learning rate 0.001 39512\n", 1220 | "Saving to id 1xef3xGHlB6ID8LH1qSfKE_8YHyJndsxa\n", 1221 | "\n", 1222 | "Epoch 00027: saving model to Imagenetvd.hdf5\n", 1223 | "Epoch 28/30\n", 1224 | "1562/1562 [==============================] - 1555s 996ms/step - loss: 2.3463 - acc: 0.4510 - val_loss: 2.3317 - val_acc: 0.4690\n", 1225 | "Current learning rate 0.001 41074\n", 1226 | "\n", 1227 | "Epoch 00028: saving model to Imagenetvd.hdf5\n", 1228 | "Epoch 29/30\n", 1229 | "1562/1562 [==============================] - 1541s 987ms/step - loss: 2.3159 - acc: 0.4582 - val_loss: 2.3071 - val_acc: 0.4666\n", 1230 | "Current learning rate 0.001 42636\n", 1231 | "\n", 1232 | "Epoch 00029: saving model to Imagenetvd.hdf5\n", 1233 | "Epoch 30/30\n", 1234 | " 13/1562 [..............................] - ETA: 24:39 - loss: 2.1212 - acc: 0.5048" 1235 | ] 1236 | }, 1237 | { 1238 | "ename": "KeyboardInterrupt", 1239 | "evalue": "ignored", 1240 | "output_type": "error", 1241 | "traceback": [ 1242 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 1243 | "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", 1244 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSTEP_SIZE_VALID\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])\n\u001b[0m", 1245 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 89\u001b[0m warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[1;32m 90\u001b[0m 'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 92\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1246 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 1416\u001b[0m \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1417\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1418\u001b[0;31m initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m 1419\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1420\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1247 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 215\u001b[0m outs = model.train_on_batch(x, y,\n\u001b[1;32m 216\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m class_weight=class_weight)\n\u001b[0m\u001b[1;32m 218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1248 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m 1215\u001b[0m \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1216\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1217\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1218\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0munpack_singleton\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1219\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 1249 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2713\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2716\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2717\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1250 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2634\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'`inputs` should be a list or tuple.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2635\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2636\u001b[0;31m \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2637\u001b[0m \u001b[0mfeed_arrays\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2638\u001b[0m \u001b[0marray_vals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1251 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mget_session\u001b[0;34m()\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0mcandidate_vars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_keras_initialized'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 194\u001b[0m \u001b[0mcandidate_vars\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcandidate_vars\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1252 | "\u001b[0;31mKeyboardInterrupt\u001b[0m: " 1253 | ] 1254 | } 1255 | ], 1256 | "source": [ 1257 | "#Continue training on 32x32\n", 1258 | "model.fit_generator(initial_epoch=25,\n", 1259 | " generator=train_generator,\n", 1260 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 1261 | " validation_data=val_generator,\n", 1262 | " validation_steps=STEP_SIZE_VALID,\n", 1263 | " epochs=30, verbose=1,\n", 1264 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 1265 | ] 1266 | }, 1267 | { 1268 | "cell_type": "code", 1269 | "execution_count": 0, 1270 | "metadata": { 1271 | "colab": { 1272 | "base_uri": "https://localhost:8080/", 1273 | "height": 34 1274 | }, 1275 | "colab_type": "code", 1276 | "id": "w1yvKUCH0P6d", 1277 | "outputId": "e13986d6-f19d-4244-d51f-11f519d32736" 1278 | }, 1279 | "outputs": [ 1280 | { 1281 | "name": "stdout", 1282 | "output_type": "stream", 1283 | "text": [ 1284 | "Saving to id 1zgPu_qtl4VjInr9EdegDO4VryNNVs31d\n" 1285 | ] 1286 | } 1287 | ], 1288 | "source": [ 1289 | "drive = drive_authenticate()\n", 1290 | "save_to_drive(drive, 29, model_file)" 1291 | ] 1292 | }, 1293 | { 1294 | "cell_type": "code", 1295 | "execution_count": 0, 1296 | "metadata": { 1297 | "colab": {}, 1298 | "colab_type": "code", 1299 | "id": "alXmwR-ae2aW" 1300 | }, 1301 | "outputs": [], 1302 | "source": [] 1303 | } 1304 | ], 1305 | "metadata": { 1306 | "accelerator": "GPU", 1307 | "colab": { 1308 | "name": "ImagenetDensenet.ipynb", 1309 | "provenance": [], 1310 | "version": "0.3.2" 1311 | }, 1312 | "kernelspec": { 1313 | "display_name": "Python 3", 1314 | "language": "python", 1315 | "name": "python3" 1316 | }, 1317 | "language_info": { 1318 | "codemirror_mode": { 1319 | "name": "ipython", 1320 | "version": 3 1321 | }, 1322 | "file_extension": ".py", 1323 | "mimetype": "text/x-python", 1324 | "name": "python", 1325 | "nbconvert_exporter": "python", 1326 | "pygments_lexer": "ipython3", 1327 | "version": "3.7.1" 1328 | } 1329 | }, 1330 | "nbformat": 4, 1331 | "nbformat_minor": 1 1332 | } 1333 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/ImagenetDensenet1-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 0, 6 | "metadata": { 7 | "colab": { 8 | "base_uri": "https://localhost:8080/", 9 | "height": 513 10 | }, 11 | "colab_type": "code", 12 | "id": "0lhx1XzJ3CIy", 13 | "outputId": "2f24eaba-605f-43f2-ad14-64e17dcfefd4" 14 | }, 15 | "outputs": [ 16 | { 17 | "name": "stdout", 18 | "output_type": "stream", 19 | "text": [ 20 | "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n", 21 | "\n", 22 | "Enter your authorization code:\n", 23 | "··········\n", 24 | "Mounted at /content/drive\n", 25 | "Cloning into 'keras-resnet'...\n", 26 | "remote: Enumerating objects: 126, done.\u001b[K\n", 27 | "remote: Total 126 (delta 0), reused 0 (delta 0), pack-reused 126\u001b[K\n", 28 | "Receiving objects: 100% (126/126), 1.65 MiB | 1.53 MiB/s, done.\n", 29 | "Resolving deltas: 100% (58/58), done.\n", 30 | "Collecting PyDrive\n", 31 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/52/e0/0e64788e5dd58ce2d6934549676243dc69d982f198524be9b99e9c2a4fd5/PyDrive-1.3.1.tar.gz (987kB)\n", 32 | "\u001b[K 100% |████████████████████████████████| 993kB 19.3MB/s \n", 33 | "\u001b[?25hRequirement already satisfied: google-api-python-client>=1.2 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (1.6.7)\n", 34 | "Requirement already satisfied: oauth2client>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (4.1.3)\n", 35 | "Requirement already satisfied: PyYAML>=3.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (3.13)\n", 36 | "Requirement already satisfied: httplib2<1dev,>=0.9.2 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (0.11.3)\n", 37 | "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (3.0.0)\n", 38 | "Requirement already satisfied: six<2dev,>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (1.11.0)\n", 39 | "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.4.5)\n", 40 | "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.2.4)\n", 41 | "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (4.0)\n", 42 | "Building wheels for collected packages: PyDrive\n", 43 | " Building wheel for PyDrive (setup.py) ... \u001b[?25ldone\n", 44 | "\u001b[?25h Stored in directory: /root/.cache/pip/wheels/fa/d2/9a/d3b6b506c2da98289e5d417215ce34b696db856643bad779f4\n", 45 | "Successfully built PyDrive\n", 46 | "Installing collected packages: PyDrive\n", 47 | "Successfully installed PyDrive-1.3.1\n" 48 | ] 49 | } 50 | ], 51 | "source": [ 52 | "from google.colab import drive\n", 53 | "drive.mount('/content/drive', force_remount=True)\n", 54 | "!rm -rf tiny-imagenet-200.zip tiny-imagenet-200\n", 55 | "!git clone https://github.com/raghakot/keras-resnet.git\n", 56 | "!cp drive/'My Drive'/tiny-imagenet-200_flow.zip .\n", 57 | "!unzip -qq tiny-imagenet-200_flow.zip\n", 58 | "!rm -rf tiny-imagenet-200/val/images tiny-imagenet-200/val/val_annotations.txt\n", 59 | "!pip install PyDrive\n", 60 | "from pydrive.auth import GoogleAuth\n", 61 | "from pydrive.drive import GoogleDrive\n", 62 | "from google.colab import auth\n", 63 | "from oauth2client.client import GoogleCredentials\n", 64 | "\n", 65 | "# 1. Authenticate and create the PyDrive client.\n", 66 | "\n", 67 | "import tensorflow as tf\n", 68 | "from keras import backend as k\n", 69 | "config = tf.ConfigProto()\n", 70 | "config.gpu_options.allow_growth = True\n", 71 | "# Create a session with the above options specified.\n", 72 | "k.tensorflow_backend.set_session(tf.Session(config=config))" 73 | ] 74 | }, 75 | { 76 | "cell_type": "code", 77 | "execution_count": 0, 78 | "metadata": { 79 | "colab": {}, 80 | "colab_type": "code", 81 | "id": "0zxrHu6fpFPI" 82 | }, 83 | "outputs": [], 84 | "source": [ 85 | "def drive_authenticate():\n", 86 | " auth.authenticate_user()\n", 87 | " gauth = GoogleAuth()\n", 88 | " gauth.credentials = GoogleCredentials.get_application_default()\n", 89 | " drive = GoogleDrive(gauth)\n", 90 | " return drive" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 0, 96 | "metadata": { 97 | "colab": {}, 98 | "colab_type": "code", 99 | "id": "HYefiC_L3LYf" 100 | }, 101 | "outputs": [], 102 | "source": [ 103 | "IMAGE_SIZE, TRAINING_IMAGES_DIR, VAL_IMAGES_DIR = None, None, None\n", 104 | "model_file=\"Imagenetvd.hdf5\" \n", 105 | "def set_image_size(image_size):\n", 106 | " global IMAGE_SIZE, TRAINING_IMAGES_DIR, VAL_IMAGES_DIR\n", 107 | " IMAGE_SIZE = image_size\n", 108 | " if IMAGE_SIZE == 32:\n", 109 | " TRAINING_IMAGES_DIR = 'tiny-imagenet-200-32/train/'\n", 110 | " VAL_IMAGES_DIR = 'tiny-imagenet-200-32/val/'\n", 111 | " else:\n", 112 | " TRAINING_IMAGES_DIR = 'tiny-imagenet-200/train/'\n", 113 | " VAL_IMAGES_DIR = 'tiny-imagenet-200/val/'\n", 114 | "set_image_size(64) " 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 0, 120 | "metadata": { 121 | "colab": { 122 | "base_uri": "https://localhost:8080/", 123 | "height": 51 124 | }, 125 | "colab_type": "code", 126 | "id": "uY7EnC8b3otl", 127 | "outputId": "eb640b9a-a3ee-43ce-e46d-5aa545cf7f6b" 128 | }, 129 | "outputs": [ 130 | { 131 | "name": "stdout", 132 | "output_type": "stream", 133 | "text": [ 134 | "Found 100000 images belonging to 200 classes.\n", 135 | "Found 10000 images belonging to 200 classes.\n" 136 | ] 137 | } 138 | ], 139 | "source": [ 140 | "import shutil\n", 141 | "import numpy as np\n", 142 | "import pandas as pd\n", 143 | "import os\n", 144 | "from sklearn import preprocessing\n", 145 | "import matplotlib.image as mpimg\n", 146 | "from keras.preprocessing.image import ImageDataGenerator\n", 147 | "from keras import backend as k\n", 148 | "from keras.utils import np_utils\n", 149 | "from keras.callbacks import ReduceLROnPlateau, CSVLogger, EarlyStopping, Callback, ModelCheckpoint\n", 150 | "from keras.utils import Sequence\n", 151 | "from keras.preprocessing.image import load_img\n", 152 | "from keras.preprocessing.image import img_to_array\n", 153 | "import imgaug as ia\n", 154 | "from imgaug import augmenters as iaa\n", 155 | "from keras.layers import Concatenate\n", 156 | "\n", 157 | "BATCH_SIZE = 32\n", 158 | "NUM_CLASSES = 200\n", 159 | "NUM_IMAGES_PER_CLASS = 500\n", 160 | "NUM_IMAGES = NUM_CLASSES * NUM_IMAGES_PER_CLASS\n", 161 | "TRAIN_SIZE = NUM_IMAGES\n", 162 | "NUM_CHANNELS = 3\n", 163 | "NUM_VAL_IMAGES = 10000\n", 164 | "NUM_EPOCHS = 50\n", 165 | "IMAGE_SIZE_T = 48\n", 166 | "\n", 167 | "ia.seed(1)\n", 168 | "sometimes = lambda aug: iaa.Sometimes(0.5, aug)\n", 169 | "\n", 170 | "# Define our sequence of augmentation steps that will be applied to every image.\n", 171 | "\n", 172 | "seq20 = iaa.Sequential(\n", 173 | " [\n", 174 | " #\n", 175 | " # Apply the following augmenters to most images.\n", 176 | " #\n", 177 | " iaa.Fliplr(0.5), # horizontally flip 50% of all images\n", 178 | "\n", 179 | " # crop some of the images by 0-10% of their height/width\n", 180 | " sometimes(iaa.Crop(percent=(0, 0.1))),\n", 181 | "\n", 182 | " # Apply affine transformations to some of the images\n", 183 | " # - scale to 80-120% of image height/width (each axis independently)\n", 184 | " # - translate by -20 to +20 relative to height/width (per axis)\n", 185 | " # - rotate by -45 to +45 degrees\n", 186 | " # - shear by -16 to +16 degrees\n", 187 | " # - order: use nearest neighbour or bilinear interpolation (fast)\n", 188 | " # - mode: use any available mode to fill newly created pixels\n", 189 | " # see API or scikit-image for which modes are available\n", 190 | " # - cval: if the mode is constant, then use a random brightness\n", 191 | " # for the newly created pixels (e.g. sometimes black,\n", 192 | " # sometimes white)\n", 193 | " sometimes(iaa.Affine(\n", 194 | " scale={\"x\": (0.9, 1.1), \"y\": (0.9, 1.1)},\n", 195 | " translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n", 196 | " rotate=(-10, 10),\n", 197 | " shear=(-4, 4),\n", 198 | " order=[0, 1],\n", 199 | " cval=(0, 255),\n", 200 | " mode=ia.ALL\n", 201 | " )),\n", 202 | "\n", 203 | " #\n", 204 | " # Execute 0 to 5 of the following (less important) augmenters per\n", 205 | " # image. Don't execute all of them, as that would often be way too\n", 206 | " # strong.\n", 207 | " #\n", 208 | " iaa.SomeOf((0, 2),\n", 209 | " [\n", 210 | " iaa.OneOf([\n", 211 | " iaa.GaussianBlur((0, 3.0)),\n", 212 | " iaa.AverageBlur(k=(2, 7)),\n", 213 | " ]),\n", 214 | " iaa.AdditiveGaussianNoise(\n", 215 | " loc=0, scale=(0.0, 0.05*255), per_channel=0.5\n", 216 | " ),\n", 217 | " iaa.OneOf([\n", 218 | " iaa.Dropout((0.01, 0.1), per_channel=0.5),\n", 219 | " iaa.CoarseDropout(\n", 220 | " (0.03, 0.15), size_percent=(0.02, 0.05),\n", 221 | " per_channel=0.2\n", 222 | " ),\n", 223 | " ]),\n", 224 | " sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) ],\n", 225 | " # do all of the above augmentations in random order\n", 226 | " random_order=True\n", 227 | " )\n", 228 | " ],\n", 229 | " # do all of the above augmentations in random order\n", 230 | " random_order=True\n", 231 | ")\n", 232 | "\n", 233 | "seq32 = iaa.Sequential(\n", 234 | " [\n", 235 | " #\n", 236 | " # Apply the following augmenters to most images.\n", 237 | " #\n", 238 | " iaa.Fliplr(0.5), # horizontally flip 50% of all images\n", 239 | "\n", 240 | " # crop some of the images by 0-10% of their height/width\n", 241 | " sometimes(iaa.Crop(percent=(0, 0.1))),\n", 242 | "\n", 243 | " # Apply affine transformations to some of the images\n", 244 | " # - scale to 80-120% of image height/width (each axis independently)\n", 245 | " # - translate by -20 to +20 relative to height/width (per axis)\n", 246 | " # - rotate by -45 to +45 degrees\n", 247 | " # - shear by -16 to +16 degrees\n", 248 | " # - order: use nearest neighbour or bilinear interpolation (fast)\n", 249 | " # - mode: use any available mode to fill newly created pixels\n", 250 | " # see API or scikit-image for which modes are available\n", 251 | " # - cval: if the mode is constant, then use a random brightness\n", 252 | " # for the newly created pixels (e.g. sometimes black,\n", 253 | " # sometimes white)\n", 254 | " sometimes(iaa.Affine(\n", 255 | " scale={\"x\": (0.625, 1), \"y\": (0.625, 1)},\n", 256 | " translate_percent={\"x\": (-0.1, 0.1), \"y\": (-0.1, 0.1)},\n", 257 | " rotate=(-20, 20),\n", 258 | " shear=(-4, 4),\n", 259 | " order=[0, 1],\n", 260 | " cval=(0, 255),\n", 261 | " mode=ia.ALL\n", 262 | " )),\n", 263 | "\n", 264 | " #\n", 265 | " # Execute 0 to 5 of the following (less important) augmenters per\n", 266 | " # image. Don't execute all of them, as that would often be way too\n", 267 | " # strong.\n", 268 | " #\n", 269 | " iaa.SomeOf((0, 4),\n", 270 | " [\n", 271 | " iaa.OneOf([\n", 272 | " iaa.GaussianBlur((0, 3.0)),\n", 273 | " iaa.AverageBlur(k=(2, 7)),\n", 274 | " ]),\n", 275 | " iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),\n", 276 | " iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),\n", 277 | " sometimes(iaa.OneOf([\n", 278 | " iaa.EdgeDetect(alpha=(0, 0.7)),\n", 279 | " iaa.DirectedEdgeDetect(\n", 280 | " alpha=(0, 0.7), direction=(0.0, 1.0)\n", 281 | " ),\n", 282 | " ])),\n", 283 | " iaa.AdditiveGaussianNoise(\n", 284 | " loc=0, scale=(0.0, 0.05*255), per_channel=0.5\n", 285 | " ),\n", 286 | " iaa.OneOf([\n", 287 | " iaa.Dropout((0.01, 0.1), per_channel=0.5),\n", 288 | " iaa.CoarseDropout(\n", 289 | " (0.03, 0.15), size_percent=(0.02, 0.05),\n", 290 | " per_channel=0.2\n", 291 | " ),\n", 292 | " ]),\n", 293 | " iaa.Invert(0.05, per_channel=True), \n", 294 | " iaa.Add((-10, 10), per_channel=0.5),\n", 295 | " iaa.Multiply((0.5, 1.5), per_channel=0.5),\n", 296 | " iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5),\n", 297 | " sometimes(\n", 298 | " iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)\n", 299 | " ),\n", 300 | " sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) ],\n", 301 | " # do all of the above augmentations in random order\n", 302 | " random_order=True\n", 303 | " )\n", 304 | " ],\n", 305 | " # do all of the above augmentations in random order\n", 306 | " random_order=True\n", 307 | ")\n", 308 | "\n", 309 | "\n", 310 | "seq = iaa.Sequential(\n", 311 | " [\n", 312 | " #\n", 313 | " # Apply the following augmenters to most images.\n", 314 | " #\n", 315 | " iaa.Fliplr(0.5), # horizontally flip 50% of all images\n", 316 | "\n", 317 | " # crop some of the images by 0-10% of their height/width\n", 318 | " sometimes(iaa.Crop(percent=(0, 0.1))),\n", 319 | "\n", 320 | " # Apply affine transformations to some of the images\n", 321 | " # - scale to 80-120% of image height/width (each axis independently)\n", 322 | " # - translate by -20 to +20 relative to height/width (per axis)\n", 323 | " # - rotate by -45 to +45 degrees\n", 324 | " # - shear by -16 to +16 degrees\n", 325 | " # - order: use nearest neighbour or bilinear interpolation (fast)\n", 326 | " # - mode: use any available mode to fill newly created pixels\n", 327 | " # see API or scikit-image for which modes are available\n", 328 | " # - cval: if the mode is constant, then use a random brightness\n", 329 | " # for the newly created pixels (e.g. sometimes black,\n", 330 | " # sometimes white)\n", 331 | " sometimes(iaa.Affine(\n", 332 | " scale={\"x\": (0.6, 1), \"y\": (0.6, 1)},\n", 333 | " translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)},\n", 334 | " rotate=(-20, 20),\n", 335 | " shear=(-8, 8),\n", 336 | " order=[0, 1],\n", 337 | " cval=(0, 255),\n", 338 | " mode=ia.ALL\n", 339 | " )),\n", 340 | "\n", 341 | " #\n", 342 | " # Execute 0 to 5 of the following (less important) augmenters per\n", 343 | " # image. Don't execute all of them, as that would often be way too\n", 344 | " # strong.\n", 345 | " #\n", 346 | " iaa.SomeOf((0, 5),\n", 347 | " [\n", 348 | " iaa.OneOf([\n", 349 | " iaa.GaussianBlur((0, 3.0)),\n", 350 | " iaa.AverageBlur(k=(2, 7)),\n", 351 | " ]),\n", 352 | " iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)),\n", 353 | " iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)),\n", 354 | " sometimes(iaa.OneOf([\n", 355 | " iaa.EdgeDetect(alpha=(0, 0.7)),\n", 356 | " iaa.DirectedEdgeDetect(\n", 357 | " alpha=(0, 0.7), direction=(0.0, 1.0)\n", 358 | " ),\n", 359 | " ])),\n", 360 | " iaa.AdditiveGaussianNoise(\n", 361 | " loc=0, scale=(0.0, 0.05*255), per_channel=0.5\n", 362 | " ),\n", 363 | " iaa.OneOf([\n", 364 | " iaa.Dropout((0.01, 0.1), per_channel=0.5),\n", 365 | " iaa.CoarseDropout(\n", 366 | " (0.03, 0.15), size_percent=(0.02, 0.05),\n", 367 | " per_channel=0.2\n", 368 | " ),\n", 369 | " ]),\n", 370 | " iaa.Invert(0.05, per_channel=True), \n", 371 | " iaa.Add((-10, 10), per_channel=0.5),\n", 372 | " iaa.Multiply((0.5, 1.5), per_channel=0.5),\n", 373 | " iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5),\n", 374 | " sometimes(\n", 375 | " iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)\n", 376 | " ),\n", 377 | " sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))) ],\n", 378 | " # do all of the above augmentations in random order\n", 379 | " random_order=True\n", 380 | " )\n", 381 | " ],\n", 382 | " # do all of the above augmentations in random order\n", 383 | " random_order=True\n", 384 | ")\n", 385 | "\n", 386 | "def save_to_drive(drive, epoch, filename): \n", 387 | " title = 'Imagenetvd' + str(epoch) + '.hdf5'\n", 388 | " model_file = drive.CreateFile({'title' : title})\n", 389 | " model_file.SetContentFile(filename)\n", 390 | " model_file.Upload()\n", 391 | " print(\"Saving to id\", model_file.get('id'))\n", 392 | " # download to google drive\n", 393 | " drive.CreateFile({'id': model_file.get('id')}) \n", 394 | " \n", 395 | " \n", 396 | "def get_from_drive(idv):\n", 397 | " global model_file\n", 398 | " gauth = GoogleAuth()\n", 399 | " gauth.credentials = GoogleCredentials.get_application_default()\n", 400 | " drive = GoogleDrive(gauth)\n", 401 | " last_weight_file = drive.CreateFile({'id': idv}) \n", 402 | " last_weight_file.GetContentFile(model_file) \n", 403 | " \n", 404 | "class EarlyStoppingByAccuracy(Callback):\n", 405 | " def __init__(self, monitor='val_acc', mode='max', value=0.98, verbose=0):\n", 406 | " super(Callback, self).__init__()\n", 407 | " self.monitor = monitor\n", 408 | " self.value = value\n", 409 | " self.verbose = verbose\n", 410 | "\n", 411 | " def on_epoch_end(self, epoch, logs={}):\n", 412 | " global model_file\n", 413 | " current = logs.get(self.monitor)\n", 414 | " if current is None:\n", 415 | " warnings.warn(\"Early stopping requires %s available!\" % self.monitor, RuntimeWarning)\n", 416 | " print(\"Current learning rate\", k.get_value(self.model.optimizer.lr), k.get_value(self.model.optimizer.iterations)) \n", 417 | " self.model.save(model_file, overwrite=True)\n", 418 | " epoch = int((k.get_value(self.model.optimizer.iterations))/(int(NUM_IMAGES/BATCH_SIZE)))\n", 419 | " #if(epoch%5==0):\n", 420 | " drive = drive_authenticate()\n", 421 | " save_to_drive(drive, epoch, model_file)\n", 422 | " if current >= self.value:\n", 423 | " if self.verbose > 0:\n", 424 | " print(\"Epoch %05d: early stopping THR\" % epoch)\n", 425 | " self.model.stop_training = True\n", 426 | " \n", 427 | "train_datagen = ImageDataGenerator(\n", 428 | " rescale=1./255,\n", 429 | " featurewise_center=True, featurewise_std_normalization=True,\n", 430 | " preprocessing_function=seq32.augment_image) \n", 431 | "train_datagen.mean = np.array([0.480337294172018, 0.44825522891986064, 0.39783786137898763], dtype=np.float32).reshape((1,1,3))\n", 432 | "train_datagen.std = np.array([0.27676311280482857, 0.26885390556775607, 0.28181860401080205], dtype=np.float32).reshape((1,1,3))\n", 433 | "valid_datagen = ImageDataGenerator(\n", 434 | " rescale=1./255,\n", 435 | " featurewise_center=True, featurewise_std_normalization=True,\n", 436 | " shear_range = 0, # random application of shearing\n", 437 | " zoom_range = 0,\n", 438 | " horizontal_flip = False) \n", 439 | "valid_datagen.mean = np.array([0.480337294172018, 0.44825522891986064, 0.39783786137898763], dtype=np.float32).reshape((1,1,3))\n", 440 | "valid_datagen.std = np.array([0.27676311280482857, 0.26885390556775607, 0.28181860401080205], dtype=np.float32).reshape((1,1,3))\n", 441 | "train_generator = train_datagen.flow_from_directory(\n", 442 | " TRAINING_IMAGES_DIR,\n", 443 | " batch_size=BATCH_SIZE,\n", 444 | " target_size=(IMAGE_SIZE_T, IMAGE_SIZE_T),\n", 445 | " color_mode=\"rgb\",\n", 446 | " class_mode='categorical',\n", 447 | " shuffle=True,\n", 448 | " seed=42)\n", 449 | "STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size\n", 450 | "val_generator = valid_datagen.flow_from_directory(\n", 451 | " directory=VAL_IMAGES_DIR,\n", 452 | " target_size=(IMAGE_SIZE_T, IMAGE_SIZE_T),\n", 453 | " batch_size=BATCH_SIZE,\n", 454 | " color_mode=\"rgb\",\n", 455 | " class_mode=\"categorical\",\n", 456 | " shuffle=True,\n", 457 | " seed=42\n", 458 | ")\n", 459 | "STEP_SIZE_VALID=val_generator.n//val_generator.batch_size \n", 460 | "\n", 461 | "lr_reducer = ReduceLROnPlateau(monitor='val_acc', factor=0.2,\n", 462 | " cooldown=0, patience=5, min_lr=0.0000001)\n", 463 | "model_checkpoint= ModelCheckpoint(model_file, monitor=\"val_acc\", save_best_only=False,\n", 464 | " verbose=1)\n", 465 | "early_stopping = EarlyStoppingByAccuracy(monitor='val_acc', value=0.70, verbose=1)\n", 466 | "csv_logger = CSVLogger('resnet50_imagenet200.csv') " 467 | ] 468 | }, 469 | { 470 | "cell_type": "code", 471 | "execution_count": 0, 472 | "metadata": { 473 | "colab": {}, 474 | "colab_type": "code", 475 | "id": "dryNSKqP3qRv" 476 | }, 477 | "outputs": [], 478 | "source": [ 479 | "def bn_relu_conv(block, num_filter, conv_size):\n", 480 | " block = BatchNormalization(epsilon=1.1e-5)(block)\n", 481 | " block = Activation('relu')(block)\n", 482 | " block = SeparableConv2D(num_filter, conv_size, use_bias=False, padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(0.0001))(block) \n", 483 | " return block\n", 484 | "\n", 485 | "def add_denseblock(input, num_filter, out_filter):\n", 486 | " temp = input\n", 487 | " for _ in range(3): \n", 488 | " block = bn_relu_conv(temp, num_filter, (3, 3))\n", 489 | " concat = Concatenate(axis=-1)([temp,block])\n", 490 | " temp = concat\n", 491 | " out_filter += num_filter\n", 492 | "\n", 493 | " return temp, out_filter\n", 494 | "\n", 495 | "def add_transition(input, num_filter, pool=False):\n", 496 | " block = bn_relu_conv(input, num_filter, (1, 1))\n", 497 | " if pool:\n", 498 | " block = AveragePooling2D(pool_size=(2,2))(block) \n", 499 | " return block, num_filter\n", 500 | "\n", 501 | "def output_layer(input):\n", 502 | " block = bn_relu_conv(input, NUM_CLASSES, (1, 1))\n", 503 | " block = BatchNormalization()(block)\n", 504 | " block = Activation('relu')(block)\n", 505 | " block = GlobalAveragePooling2D()(block)\n", 506 | " output = Softmax(NUM_CLASSES)(block)\n", 507 | " return output" 508 | ] 509 | }, 510 | { 511 | "cell_type": "code", 512 | "execution_count": 0, 513 | "metadata": { 514 | "colab": {}, 515 | "colab_type": "code", 516 | "id": "oQND6issBo-t" 517 | }, 518 | "outputs": [], 519 | "source": [ 520 | "import tensorflow as tf\n", 521 | "def space_to_depth_2(x):\n", 522 | " return tf.space_to_depth(x, block_size=2)\n", 523 | " \n", 524 | "def space_to_depth_4(x):\n", 525 | " return tf.space_to_depth(x, block_size=4) " 526 | ] 527 | }, 528 | { 529 | "cell_type": "code", 530 | "execution_count": 0, 531 | "metadata": { 532 | "colab": {}, 533 | "colab_type": "code", 534 | "id": "DVtjAdI63sC1" 535 | }, 536 | "outputs": [], 537 | "source": [ 538 | "def densenet(num_filter=32):\n", 539 | " block = Conv2D(int(num_filter/2), (3,3), use_bias=False ,padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(0.0001))(input)\n", 540 | " block1, out1 = add_denseblock(block, num_filter, num_filter*2)\n", 541 | " block1, out1 = add_transition(block1, out1)\n", 542 | " block2, out2 = add_denseblock(block1, num_filter, out1)\n", 543 | " block2, out2 = add_transition(block2, out2)\n", 544 | " block3, out3 = add_denseblock(block2, num_filter, out2)\n", 545 | " block3, out3 = add_transition(block3, out3, pool=True)\n", 546 | " block4, out4 = add_denseblock(block3, num_filter, out3)\n", 547 | " block4, out4 = add_transition(block4, out4, pool=True)\n", 548 | " c1 = bn_relu_conv(block2, 64, (1,1))\n", 549 | " c1 = Lambda(space_to_depth_4)(c1)\n", 550 | " c2 = bn_relu_conv(block3, 256, (1,1))\n", 551 | " c2 = Lambda(space_to_depth_2)(c2)\n", 552 | " block = Concatenate(axis=-1)([c1,c2,block4])\n", 553 | " block = bn_relu_conv(block, NUM_CLASSES*8, (1,1))\n", 554 | " block = bn_relu_conv(block, NUM_CLASSES*4, (3,3)) \n", 555 | " block = bn_relu_conv(block, NUM_CLASSES*2, (3,3)) \n", 556 | " output = output_layer(block) \n", 557 | " model = Model(inputs=[input], outputs=[output])\n", 558 | " return model" 559 | ] 560 | }, 561 | { 562 | "cell_type": "code", 563 | "execution_count": 0, 564 | "metadata": { 565 | "colab": { 566 | "base_uri": "https://localhost:8080/", 567 | "height": 3360 568 | }, 569 | "colab_type": "code", 570 | "id": "AX5-NpZC3vRB", 571 | "outputId": "80393c89-b2c5-4b2a-a08f-db8c8d85036d" 572 | }, 573 | "outputs": [ 574 | { 575 | "name": "stdout", 576 | "output_type": "stream", 577 | "text": [ 578 | "__________________________________________________________________________________________________\n", 579 | "Layer (type) Output Shape Param # Connected to \n", 580 | "==================================================================================================\n", 581 | "input_2 (InputLayer) (None, None, None, 3 0 \n", 582 | "__________________________________________________________________________________________________\n", 583 | "conv2d_2 (Conv2D) (None, None, None, 3 864 input_2[0][0] \n", 584 | "__________________________________________________________________________________________________\n", 585 | "batch_normalization_24 (BatchNo (None, None, None, 3 128 conv2d_2[0][0] \n", 586 | "__________________________________________________________________________________________________\n", 587 | "activation_24 (Activation) (None, None, None, 3 0 batch_normalization_24[0][0] \n", 588 | "__________________________________________________________________________________________________\n", 589 | "separable_conv2d_23 (SeparableC (None, None, None, 6 2336 activation_24[0][0] \n", 590 | "__________________________________________________________________________________________________\n", 591 | "concatenate_14 (Concatenate) (None, None, None, 9 0 conv2d_2[0][0] \n", 592 | " separable_conv2d_23[0][0] \n", 593 | "__________________________________________________________________________________________________\n", 594 | "batch_normalization_25 (BatchNo (None, None, None, 9 384 concatenate_14[0][0] \n", 595 | "__________________________________________________________________________________________________\n", 596 | "activation_25 (Activation) (None, None, None, 9 0 batch_normalization_25[0][0] \n", 597 | "__________________________________________________________________________________________________\n", 598 | "separable_conv2d_24 (SeparableC (None, None, None, 6 7008 activation_25[0][0] \n", 599 | "__________________________________________________________________________________________________\n", 600 | "concatenate_15 (Concatenate) (None, None, None, 1 0 concatenate_14[0][0] \n", 601 | " separable_conv2d_24[0][0] \n", 602 | "__________________________________________________________________________________________________\n", 603 | "batch_normalization_26 (BatchNo (None, None, None, 1 640 concatenate_15[0][0] \n", 604 | "__________________________________________________________________________________________________\n", 605 | "activation_26 (Activation) (None, None, None, 1 0 batch_normalization_26[0][0] \n", 606 | "__________________________________________________________________________________________________\n", 607 | "separable_conv2d_25 (SeparableC (None, None, None, 6 11680 activation_26[0][0] \n", 608 | "__________________________________________________________________________________________________\n", 609 | "concatenate_16 (Concatenate) (None, None, None, 2 0 concatenate_15[0][0] \n", 610 | " separable_conv2d_25[0][0] \n", 611 | "__________________________________________________________________________________________________\n", 612 | "batch_normalization_27 (BatchNo (None, None, None, 2 896 concatenate_16[0][0] \n", 613 | "__________________________________________________________________________________________________\n", 614 | "activation_27 (Activation) (None, None, None, 2 0 batch_normalization_27[0][0] \n", 615 | "__________________________________________________________________________________________________\n", 616 | "separable_conv2d_26 (SeparableC (None, None, None, 3 71904 activation_27[0][0] \n", 617 | "__________________________________________________________________________________________________\n", 618 | "batch_normalization_28 (BatchNo (None, None, None, 3 1280 separable_conv2d_26[0][0] \n", 619 | "__________________________________________________________________________________________________\n", 620 | "activation_28 (Activation) (None, None, None, 3 0 batch_normalization_28[0][0] \n", 621 | "__________________________________________________________________________________________________\n", 622 | "separable_conv2d_27 (SeparableC (None, None, None, 6 23360 activation_28[0][0] \n", 623 | "__________________________________________________________________________________________________\n", 624 | "concatenate_17 (Concatenate) (None, None, None, 3 0 separable_conv2d_26[0][0] \n", 625 | " separable_conv2d_27[0][0] \n", 626 | "__________________________________________________________________________________________________\n", 627 | "batch_normalization_29 (BatchNo (None, None, None, 3 1536 concatenate_17[0][0] \n", 628 | "__________________________________________________________________________________________________\n", 629 | "activation_29 (Activation) (None, None, None, 3 0 batch_normalization_29[0][0] \n", 630 | "__________________________________________________________________________________________________\n", 631 | "separable_conv2d_28 (SeparableC (None, None, None, 6 28032 activation_29[0][0] \n", 632 | "__________________________________________________________________________________________________\n", 633 | "concatenate_18 (Concatenate) (None, None, None, 4 0 concatenate_17[0][0] \n", 634 | " separable_conv2d_28[0][0] \n", 635 | "__________________________________________________________________________________________________\n", 636 | "batch_normalization_30 (BatchNo (None, None, None, 4 1792 concatenate_18[0][0] \n", 637 | "__________________________________________________________________________________________________\n", 638 | "activation_30 (Activation) (None, None, None, 4 0 batch_normalization_30[0][0] \n", 639 | "__________________________________________________________________________________________________\n", 640 | "separable_conv2d_29 (SeparableC (None, None, None, 6 32704 activation_30[0][0] \n", 641 | "__________________________________________________________________________________________________\n", 642 | "concatenate_19 (Concatenate) (None, None, None, 5 0 concatenate_18[0][0] \n", 643 | " separable_conv2d_29[0][0] \n", 644 | "__________________________________________________________________________________________________\n", 645 | "batch_normalization_31 (BatchNo (None, None, None, 5 2048 concatenate_19[0][0] \n", 646 | "__________________________________________________________________________________________________\n", 647 | "activation_31 (Activation) (None, None, None, 5 0 batch_normalization_31[0][0] \n", 648 | "__________________________________________________________________________________________________\n", 649 | "separable_conv2d_30 (SeparableC (None, None, None, 5 262656 activation_31[0][0] \n", 650 | "__________________________________________________________________________________________________\n", 651 | "batch_normalization_32 (BatchNo (None, None, None, 5 2048 separable_conv2d_30[0][0] \n", 652 | "__________________________________________________________________________________________________\n", 653 | "activation_32 (Activation) (None, None, None, 5 0 batch_normalization_32[0][0] \n", 654 | "__________________________________________________________________________________________________\n", 655 | "separable_conv2d_31 (SeparableC (None, None, None, 6 37376 activation_32[0][0] \n", 656 | "__________________________________________________________________________________________________\n", 657 | "concatenate_20 (Concatenate) (None, None, None, 5 0 separable_conv2d_30[0][0] \n", 658 | " separable_conv2d_31[0][0] \n", 659 | "__________________________________________________________________________________________________\n", 660 | "batch_normalization_33 (BatchNo (None, None, None, 5 2304 concatenate_20[0][0] \n", 661 | "__________________________________________________________________________________________________\n", 662 | "activation_33 (Activation) (None, None, None, 5 0 batch_normalization_33[0][0] \n", 663 | "__________________________________________________________________________________________________\n", 664 | "separable_conv2d_32 (SeparableC (None, None, None, 6 42048 activation_33[0][0] \n", 665 | "__________________________________________________________________________________________________\n", 666 | "concatenate_21 (Concatenate) (None, None, None, 6 0 concatenate_20[0][0] \n", 667 | " separable_conv2d_32[0][0] \n", 668 | "__________________________________________________________________________________________________\n", 669 | "batch_normalization_34 (BatchNo (None, None, None, 6 2560 concatenate_21[0][0] \n", 670 | "__________________________________________________________________________________________________\n", 671 | "activation_34 (Activation) (None, None, None, 6 0 batch_normalization_34[0][0] \n", 672 | "__________________________________________________________________________________________________\n", 673 | "separable_conv2d_33 (SeparableC (None, None, None, 6 46720 activation_34[0][0] \n", 674 | "__________________________________________________________________________________________________\n", 675 | "concatenate_22 (Concatenate) (None, None, None, 7 0 concatenate_21[0][0] \n", 676 | " separable_conv2d_33[0][0] \n", 677 | "__________________________________________________________________________________________________\n", 678 | "batch_normalization_35 (BatchNo (None, None, None, 7 2816 concatenate_22[0][0] \n", 679 | "__________________________________________________________________________________________________\n", 680 | "activation_35 (Activation) (None, None, None, 7 0 batch_normalization_35[0][0] \n", 681 | "__________________________________________________________________________________________________\n", 682 | "separable_conv2d_34 (SeparableC (None, None, None, 7 496320 activation_35[0][0] \n", 683 | "__________________________________________________________________________________________________\n", 684 | "average_pooling2d_3 (AveragePoo (None, None, None, 7 0 separable_conv2d_34[0][0] \n", 685 | "__________________________________________________________________________________________________\n", 686 | "batch_normalization_36 (BatchNo (None, None, None, 7 2816 average_pooling2d_3[0][0] \n", 687 | "__________________________________________________________________________________________________\n", 688 | "activation_36 (Activation) (None, None, None, 7 0 batch_normalization_36[0][0] \n", 689 | "__________________________________________________________________________________________________\n", 690 | "separable_conv2d_35 (SeparableC (None, None, None, 6 51392 activation_36[0][0] \n", 691 | "__________________________________________________________________________________________________\n", 692 | "concatenate_23 (Concatenate) (None, None, None, 7 0 average_pooling2d_3[0][0] \n", 693 | " separable_conv2d_35[0][0] \n", 694 | "__________________________________________________________________________________________________\n", 695 | "batch_normalization_37 (BatchNo (None, None, None, 7 3072 concatenate_23[0][0] \n", 696 | "__________________________________________________________________________________________________\n", 697 | "activation_37 (Activation) (None, None, None, 7 0 batch_normalization_37[0][0] \n", 698 | "__________________________________________________________________________________________________\n", 699 | "separable_conv2d_36 (SeparableC (None, None, None, 6 56064 activation_37[0][0] \n", 700 | "__________________________________________________________________________________________________\n", 701 | "concatenate_24 (Concatenate) (None, None, None, 8 0 concatenate_23[0][0] \n", 702 | " separable_conv2d_36[0][0] \n", 703 | "__________________________________________________________________________________________________\n", 704 | "batch_normalization_38 (BatchNo (None, None, None, 8 3328 concatenate_24[0][0] \n", 705 | "__________________________________________________________________________________________________\n", 706 | "activation_38 (Activation) (None, None, None, 8 0 batch_normalization_38[0][0] \n", 707 | "__________________________________________________________________________________________________\n", 708 | "separable_conv2d_37 (SeparableC (None, None, None, 6 60736 activation_38[0][0] \n", 709 | "__________________________________________________________________________________________________\n", 710 | "concatenate_25 (Concatenate) (None, None, None, 8 0 concatenate_24[0][0] \n", 711 | " separable_conv2d_37[0][0] \n", 712 | "__________________________________________________________________________________________________\n", 713 | "batch_normalization_40 (BatchNo (None, None, None, 5 2048 separable_conv2d_30[0][0] \n", 714 | "__________________________________________________________________________________________________\n", 715 | "batch_normalization_41 (BatchNo (None, None, None, 7 2816 average_pooling2d_3[0][0] \n", 716 | "__________________________________________________________________________________________________\n", 717 | "batch_normalization_39 (BatchNo (None, None, None, 8 3584 concatenate_25[0][0] \n", 718 | "__________________________________________________________________________________________________\n", 719 | "activation_40 (Activation) (None, None, None, 5 0 batch_normalization_40[0][0] \n", 720 | "__________________________________________________________________________________________________\n", 721 | "activation_41 (Activation) (None, None, None, 7 0 batch_normalization_41[0][0] \n", 722 | "__________________________________________________________________________________________________\n", 723 | "activation_39 (Activation) (None, None, None, 8 0 batch_normalization_39[0][0] \n", 724 | "__________________________________________________________________________________________________\n", 725 | "separable_conv2d_39 (SeparableC (None, None, None, 6 33280 activation_40[0][0] \n", 726 | "__________________________________________________________________________________________________\n", 727 | "separable_conv2d_40 (SeparableC (None, None, None, 2 180928 activation_41[0][0] \n", 728 | "__________________________________________________________________________________________________\n", 729 | "separable_conv2d_38 (SeparableC (None, None, None, 8 803712 activation_39[0][0] \n", 730 | "__________________________________________________________________________________________________\n", 731 | "lambda_3 (Lambda) (None, None, None, 1 0 separable_conv2d_39[0][0] \n", 732 | "__________________________________________________________________________________________________\n", 733 | "lambda_4 (Lambda) (None, None, None, 1 0 separable_conv2d_40[0][0] \n", 734 | "__________________________________________________________________________________________________\n", 735 | "average_pooling2d_4 (AveragePoo (None, None, None, 8 0 separable_conv2d_38[0][0] \n", 736 | "__________________________________________________________________________________________________\n", 737 | "concatenate_26 (Concatenate) (None, None, None, 2 0 lambda_3[0][0] \n", 738 | " lambda_4[0][0] \n", 739 | " average_pooling2d_4[0][0] \n", 740 | "__________________________________________________________________________________________________\n", 741 | "batch_normalization_42 (BatchNo (None, None, None, 2 11776 concatenate_26[0][0] \n", 742 | "__________________________________________________________________________________________________\n", 743 | "activation_42 (Activation) (None, None, None, 2 0 batch_normalization_42[0][0] \n", 744 | "__________________________________________________________________________________________________\n", 745 | "separable_conv2d_41 (SeparableC (None, None, None, 1 4713344 activation_42[0][0] \n", 746 | "__________________________________________________________________________________________________\n", 747 | "batch_normalization_43 (BatchNo (None, None, None, 1 6400 separable_conv2d_41[0][0] \n", 748 | "__________________________________________________________________________________________________\n", 749 | "activation_43 (Activation) (None, None, None, 1 0 batch_normalization_43[0][0] \n", 750 | "__________________________________________________________________________________________________\n", 751 | "separable_conv2d_42 (SeparableC (None, None, None, 8 1294400 activation_43[0][0] \n", 752 | "__________________________________________________________________________________________________\n", 753 | "batch_normalization_44 (BatchNo (None, None, None, 8 3200 separable_conv2d_42[0][0] \n", 754 | "__________________________________________________________________________________________________\n", 755 | "activation_44 (Activation) (None, None, None, 8 0 batch_normalization_44[0][0] \n", 756 | "__________________________________________________________________________________________________\n", 757 | "separable_conv2d_43 (SeparableC (None, None, None, 4 327200 activation_44[0][0] \n", 758 | "__________________________________________________________________________________________________\n", 759 | "batch_normalization_45 (BatchNo (None, None, None, 4 1600 separable_conv2d_43[0][0] \n", 760 | "__________________________________________________________________________________________________\n", 761 | "activation_45 (Activation) (None, None, None, 4 0 batch_normalization_45[0][0] \n", 762 | "__________________________________________________________________________________________________\n", 763 | "separable_conv2d_44 (SeparableC (None, None, None, 2 80400 activation_45[0][0] \n", 764 | "__________________________________________________________________________________________________\n", 765 | "batch_normalization_46 (BatchNo (None, None, None, 2 800 separable_conv2d_44[0][0] \n", 766 | "__________________________________________________________________________________________________\n", 767 | "activation_46 (Activation) (None, None, None, 2 0 batch_normalization_46[0][0] \n", 768 | "__________________________________________________________________________________________________\n", 769 | "global_average_pooling2d_2 (Glo (None, 200) 0 activation_46[0][0] \n", 770 | "__________________________________________________________________________________________________\n", 771 | "softmax_2 (Softmax) (None, 200) 0 global_average_pooling2d_2[0][0] \n", 772 | "==================================================================================================\n", 773 | "Total params: 8,724,336\n", 774 | "Trainable params: 8,694,400\n", 775 | "Non-trainable params: 29,936\n", 776 | "__________________________________________________________________________________________________\n" 777 | ] 778 | } 779 | ], 780 | "source": [ 781 | "from keras.layers.advanced_activations import Softmax\n", 782 | "from keras.layers import Dense, Dropout, Flatten, Input, AveragePooling2D, merge, Activation, GlobalAveragePooling2D, SeparableConv2D\n", 783 | "from keras.layers import Reshape, Activation, Conv2D, Input, MaxPooling2D, BatchNormalization, Flatten, Dense, Lambda\n", 784 | "from keras.models import Model\n", 785 | "from keras.optimizers import Adam\n", 786 | "from keras.regularizers import l2\n", 787 | "input = Input(shape=(None, None, NUM_CHANNELS,))\n", 788 | "model = densenet(num_filter=64)\n", 789 | "model.summary()\n", 790 | "model.compile(loss='categorical_crossentropy',\n", 791 | " optimizer=Adam(lr=0.001),\n", 792 | " metrics=['accuracy'])" 793 | ] 794 | }, 795 | { 796 | "cell_type": "code", 797 | "execution_count": 0, 798 | "metadata": { 799 | "colab": { 800 | "base_uri": "https://localhost:8080/", 801 | "height": 1254 802 | }, 803 | "colab_type": "code", 804 | "id": "egp1Echg3xUA", 805 | "outputId": "4e7b9a8a-b66c-45e2-ebfe-f721ac7ebe33" 806 | }, 807 | "outputs": [ 808 | { 809 | "name": "stdout", 810 | "output_type": "stream", 811 | "text": [ 812 | "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", 813 | "Instructions for updating:\n", 814 | "Use tf.cast instead.\n", 815 | "Epoch 1/50\n", 816 | "1562/1562 [==============================] - 1592s 1s/step - loss: 4.8105 - acc: 0.0605 - val_loss: 4.3155 - val_acc: 0.1087\n", 817 | "Current learning rate 0.001 1562\n", 818 | "\n", 819 | "Epoch 00001: saving model to Imagenetvd.hdf5\n", 820 | "Epoch 2/50\n", 821 | "1562/1562 [==============================] - 1580s 1s/step - loss: 4.2235 - acc: 0.1411 - val_loss: 3.8027 - val_acc: 0.1886\n", 822 | "Current learning rate 0.001 3124\n", 823 | "\n", 824 | "Epoch 00002: saving model to Imagenetvd.hdf5\n", 825 | "Epoch 3/50\n", 826 | "1562/1562 [==============================] - 1576s 1s/step - loss: 3.8336 - acc: 0.1946 - val_loss: 3.4333 - val_acc: 0.2470\n", 827 | "Current learning rate 0.001 4686\n", 828 | "\n", 829 | "Epoch 00003: saving model to Imagenetvd.hdf5\n", 830 | "Epoch 4/50\n", 831 | "1562/1562 [==============================] - 1577s 1s/step - loss: 3.5559 - acc: 0.2360 - val_loss: 3.2340 - val_acc: 0.2766\n", 832 | "Current learning rate 0.001 6248\n", 833 | "\n", 834 | "Epoch 00004: saving model to Imagenetvd.hdf5\n", 835 | "Epoch 5/50\n", 836 | "1562/1562 [==============================] - 1578s 1s/step - loss: 3.3554 - acc: 0.2684 - val_loss: 3.1185 - val_acc: 0.3032\n", 837 | "Current learning rate 0.001 7810\n" 838 | ] 839 | }, 840 | { 841 | "ename": "NameError", 842 | "evalue": "ignored", 843 | "output_type": "error", 844 | "traceback": [ 845 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 846 | "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 847 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSTEP_SIZE_VALID\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNUM_EPOCHS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])\n\u001b[0m", 848 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 89\u001b[0m warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[1;32m 90\u001b[0m 'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 92\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 849 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 1416\u001b[0m \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1417\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1418\u001b[0;31m initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m 1419\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1420\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 850 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 249\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 250\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 251\u001b[0;31m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 252\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 253\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallback_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 851 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/callbacks.py\u001b[0m in \u001b[0;36mon_epoch_end\u001b[0;34m(self, epoch, logs)\u001b[0m\n\u001b[1;32m 77\u001b[0m \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlogs\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 78\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 79\u001b[0;31m \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 80\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 852 | "\u001b[0;32m\u001b[0m in \u001b[0;36mon_epoch_end\u001b[0;34m(self, epoch, logs)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miterations\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mNUM_IMAGES\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 220\u001b[0;31m \u001b[0mdrive\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdrive_authenticate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 221\u001b[0m \u001b[0msave_to_drive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdrive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcurrent\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 853 | "\u001b[0;31mNameError\u001b[0m: name 'drive_authenticate' is not defined" 854 | ] 855 | } 856 | ], 857 | "source": [ 858 | "model.fit_generator(generator=train_generator,\n", 859 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 860 | " validation_data=val_generator,\n", 861 | " validation_steps=STEP_SIZE_VALID,\n", 862 | " epochs=NUM_EPOCHS, verbose=1,\n", 863 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 864 | ] 865 | }, 866 | { 867 | "cell_type": "code", 868 | "execution_count": 0, 869 | "metadata": { 870 | "colab": { 871 | "base_uri": "https://localhost:8080/", 872 | "height": 1260 873 | }, 874 | "colab_type": "code", 875 | "id": "Pz6okkTLogxI", 876 | "outputId": "4f4aad10-ad25-4635-bfca-469dada172e2" 877 | }, 878 | "outputs": [ 879 | { 880 | "name": "stdout", 881 | "output_type": "stream", 882 | "text": [ 883 | "Epoch 6/50\n", 884 | "1562/1562 [==============================] - 1587s 1s/step - loss: 3.1948 - acc: 0.2952 - val_loss: 2.9732 - val_acc: 0.3278\n", 885 | "Current learning rate 0.001 9372\n", 886 | "\n", 887 | "Epoch 00006: saving model to Imagenetvd.hdf5\n", 888 | "Epoch 7/50\n", 889 | "1562/1562 [==============================] - 1585s 1s/step - loss: 3.0575 - acc: 0.3168 - val_loss: 2.7520 - val_acc: 0.3686\n", 890 | "Current learning rate 0.001 10934\n", 891 | "\n", 892 | "Epoch 00007: saving model to Imagenetvd.hdf5\n", 893 | "Epoch 8/50\n", 894 | "1562/1562 [==============================] - 1581s 1s/step - loss: 2.9428 - acc: 0.3387 - val_loss: 2.6633 - val_acc: 0.3871\n", 895 | "Current learning rate 0.001 12496\n", 896 | "\n", 897 | "Epoch 00008: saving model to Imagenetvd.hdf5\n", 898 | "Epoch 9/50\n", 899 | "1562/1562 [==============================] - 1579s 1s/step - loss: 2.8363 - acc: 0.3583 - val_loss: 2.7351 - val_acc: 0.3737\n", 900 | "Current learning rate 0.001 14058\n", 901 | "\n", 902 | "Epoch 00009: saving model to Imagenetvd.hdf5\n", 903 | "Epoch 10/50\n", 904 | "1562/1562 [==============================] - 1575s 1s/step - loss: 2.7361 - acc: 0.3767 - val_loss: 2.6234 - val_acc: 0.3937\n", 905 | "Current learning rate 0.001 15620\n", 906 | "Saving to id 1RcSoWtoolBgk-U8z7upzFUeP5KT8v3d5\n", 907 | "\n", 908 | "Epoch 00010: saving model to Imagenetvd.hdf5\n", 909 | "Epoch 11/50\n", 910 | "1562/1562 [==============================] - 1577s 1s/step - loss: 2.6561 - acc: 0.3922 - val_loss: 2.5113 - val_acc: 0.4150\n", 911 | "Current learning rate 0.001 17182\n", 912 | "\n", 913 | "Epoch 00011: saving model to Imagenetvd.hdf5\n", 914 | "Epoch 12/50\n", 915 | "1562/1562 [==============================] - 1574s 1s/step - loss: 2.5775 - acc: 0.4067 - val_loss: 2.4428 - val_acc: 0.4308\n", 916 | "Current learning rate 0.001 18744\n", 917 | "\n", 918 | "Epoch 00012: saving model to Imagenetvd.hdf5\n", 919 | "Epoch 13/50\n", 920 | "1562/1562 [==============================] - 1575s 1s/step - loss: 2.5041 - acc: 0.4196 - val_loss: 2.4277 - val_acc: 0.4355\n", 921 | "Current learning rate 0.001 20306\n", 922 | "\n", 923 | "Epoch 00013: saving model to Imagenetvd.hdf5\n", 924 | "Epoch 14/50\n", 925 | "1562/1562 [==============================] - 1573s 1s/step - loss: 2.4260 - acc: 0.4355 - val_loss: 2.4518 - val_acc: 0.4330\n", 926 | "Current learning rate 0.001 21868\n", 927 | "\n", 928 | "Epoch 00014: saving model to Imagenetvd.hdf5\n", 929 | "Epoch 15/50\n", 930 | "1562/1562 [==============================] - 1564s 1s/step - loss: 2.3595 - acc: 0.4479 - val_loss: 2.3932 - val_acc: 0.4416\n", 931 | "Current learning rate 0.001 23430\n", 932 | "Saving to id 1xd_cUTGNRavpjYFrmin52dEXyO23UQIm\n", 933 | "\n", 934 | "Epoch 00015: saving model to Imagenetvd.hdf5\n", 935 | "Epoch 16/50\n", 936 | "1562/1562 [==============================] - 1560s 999ms/step - loss: 2.2924 - acc: 0.4622 - val_loss: 2.3120 - val_acc: 0.4578\n", 937 | "Current learning rate 0.001 24992\n", 938 | "\n", 939 | "Epoch 00016: saving model to Imagenetvd.hdf5\n", 940 | "Epoch 17/50\n", 941 | "1562/1562 [==============================] - 1559s 998ms/step - loss: 2.2273 - acc: 0.4755 - val_loss: 2.3026 - val_acc: 0.4611\n", 942 | "Current learning rate 0.001 26554\n", 943 | "\n", 944 | "Epoch 00017: saving model to Imagenetvd.hdf5\n", 945 | "Epoch 18/50\n", 946 | "1562/1562 [==============================] - 1559s 998ms/step - loss: 2.1666 - acc: 0.4878 - val_loss: 2.3460 - val_acc: 0.4541\n", 947 | "Current learning rate 0.001 28116\n", 948 | "\n", 949 | "Epoch 00018: saving model to Imagenetvd.hdf5\n", 950 | "Epoch 19/50\n", 951 | "1562/1562 [==============================] - 1559s 998ms/step - loss: 2.1177 - acc: 0.4952 - val_loss: 2.3441 - val_acc: 0.4584\n", 952 | "Current learning rate 0.001 29678\n", 953 | "\n", 954 | "Epoch 00019: saving model to Imagenetvd.hdf5\n", 955 | "Epoch 20/50\n", 956 | "1500/1562 [===========================>..] - ETA: 1:00 - loss: 2.0555 - acc: 0.5098" 957 | ] 958 | } 959 | ], 960 | "source": [ 961 | "model.fit_generator(initial_epoch=5,\n", 962 | " generator=train_generator,\n", 963 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 964 | " validation_data=val_generator,\n", 965 | " validation_steps=STEP_SIZE_VALID,\n", 966 | " epochs=NUM_EPOCHS, verbose=1,\n", 967 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 968 | ] 969 | }, 970 | { 971 | "cell_type": "code", 972 | "execution_count": 0, 973 | "metadata": { 974 | "colab": { 975 | "base_uri": "https://localhost:8080/", 976 | "height": 87 977 | }, 978 | "colab_type": "code", 979 | "id": "lyXQliKeqWUw", 980 | "outputId": "da035c7b-9b09-461c-9c79-7fbbeb38a26a" 981 | }, 982 | "outputs": [ 983 | { 984 | "name": "stdout", 985 | "output_type": "stream", 986 | "text": [ 987 | "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", 988 | "Instructions for updating:\n", 989 | "Use tf.cast instead.\n" 990 | ] 991 | } 992 | ], 993 | "source": [ 994 | "from keras.models import load_model\n", 995 | "!cp drive/'My Drive'/Imagenetvd15.hdf5 .\n", 996 | "model = load_model('Imagenetvd15.hdf5', custom_objects={\"tf\": tf})" 997 | ] 998 | }, 999 | { 1000 | "cell_type": "code", 1001 | "execution_count": 0, 1002 | "metadata": { 1003 | "colab": { 1004 | "base_uri": "https://localhost:8080/", 1005 | "height": 470 1006 | }, 1007 | "colab_type": "code", 1008 | "id": "XVS_GVIRI7Dg", 1009 | "outputId": "8883860d-b4e3-4bd2-bc5c-aaee209c1ad4" 1010 | }, 1011 | "outputs": [ 1012 | { 1013 | "name": "stdout", 1014 | "output_type": "stream", 1015 | "text": [ 1016 | "Epoch 16/20\n", 1017 | "781/781 [==============================] - 1001s 1s/step - loss: 2.2861 - acc: 0.4626 - val_loss: 2.8079 - val_acc: 0.3857\n", 1018 | "Current learning rate 0.001 25370\n", 1019 | "\n", 1020 | "Epoch 00016: saving model to Imagenetvd.hdf5\n", 1021 | "Epoch 17/20\n", 1022 | "781/781 [==============================] - 1007s 1s/step - loss: 2.5634 - acc: 0.4003 - val_loss: 2.6938 - val_acc: 0.3931\n", 1023 | "Current learning rate 0.001 26151\n", 1024 | "\n", 1025 | "Epoch 00017: saving model to Imagenetvd.hdf5\n", 1026 | "Epoch 18/20\n", 1027 | "781/781 [==============================] - 998s 1s/step - loss: 2.4797 - acc: 0.4150 - val_loss: 2.6603 - val_acc: 0.3956\n", 1028 | "Current learning rate 0.001 26932\n", 1029 | "\n", 1030 | "Epoch 00018: saving model to Imagenetvd.hdf5\n", 1031 | "Epoch 19/20\n", 1032 | "781/781 [==============================] - 1003s 1s/step - loss: 2.4064 - acc: 0.4307 - val_loss: 2.6281 - val_acc: 0.4009\n", 1033 | "Current learning rate 0.001 27713\n", 1034 | "Saving to id 1IGJWcVW-iiEjZzKaz4Y0rfK_2o9gW4Dt\n", 1035 | "\n", 1036 | "Epoch 00019: saving model to Imagenetvd.hdf5\n", 1037 | "Epoch 20/20\n", 1038 | "781/781 [==============================] - 989s 1s/step - loss: 2.3469 - acc: 0.4427 - val_loss: 2.6095 - val_acc: 0.4119\n", 1039 | "Current learning rate 0.001 28494\n", 1040 | "\n", 1041 | "Epoch 00020: saving model to Imagenetvd.hdf5\n" 1042 | ] 1043 | }, 1044 | { 1045 | "data": { 1046 | "text/plain": [ 1047 | "" 1048 | ] 1049 | }, 1050 | "execution_count": 30, 1051 | "metadata": { 1052 | "tags": [] 1053 | }, 1054 | "output_type": "execute_result" 1055 | } 1056 | ], 1057 | "source": [ 1058 | "#Loaded checkpoint from epoch 15, Training with 20x20\n", 1059 | "model.fit_generator(initial_epoch=15,\n", 1060 | " generator=train_generator,\n", 1061 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 1062 | " validation_data=val_generator,\n", 1063 | " validation_steps=STEP_SIZE_VALID,\n", 1064 | " epochs=20, verbose=1,\n", 1065 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 1066 | ] 1067 | }, 1068 | { 1069 | "cell_type": "code", 1070 | "execution_count": 0, 1071 | "metadata": { 1072 | "colab": {}, 1073 | "colab_type": "code", 1074 | "id": "UayVzRr2mwP3" 1075 | }, 1076 | "outputs": [], 1077 | "source": [ 1078 | "from keras.models import load_model\n", 1079 | "!cp drive/'My Drive'/Imagenetvd20.hdf5 .\n", 1080 | "model = load_model('Imagenetvd20.hdf5', custom_objects={\"tf\": tf})" 1081 | ] 1082 | }, 1083 | { 1084 | "cell_type": "code", 1085 | "execution_count": 0, 1086 | "metadata": { 1087 | "colab": { 1088 | "base_uri": "https://localhost:8080/", 1089 | "height": 218 1090 | }, 1091 | "colab_type": "code", 1092 | "id": "lnDLHABGJ2Rf", 1093 | "outputId": "c9c8745e-ef41-4070-ecae-ccdcf9e44cb8" 1094 | }, 1095 | "outputs": [ 1096 | { 1097 | "name": "stdout", 1098 | "output_type": "stream", 1099 | "text": [ 1100 | "Epoch 21/22\n", 1101 | "1562/1562 [==============================] - 1553s 994ms/step - loss: 2.7223 - acc: 0.3797 - val_loss: 2.3829 - val_acc: 0.4442\n", 1102 | "Current learning rate 0.001 30140\n", 1103 | "\n", 1104 | "Epoch 00021: saving model to Imagenetvd.hdf5\n", 1105 | "Epoch 22/22\n", 1106 | "1562/1562 [==============================] - 1563s 1s/step - loss: 2.6052 - acc: 0.4011 - val_loss: 2.3326 - val_acc: 0.4622\n", 1107 | "Current learning rate 0.001 31702\n", 1108 | "Saving to id 1Jn3aYMi7JlFesaJ01RqDcNRhcmhf6Mph\n", 1109 | "\n", 1110 | "Epoch 00022: saving model to Imagenetvd.hdf5\n" 1111 | ] 1112 | }, 1113 | { 1114 | "data": { 1115 | "text/plain": [ 1116 | "" 1117 | ] 1118 | }, 1119 | "execution_count": 32, 1120 | "metadata": { 1121 | "tags": [] 1122 | }, 1123 | "output_type": "execute_result" 1124 | } 1125 | ], 1126 | "source": [ 1127 | "#Training on 32x32\n", 1128 | "model.fit_generator(initial_epoch=20,\n", 1129 | " generator=train_generator,\n", 1130 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 1131 | " validation_data=val_generator,\n", 1132 | " validation_steps=STEP_SIZE_VALID,\n", 1133 | " epochs=22, verbose=1,\n", 1134 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 1135 | ] 1136 | }, 1137 | { 1138 | "cell_type": "code", 1139 | "execution_count": 0, 1140 | "metadata": { 1141 | "colab": { 1142 | "base_uri": "https://localhost:8080/", 1143 | "height": 286 1144 | }, 1145 | "colab_type": "code", 1146 | "id": "1KdJi7v4mPqf", 1147 | "outputId": "a4aca2d6-0f22-431d-861f-154b27d229a7" 1148 | }, 1149 | "outputs": [ 1150 | { 1151 | "name": "stdout", 1152 | "output_type": "stream", 1153 | "text": [ 1154 | "Epoch 23/25\n", 1155 | "1562/1562 [==============================] - 1561s 1000ms/step - loss: 2.5498 - acc: 0.4108 - val_loss: 2.3393 - val_acc: 0.4516\n", 1156 | "Current learning rate 0.001 33264\n", 1157 | "\n", 1158 | "Epoch 00023: saving model to Imagenetvd.hdf5\n", 1159 | "Epoch 24/25\n", 1160 | "1562/1562 [==============================] - 1555s 996ms/step - loss: 2.5113 - acc: 0.4195 - val_loss: 2.3512 - val_acc: 0.4597\n", 1161 | "Current learning rate 0.001 34826\n", 1162 | "\n", 1163 | "Epoch 00024: saving model to Imagenetvd.hdf5\n", 1164 | "Epoch 25/25\n", 1165 | "1562/1562 [==============================] - 1555s 996ms/step - loss: 2.4594 - acc: 0.4290 - val_loss: 2.3073 - val_acc: 0.4688\n", 1166 | "Current learning rate 0.001 36388\n", 1167 | "\n", 1168 | "Epoch 00025: saving model to Imagenetvd.hdf5\n" 1169 | ] 1170 | }, 1171 | { 1172 | "data": { 1173 | "text/plain": [ 1174 | "" 1175 | ] 1176 | }, 1177 | "execution_count": 33, 1178 | "metadata": { 1179 | "tags": [] 1180 | }, 1181 | "output_type": "execute_result" 1182 | } 1183 | ], 1184 | "source": [ 1185 | "#Continue training on 32x32\n", 1186 | "model.fit_generator(initial_epoch=22,\n", 1187 | " generator=train_generator,\n", 1188 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 1189 | " validation_data=val_generator,\n", 1190 | " validation_steps=STEP_SIZE_VALID,\n", 1191 | " epochs=25, verbose=1,\n", 1192 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 1193 | ] 1194 | }, 1195 | { 1196 | "cell_type": "code", 1197 | "execution_count": 0, 1198 | "metadata": { 1199 | "colab": { 1200 | "base_uri": "https://localhost:8080/", 1201 | "height": 1425 1202 | }, 1203 | "colab_type": "code", 1204 | "id": "ZNoGbVk-ZZ3O", 1205 | "outputId": "27acc8f4-6256-43e5-9200-01e1f1a704ce" 1206 | }, 1207 | "outputs": [ 1208 | { 1209 | "name": "stdout", 1210 | "output_type": "stream", 1211 | "text": [ 1212 | "Epoch 26/30\n", 1213 | "1562/1562 [==============================] - 1562s 1000ms/step - loss: 2.4247 - acc: 0.4346 - val_loss: 2.2871 - val_acc: 0.4694\n", 1214 | "Current learning rate 0.001 37950\n", 1215 | "\n", 1216 | "Epoch 00026: saving model to Imagenetvd.hdf5\n", 1217 | "Epoch 27/30\n", 1218 | "1562/1562 [==============================] - 1561s 999ms/step - loss: 2.3984 - acc: 0.4414 - val_loss: 2.3543 - val_acc: 0.4573\n", 1219 | "Current learning rate 0.001 39512\n", 1220 | "Saving to id 1xef3xGHlB6ID8LH1qSfKE_8YHyJndsxa\n", 1221 | "\n", 1222 | "Epoch 00027: saving model to Imagenetvd.hdf5\n", 1223 | "Epoch 28/30\n", 1224 | "1562/1562 [==============================] - 1555s 996ms/step - loss: 2.3463 - acc: 0.4510 - val_loss: 2.3317 - val_acc: 0.4690\n", 1225 | "Current learning rate 0.001 41074\n", 1226 | "\n", 1227 | "Epoch 00028: saving model to Imagenetvd.hdf5\n", 1228 | "Epoch 29/30\n", 1229 | "1562/1562 [==============================] - 1541s 987ms/step - loss: 2.3159 - acc: 0.4582 - val_loss: 2.3071 - val_acc: 0.4666\n", 1230 | "Current learning rate 0.001 42636\n", 1231 | "\n", 1232 | "Epoch 00029: saving model to Imagenetvd.hdf5\n", 1233 | "Epoch 30/30\n", 1234 | " 13/1562 [..............................] - ETA: 24:39 - loss: 2.1212 - acc: 0.5048" 1235 | ] 1236 | }, 1237 | { 1238 | "ename": "KeyboardInterrupt", 1239 | "evalue": "ignored", 1240 | "output_type": "error", 1241 | "traceback": [ 1242 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 1243 | "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", 1244 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSTEP_SIZE_VALID\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])\n\u001b[0m", 1245 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 89\u001b[0m warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[1;32m 90\u001b[0m 'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 92\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1246 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 1416\u001b[0m \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1417\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1418\u001b[0;31m initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m 1419\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1420\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1247 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m 215\u001b[0m outs = model.train_on_batch(x, y,\n\u001b[1;32m 216\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m class_weight=class_weight)\n\u001b[0m\u001b[1;32m 218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1248 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m 1215\u001b[0m \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1216\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1217\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1218\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0munpack_singleton\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1219\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 1249 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2713\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2716\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2717\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1250 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 2634\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'`inputs` should be a list or tuple.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2635\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2636\u001b[0;31m \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2637\u001b[0m \u001b[0mfeed_arrays\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2638\u001b[0m \u001b[0marray_vals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1251 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mget_session\u001b[0;34m()\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0mcandidate_vars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mvariables\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_keras_initialized'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 194\u001b[0m \u001b[0mcandidate_vars\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcandidate_vars\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 1252 | "\u001b[0;31mKeyboardInterrupt\u001b[0m: " 1253 | ] 1254 | } 1255 | ], 1256 | "source": [ 1257 | "#Continue training on 32x32\n", 1258 | "model.fit_generator(initial_epoch=25,\n", 1259 | " generator=train_generator,\n", 1260 | " steps_per_epoch=STEP_SIZE_TRAIN,\n", 1261 | " validation_data=val_generator,\n", 1262 | " validation_steps=STEP_SIZE_VALID,\n", 1263 | " epochs=30, verbose=1,\n", 1264 | " callbacks=[lr_reducer, early_stopping, csv_logger, model_checkpoint])" 1265 | ] 1266 | }, 1267 | { 1268 | "cell_type": "code", 1269 | "execution_count": 0, 1270 | "metadata": { 1271 | "colab": { 1272 | "base_uri": "https://localhost:8080/", 1273 | "height": 34 1274 | }, 1275 | "colab_type": "code", 1276 | "id": "w1yvKUCH0P6d", 1277 | "outputId": "e13986d6-f19d-4244-d51f-11f519d32736" 1278 | }, 1279 | "outputs": [ 1280 | { 1281 | "name": "stdout", 1282 | "output_type": "stream", 1283 | "text": [ 1284 | "Saving to id 1zgPu_qtl4VjInr9EdegDO4VryNNVs31d\n" 1285 | ] 1286 | } 1287 | ], 1288 | "source": [ 1289 | "drive = drive_authenticate()\n", 1290 | "save_to_drive(drive, 29, model_file)" 1291 | ] 1292 | }, 1293 | { 1294 | "cell_type": "code", 1295 | "execution_count": 0, 1296 | "metadata": { 1297 | "colab": {}, 1298 | "colab_type": "code", 1299 | "id": "alXmwR-ae2aW" 1300 | }, 1301 | "outputs": [], 1302 | "source": [] 1303 | } 1304 | ], 1305 | "metadata": { 1306 | "accelerator": "GPU", 1307 | "colab": { 1308 | "name": "ImagenetDensenet.ipynb", 1309 | "provenance": [], 1310 | "version": "0.3.2" 1311 | }, 1312 | "kernelspec": { 1313 | "display_name": "Python 3", 1314 | "language": "python", 1315 | "name": "python3" 1316 | }, 1317 | "language_info": { 1318 | "codemirror_mode": { 1319 | "name": "ipython", 1320 | "version": 3 1321 | }, 1322 | "file_extension": ".py", 1323 | "mimetype": "text/x-python", 1324 | "name": "python", 1325 | "nbconvert_exporter": "python", 1326 | "pygments_lexer": "ipython3", 1327 | "version": "3.7.1" 1328 | } 1329 | }, 1330 | "nbformat": 4, 1331 | "nbformat_minor": 1 1332 | } 1333 | --------------------------------------------------------------------------------