├── .gitignore ├── Bangla_sentiment_using_socian.ipynb ├── CPU_GPU_tensorflow.ipynb ├── Classification_CNNs-v2.ipynb ├── Classification_CNNs.ipynb ├── GAP_cnn_tf2.ipynb ├── GAP_cnn_tf3.ipynb ├── LICENSE ├── NNLayers.ipynb ├── README.md ├── Tensorflow_Eager_Exec_1_x_2_0ipynb.ipynb ├── assets ├── curl.png ├── life.PNG ├── nn0.PNG ├── nn01.png ├── nn1.1.PNG ├── nn1.png ├── nn2.1.png ├── nn2.png ├── nn3.png ├── nn5.PNG ├── nn6.png ├── nn_d.jpg ├── readme.md ├── tensorboad1.PNG └── training-test2.jpg ├── classification_book.ipynb ├── cnn_tensorboard.ipynb ├── cricket-therm.ipynb ├── cricket-therm_need_to_work_on.ipynb ├── datasets ├── geoloc_elev.csv └── readme.md ├── deep_learning_decision_boundary.ipynb ├── deleted-final_sentiment_analysis_v2.ipynb ├── embedding_bangla_v1.ipynb ├── feature_crosses.ipynb ├── final_sentiment_analysis_v2.ipynb ├── first_steps_with_tensor_flow.ipynb ├── gradient_tape_linear_regression.ipynb ├── jax.ipynb ├── object.jpg ├── scikit_learn_tensorflow.ipynb ├── scratch_model_weight_changes_affect_accuracy.ipynb ├── sentiment_model_positive_negetive.ipynb ├── similarity.ipynb ├── small_sent_analysis.ipynb ├── small_sent_analysis_v1.ipynb ├── small_sent_analysis_v2.ipynb ├── temp └── word_embeddings.ipynb ├── tensor_numpy_shape_batch.ipynb ├── tensorflow_2_0.ipynb ├── tensorflow_serving ├── README.md ├── curl.png └── docker_curl.md ├── test_feature_crosses_nn.ipynb ├── text_classification.ipynb ├── tf2_transfer_learning.ipynb ├── tf2_transfer_learning_v1.ipynb ├── tf2_transfer_learning_v2.ipynb ├── tf2_transfer_learning_v3.ipynb ├── tf_keras_pretraind.ipynb └── wikipedia_embeddings.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /CPU_GPU_tensorflow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "CPU_GPU_tensorflow.ipynb", 7 | "provenance": [], 8 | "include_colab_link": true 9 | }, 10 | "kernelspec": { 11 | "name": "python3", 12 | "display_name": "Python 3" 13 | }, 14 | "accelerator": "GPU" 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "markdown", 19 | "metadata": { 20 | "id": "view-in-github", 21 | "colab_type": "text" 22 | }, 23 | "source": [ 24 | "\"Open" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "metadata": { 30 | "id": "0_vRX29fPmEM", 31 | "colab_type": "code", 32 | "colab": {} 33 | }, 34 | "source": [ 35 | "from __future__ import absolute_import, division, print_function, unicode_literals\n", 36 | "import tensorflow as tf" 37 | ], 38 | "execution_count": 0, 39 | "outputs": [] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "metadata": { 44 | "id": "d_j-EXTFQyqN", 45 | "colab_type": "code", 46 | "outputId": "33bc5997-c07f-44bc-8a21-4141add9cc75", 47 | "colab": { 48 | "base_uri": "https://localhost:8080/", 49 | "height": 87 50 | } 51 | }, 52 | "source": [ 53 | "tf.config.experimental.list_physical_devices()" 54 | ], 55 | "execution_count": 10, 56 | "outputs": [ 57 | { 58 | "output_type": "execute_result", 59 | "data": { 60 | "text/plain": [ 61 | "[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'),\n", 62 | " PhysicalDevice(name='/physical_device:XLA_CPU:0', device_type='XLA_CPU'),\n", 63 | " PhysicalDevice(name='/physical_device:XLA_GPU:0', device_type='XLA_GPU'),\n", 64 | " PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]" 65 | ] 66 | }, 67 | "metadata": { 68 | "tags": [] 69 | }, 70 | "execution_count": 10 71 | } 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "metadata": { 77 | "id": "uQ3DeAzfRZba", 78 | "colab_type": "code", 79 | "outputId": "277de6e5-e5dd-4642-8fe6-41f44cecc68a", 80 | "colab": { 81 | "base_uri": "https://localhost:8080/", 82 | "height": 35 83 | } 84 | }, 85 | "source": [ 86 | "len(tf.config.experimental.list_physical_devices('GPU'))" 87 | ], 88 | "execution_count": 11, 89 | "outputs": [ 90 | { 91 | "output_type": "execute_result", 92 | "data": { 93 | "text/plain": [ 94 | "1" 95 | ] 96 | }, 97 | "metadata": { 98 | "tags": [] 99 | }, 100 | "execution_count": 11 101 | } 102 | ] 103 | }, 104 | { 105 | "cell_type": "code", 106 | "metadata": { 107 | "id": "oTTpI27rSUPN", 108 | "colab_type": "code", 109 | "outputId": "02ece91e-897c-4e9a-ce3e-499ccd8701d0", 110 | "colab": { 111 | "base_uri": "https://localhost:8080/", 112 | "height": 35 113 | } 114 | }, 115 | "source": [ 116 | "tf.debugging.set_log_device_placement(True)\n", 117 | "\n", 118 | "# Place tensors on the CPU\n", 119 | "with tf.device('/CPU:0'):\n", 120 | " a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n", 121 | " b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n", 122 | "\n", 123 | "c = tf.matmul(a, b)\n", 124 | "print(c)" 125 | ], 126 | "execution_count": 12, 127 | "outputs": [ 128 | { 129 | "output_type": "stream", 130 | "text": [ 131 | "Tensor(\"MatMul_1:0\", shape=(2, 2), dtype=float32)\n" 132 | ], 133 | "name": "stdout" 134 | } 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "metadata": { 140 | "id": "qiCeKR4HR1OM", 141 | "colab_type": "code", 142 | "outputId": "47011f4d-6e59-4be4-9652-f42da1cc8c43", 143 | "colab": { 144 | "base_uri": "https://localhost:8080/", 145 | "height": 35 146 | } 147 | }, 148 | "source": [ 149 | "gpus = tf.config.experimental.list_physical_devices('GPU')\n", 150 | "if gpus:\n", 151 | " # Create 2 virtual GPUs with 512MB memory each\n", 152 | " try:\n", 153 | " tf.config.experimental.set_virtual_device_configuration(\n", 154 | " gpus[0],\n", 155 | " [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=512),\n", 156 | " tf.config.experimental.VirtualDeviceConfiguration(memory_limit=512)])\n", 157 | " logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n", 158 | " print(len(gpus), \"Physical GPU,\", len(logical_gpus), \"Logical GPUs\")\n", 159 | " except RuntimeError as e:\n", 160 | " # Virtual devices must be set before GPUs have been initialized\n", 161 | " print(e)" 162 | ], 163 | "execution_count": 13, 164 | "outputs": [ 165 | { 166 | "output_type": "stream", 167 | "text": [ 168 | "1 Physical GPU, 2 Logical GPUs\n" 169 | ], 170 | "name": "stdout" 171 | } 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "metadata": { 177 | "id": "JjNGiv2STVjO", 178 | "colab_type": "code", 179 | "outputId": "2858de66-d5b1-432b-fc27-41c4ce8e3de1", 180 | "colab": { 181 | "base_uri": "https://localhost:8080/", 182 | "height": 87 183 | } 184 | }, 185 | "source": [ 186 | "import time\n", 187 | "\n", 188 | "cpu_slot = 0\n", 189 | "gpu_slot = 0\n", 190 | "\n", 191 | "# Using CPU at slot 0\n", 192 | "with tf.device('/CPU:' + str(cpu_slot)):\n", 193 | " # Starting a timer\n", 194 | " start = time.time()\n", 195 | "\n", 196 | " # Doing operations on CPU\n", 197 | " A = tf.constant([[3, 2], [5, 2]])\n", 198 | " print(tf.eye(2,2))\n", 199 | "\n", 200 | " # Printing how long it took with CPU\n", 201 | " end = time.time() - start\n", 202 | " print(end)\n", 203 | "\n", 204 | "# Using the GPU at slot 0\n", 205 | "with tf.device('/GPU:' + str(gpu_slot)):\n", 206 | " # Starting a timer\n", 207 | " start = time.time()\n", 208 | "\n", 209 | " # Doing operations on GPU\n", 210 | " A = tf.constant([[3, 2], [5, 2]])\n", 211 | " print(tf.eye(2,2))\n", 212 | "\n", 213 | " # Printing how long it took with GPU\n", 214 | " end = time.time() - start\n", 215 | " print(end)" 216 | ], 217 | "execution_count": 14, 218 | "outputs": [ 219 | { 220 | "output_type": "stream", 221 | "text": [ 222 | "Tensor(\"eye_2/diag:0\", shape=(2, 2), dtype=float32, device=/device:CPU:0)\n", 223 | "0.002489328384399414\n", 224 | "Tensor(\"eye_3/diag:0\", shape=(2, 2), dtype=float32, device=/device:GPU:0)\n", 225 | "0.0018601417541503906\n" 226 | ], 227 | "name": "stdout" 228 | } 229 | ] 230 | } 231 | ] 232 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Rakibul Hassan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /NNLayers.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "NNLayers", 7 | "version": "0.3.2", 8 | "provenance": [], 9 | "include_colab_link": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "markdown", 19 | "metadata": { 20 | "id": "view-in-github", 21 | "colab_type": "text" 22 | }, 23 | "source": [ 24 | "\"Open" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": { 30 | "id": "IRCc-MaY-eZY", 31 | "colab_type": "text" 32 | }, 33 | "source": [ 34 | "# ডিপ লার্নিং দিয়ে প্রথম নিউরাল নেটওয়ার্ক \n", 35 | "\n", 36 | "প্রথম নিউরাল নেটওয়ার্ক, আমাদের মেশিন লার্নিং মডেলে। চেষ্টা করছি ব্যাপারটাকে খুবই সাধারণ রাখার জন্য, বিশেষ করে ''কনসেপ্ট হেভি এবং কোড লাইট'' সিস্টেমে। আমরা চেষ্টা করবো আপনার কনসেপ্ট যাতে ঠিকমত দাঁড়ায়, কারণ কোড এখন পুরো ইন্টারনেট জুড়ে। এমনকি গুগল কোলাবে হাজার হাজার কোড স্নিপেট দেয়া আছে পাশাপাশি। একদম ভয় পাবার কিছু নেই। \n" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": { 42 | "id": "ajZKrr3Ngp5N", 43 | "colab_type": "text" 44 | }, 45 | "source": [ 46 | "## নামপাই কম্প্যাটিবিলিটি\n", 47 | "\n", 48 | "আমাদের কাজের সুবিধার জন্য টেনসর ফ্লো’র টেনসর এবং নামপাই এন ডি অ্যারেগুলোর নিজেদের মধ্যে ‘কনভারশন’ অনেকটাই সোজা। টেনসর ফ্লো অপারেশনগুলো স্বয়ংক্রিয়ভাবে কনভার্ট করে নামপাই এন ডি অ্যারেকে টেনসরে। আবার নামপাই অপারেশনগুলোকে একইভাবে টেনসর থেকে কনভার্ট করে নামপাই এন ডি অ্যারেতে।\n" 49 | ] 50 | }, 51 | { 52 | "cell_type": "markdown", 53 | "metadata": { 54 | "id": "uFFpexkB3sKo", 55 | "colab_type": "text" 56 | }, 57 | "source": [ 58 | "শুরুতেই ইম্পোর্ট করছি বেশ কিছু লাইব্রেরি। টেনসর ফ্লো তো থাকছেই, সঙ্গে নামপাই। " 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "metadata": { 64 | "id": "szbFLdZwPRhK", 65 | "colab_type": "code", 66 | "outputId": "e613f7db-d412-488b-e61b-9ab52a5fa571", 67 | "colab": { 68 | "base_uri": "https://localhost:8080/", 69 | "height": 34 70 | } 71 | }, 72 | "source": [ 73 | "# টেনসর ফ্লো, কেরাস আর কিছু হেল্পার লাইব্রেরি \n", 74 | "# নামপাই তো থাকবেই \n", 75 | "\n", 76 | "import tensorflow as tf\n", 77 | "import numpy as np\n", 78 | "from tensorflow import keras\n", 79 | "\n", 80 | "print(tf.__version__)" 81 | ], 82 | "execution_count": 1, 83 | "outputs": [ 84 | { 85 | "output_type": "stream", 86 | "text": [ 87 | "1.13.1\n" 88 | ], 89 | "name": "stdout" 90 | } 91 | ] 92 | }, 93 | { 94 | "cell_type": "markdown", 95 | "metadata": { 96 | "id": "w2fyLX5R_4bs", 97 | "colab_type": "text" 98 | }, 99 | "source": [ 100 | "## একটা লিনিয়ার রিগ্রেশন সমস্যা \n", 101 | "\n", 102 | "দুটো নামপাই অ্যারে ''ইনিশিয়ালাইজ'' করছি নিচের একুয়েশনের ভিত্তিতে। \n", 103 | "\n", 104 | "y = 2x + 30\n", 105 | "\n", 106 | "## তৈরি করি ট্রেনিং ডেটা \n", 107 | "\n", 108 | "এখানে প্রথম অ্যারে 'xs' ইনপুট ভ্যালুতে এ আমরা -১০, ০, ২, ৬, ১২, ১৫ দেয়ার পর y = 2x + 30 তার কর্রেসপন্ডিং 'ys' পেয়ে গিয়েছি আউটপুট ভ্যালু হিসেবে। প্রতিটা 'xs' এর কর্রেসপন্ডিং ভ্যালু দিয়ে দিয়েছি আমাদের একুয়েশন থেকে। \n", 109 | "\n", 110 | "এরপর একটা ''ফর লুপ'' চালালাম দুটো ভ্যালুকে পাশাপাশি দেখাতে। " 111 | ] 112 | }, 113 | { 114 | "cell_type": "code", 115 | "metadata": { 116 | "id": "X1si3wsO29dB", 117 | "colab_type": "code", 118 | "colab": { 119 | "base_uri": "https://localhost:8080/", 120 | "height": 118 121 | }, 122 | "outputId": "c5624a4c-b4b9-4af2-be54-8a1b7be66662" 123 | }, 124 | "source": [ 125 | "xs = np.array([-10, 0, 2, 6, 12, 15], dtype=float)\n", 126 | "ys = np.array([10, 30, 34, 42, 54, 60], dtype=float)\n", 127 | "\n", 128 | "for i,x in enumerate(xs):\n", 129 | " print(\"X: {} Y: {}\".format(x, ys[i]))" 130 | ], 131 | "execution_count": 4, 132 | "outputs": [ 133 | { 134 | "output_type": "stream", 135 | "text": [ 136 | "X: -10.0 Y: 10.0\n", 137 | "X: 0.0 Y: 30.0\n", 138 | "X: 2.0 Y: 34.0\n", 139 | "X: 6.0 Y: 42.0\n", 140 | "X: 12.0 Y: 54.0\n", 141 | "X: 15.0 Y: 60.0\n" 142 | ], 143 | "name": "stdout" 144 | } 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": { 150 | "id": "VraeEQg4Ijfx", 151 | "colab_type": "text" 152 | }, 153 | "source": [ 154 | "## লিনিয়ার রিগ্রেশন একুয়েশনটা দেখি ভিজ্যুয়ালি \n", 155 | "\n", 156 | "আমাদের ম্যাটপ্লটলিব দিয়ে এই লিনিয়ার রিগ্রেশনটা 'y = 2x + 30' দেখি গ্রাফ আকারে " 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "metadata": { 162 | "id": "3gS4N7zUJC5y", 163 | "colab_type": "code", 164 | "colab": { 165 | "base_uri": "https://localhost:8080/", 166 | "height": 281 167 | }, 168 | "outputId": "6dd7fa5b-b599-4906-d563-6c90e0b78081" 169 | }, 170 | "source": [ 171 | "import matplotlib.pyplot as plt\n", 172 | "\n", 173 | "x = np.linspace(-10,10,100)\n", 174 | "plt.title('Graph of y=2x+30')\n", 175 | "plt.plot(x, x*2+30);" 176 | ], 177 | "execution_count": 5, 178 | "outputs": [ 179 | { 180 | "output_type": "display_data", 181 | "data": { 182 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd8VfX9x/HXh733CiNsZIsYwVVL\nQS1uwFGtA0dFa9tfhy3gxrrQOmpr1WLdWxmCiBNBtE5QSMIOe4QNAQKEjM/vj3voI00TuAn35ube\nvJ+PRx7ce8Y9n5xc3jn53nM+x9wdERGJf1ViXYCIiESGAl1EJEEo0EVEEoQCXUQkQSjQRUQShAJd\nRCRBKNClQjOzcWb2SoReq6WZzTGzPWb2SCReU6QiUaBLqZjZpWb2jZllm9mW4PFNZmaxri0Mo4Bt\nQAN3vzkaGzCzkWY2z8x2m9l6M3vIzKpFeBsPmdm6YBtrzOzWIvP7BTXsC/7tF8ntS8WlQJewmdnN\nwOPAX4BWQEvgRuAUoEYJ61QttwKPrD2wyKN7NV0d4HdAM2AgMAT4Y2lfxMyuNrMXSpj9LNDd3RsA\nJwOXm9mIYL0awFTgFaAx8CIwNZguCU6BLmExs4bAn4Gb3H2iu+/xkB/c/XJ3zwmWe8HMnjKzGWaW\nDfzEzM4xsx+CI8p1Zjau0Ot2MDM3s1FmttHMMs2saADWMLOXgqGShWaWcpg6Tzaz78wsK/j35EN1\nASOB0Wa218xOL7LeCWa2ufAvIDMbYWYLSrOf3P0pd//c3Q+6+wbgVUK/8DCzzma2w8z6B89bm9lW\nMxtUym0sdffsQpMKgC7B40FANeCv7p7j7n8DDBhcmm1IfFKgS7hOAmoSOvo7kp8D9wH1gS+AbOAq\noBFwDvBLMxtWZJ2fAF2BM4ExRQL3fOCNYP1pwBPFbdTMmgDvAX8DmgKPAu+ZWVN3v5pQuD7k7vXc\n/ZPC67r7d8D2YPuHXAm8FLz2z81s12G+kkvYF6cBC4NtrADGAK+YWR3geeBFd59dwrolMrOxZrYX\nWA/UBV4LZvUCUov8FZIaTJcEp0CXcDUDtrl73qEJZvZlEGb7zey0QstOdfd/u3uBux9w99nunhY8\nTwVeB35c5PXvdvdsd08jFHSXFZr3hbvPcPd84GXg2BJqPAdY7u4vu3ueu78OLAHOC/N7fBG4Ivje\nmgA/JQhKd3/N3Rsd5mtt0Rczs2uBFODhQ9Pc/RkgA/gGSAJuC7O2/+Lu4wn9wuxPaJ9kBbPqFXp8\nSFawrCQ4BbqEazvQrPAHfO5+srs3CuYVfi+tK7yimQ00s1nB8EIWoXH3ZkVev/A6a4DWhZ5vKvR4\nH1CrhA8aWwfrFrYGaFPyt/VfXgHOM7O6wCXA5+6eGea6/yX4C+QB4Cx331Zk9jNAb+Dvh4aqgnWe\nPHTEDzwJFP6rILXoNg4NeQH7gbuDyXuBBkUWbQDsKcv3IfFFgS7h+grIAS4IY9miHzq+RmiopJ27\nNwSeJjSuW1i7Qo+TgY1lqHEjoQ8+C0sGNoSzcjDm/RUwgtBwy8uH5pnZ5cHYe0lfyYWWHUootM8L\n/uKg0Lx6wF8JfbA5LvhL4ND2bzp0xA/cBBT+q6DvYUqvBnQOHi8E+hY566hvMF0SnAJdwuLuuwgd\nBT5pZheZWX0zqxKcElf3CKvXB3a4+wEzG0BojL2oO8ysjpn1Aq4B3ixDmTOAbsF4dzUz+xnQE5he\nitd4CRgN9AEmH5ro7q8GY+8lfa0FMLPBhMbqL3T3b4t5/ceBue7+C0Lj/U+X5hsM9vkNZtbYQgYA\nvwJmBovMBvKB/zOzmmb262D6p6XZjsQnBbqEzd0fAv5AKPA2B1//JPRB35eHWfUm4M9mtge4E3ir\nmGU+IzS2PBN42N0/KkN924FzgZsJDQONBs4tZsjjcKYQOsqf4u77SlsDcAfQEJhR6Oj9fQAzuwAY\nCvwyWPYPQH8zu7yU2xgOrCA0jPIK8PfgC3c/CAwj9CH0LuBaYFgwXRKc6QYXEktm1gFYBVQv/IFr\nLJnZCuCGomfCiFR0OkIXKcTMLiT0GYCGKCTuRPSSZJF4ZmazCY25X+nuBTEuR6TUNOQiIpIgNOQi\nIpIgynXIpVmzZt6hQ4fy3KSISNybN2/eNndvfqTlyjXQO3TowNy5c8tzkyIicc/Mil4BXSwNuYiI\nJAgFuohIglCgi4gkCAW6iEiCUKCLiCSIsM5yMbPVhBoB5QN57p4StP18E+gArAYucfed0SlTRESO\npDRH6D9x937ufuh+jmOBme7elVCHvLERr05ERMJ2NEMuFxC6ZRfBv0XvESkiUuntzD7I3e8uZPeB\n3KhvK9xAd+AjM5tnZqOCaS0L3Z5rE9CyuBWDu7nPNbO5W7duPcpyRUTig7vzXmomZzz2GS9/tYZv\nV+6I+jbDvVL0VHffYGYtgI/NbEnhme7uZlZsly93nwBMAEhJSVEnMBFJeFt2H+COqel8uHAzfdo0\n5OXrBtIjqeitXiMvrEAP7rWIu28xsynAAGCzmSW5e6aZJQFboliniEiF5+68PXc997y3iIN5Bdxy\nVneuO7Uj1aqWzwmFRwz04A7oVdx9T/D4TODPhG76OxIYH/w7NZqFiohUZGu37+PWKWl8kbGNAR2b\n8OCFfenY7Ei3242scI7QWwJTgpuIVyN0J/IPzOw74C0zuw5YA1wSvTJFRCqm/ALnhS9X8/CHS6la\nxbh3WG9+PiCZKlWs3Gs5YqC7+0rg2GKmbweGRKMoEZF4sHzzHkZPSuWHtbsY3L0F9w7rTetGtWNW\nj25BJyJSSgfzCnj6sxU88WkG9WpV4/FL+3H+sa0JRjJiRoEuIlIKqet3MXpiKks27eG8Y1sz7rye\nNK1XM9ZlAQp0EZGw7D+Yz2OfLONfn6+kef2aPHNVCmf0LPbym5hRoIuIHMFXK7Zzy+RUVm/fx2UD\n2jH2rB40rF091mX9DwW6iEgJ9hzI5YH3l/DaN2tJblKH134xkJO7NIt1WSVSoIuIFGPm4s3c/k46\nm3cf4PofdeQPZxxD7RpVY13WYSnQRUQK2b43h7vfXcS0BRs5pmV9nrriePq1axTrssKiQBcRIXTZ\n/rQFG7n73UXsOZDL70/vxi8HdaZGtfi5D5ACXUQqvcys/dw+JZ2ZS7ZwbLtGPHRhX45pVT/WZZWa\nAl1EKq2CAueN79bxwIzF5BYUcPs5PbjmlI5UjcFl+5GgQBeRSmnN9mzGTErl65U7OKlTU8Zf2If2\nTcu3mVakKdBFpFLJL3Ce+2IVj3y8lOpVqvDAiD5cekK7mF+2HwkKdBGpNJZuCjXTWrBuF6f3aMG9\nw/rQqmGtWJcVMQp0EUl4B/MK+MesDJ6cnUGDWtX5+2XHcW7fpIQ4Ki9MgS4iCW3+ul2MnriAZZv3\nckG/1tx1Xi+a1K0R67KiQoEuIglp38E8Hv1oGc/9exUt6tfi2ZEpDOlRsZppRVrYgW5mVYG5wAZ3\nP9fMXgB+DGQFi1zt7vMjX6KISOl8mbGNsZPTWLtjH5cPTGbsWd2pX6viNdOKtNIcof8WWAwUvnX1\nn9x9YmRLEhEpm6z9uYx/fzGvf7uODk3r8MaoEzmxU9NYl1Vuwgp0M2sLnAPcB/whqhWJiJTBx4s2\nc/s7aWzdk8MNp3Xi92d0o1b1it1MK9LCPUL/KzAaKHot7H1mdicwExjr7jlFVzSzUcAogOTk5KMo\nVUTkf23bm8O4aQuZnppJ91b1eeaqFPq2jY9mWpF2xK4zZnYusMXd5xWZdQvQHTgBaAKMKW59d5/g\n7inuntK8efOjrVdEBAg103rnhw2c8ehnfLRwMzef0Y13f3NqpQ1zCO8I/RTgfDM7G6gFNDCzV9z9\nimB+jpk9D/wxWkWKiBS2cdd+bpuSxqylWzkuOdRMq2vL+GumFWlHDHR3v4XQ0ThmNgj4o7tfYWZJ\n7p5poTPzhwHpUa1URCq9ggLntW/XMv79JeQXOHee25ORJ3eI22ZakXY056G/ambNAQPmAzdGpiQR\nkf+1als2Yyel8s2qHZzapRkPjOhDuyZ1Yl1WhVKqQHf32cDs4PHgKNQjIvJf8vILePaLVTz68TJq\nVKvCQxf25eKUtgl32X4k6EpREamwFmfuZvTEVNI2ZHFmz5bcM6w3LRskTjOtSFOgi0iFk5OXzxOf\nZvDU7BU0qlOdf/y8P2f3aaWj8iNQoItIhfL92p2MnphKxpa9jOjfhjvO6UnjBG2mFWkKdBGpELJz\n8nj4o6W88OVqkhrU4oVrTmDQMS1iXVZcUaCLSMx9vnwrt0xOY/3O/Vx1UntGD+1OvZqKp9LSHhOR\nmMnal8t9Mxbx1tz1dGpWl7duOIkBHZvEuqy4pUAXkZj4IH0Td0xNZ0f2QX45qDO/HdK10jXTijQF\nuoiUq617Qs203kvLpGdSA56/+gR6t2kY67ISggJdRMqFuzP5+w38efoi9ufm86efHsOo0zpRveoR\newRKmBToIhJ1G3bt59bJaXy2bCvHt2/Mgxf2pUuLerEuK+Eo0EUkagoKnFe/WcP495fgwN3n9+LK\nE9tTRc20okKBLiJRsWLrXsZOSuW71Tv5Uddm3D9czbSiTYEuIhGVm1/AM5+v5K+fLKd29ar85aK+\nXHS8mmmVBwW6iERM+oYsxkxKZeHG3ZzdpxXjzu9Fi/pqplVeFOgictQO5Obz90+X8/RnK2lcpwZP\nX9Gfob2TYl1WpaNAF5GjMnf1DkZPSmXl1mwuPr4tt5/Tk4Z1qse6rEop7EA3s6rAXGCDu59rZh2B\nN4CmwDzgSnc/GJ0yRaSiyc7J46EPlvDS12to3bA2L107gNO66UbwsVSaM/p/Cywu9PxB4DF37wLs\nBK6LZGEiUnF9tmwrZz42h5e+XsPIkzrw0e9PU5hXAGEFupm1Bc4B/hU8N2AwMDFY5EVCN4oWkQS2\na99Bbn5rASOf+5Za1avw9g0nMe78XtRVZ8QKIdyfwl+B0UD94HlTYJe75wXP1wNtilvRzEYBowCS\nk5PLXqmIxNSMtEzunJrOrn25/OonnfnNYDXTqmiOGOhmdi6wxd3nmdmg0m7A3ScAEwBSUlK81BWK\nSExt2X2AO6cu5IOFm+jdpgEvXjuAXq3VTKsiCucI/RTgfDM7G6gFNAAeBxqZWbXgKL0tsCF6ZYpI\neXN33p63nnunL+JAXgFjhnbn+h91pJqaaVVYRwx0d78FuAUgOEL/o7tfbmZvAxcROtNlJDA1inWK\nSDlat2Mft05J4/Pl2xjQoQnjL+xDp+ZqplXRHc0nGWOAN8zsXuAH4NnIlCQisZJf4Lz01Wr+8uFS\nDLjngl5cPlDNtOJFqQLd3WcDs4PHK4EBkS9JRGIhY8sexkxKY96anfy4W3PuG96bto3VTCue6Fwj\nkUouN7+Af362gr/NzKBOzao8esmxDD+ujZppxSEFukgllrY+iz9NXMCSTXs4p28S487rRfP6NWNd\nlpSRAl2kEjqQm89jnyzjmTkraVavJv+88nh+2qtVrMuSo6RAF6lkvl21gzGTUlm1LZtLT2jHLWf3\noGFtNdNKBAp0kUpiz4FcHvxgCa98vZZ2TWrz6i8GckqXZrEuSyJIgS5SCcxauoXbJqeRufsA153a\nkZvP7EadGvrvn2j0ExVJYDuyD3LP9EVM+WEDXVvUY9IvT6Z/cuNYlyVRokAXSUDuzvTUTMZNW0jW\n/lz+b0hXfvWTztSspmZaiUyBLpJgNu8+wG1T0vlk8Wb6tm3Iq9cPpHurBrEuS8qBAl0kQbg7b363\njvtmLOZgXgG3nd2Da07poGZalYgCXSQBrN2+j7GTU/lyxXYGdmzCgxf2pUOzurEuS8qZAl0kjuUX\nOC98uZqHP1xK1SrG/cP7cOkJ7dRMq5JSoIvEqWWb9zB6Yirz1+1icPcW3De8N0kNa8e6LIkhBbpI\nnDmYV8BTs1fwxKzl1KtZjccv7cf5x7ZWMy1RoIvEkwXrdjFmUipLNu3hvGNbM+68njStp2ZaEhLO\nPUVrAXOAmsHyE939LjN7AfgxkBUserW7z49WoSKV2f6DoWZa//p8JS3q1+JfV6Vwes+WsS5LKphw\njtBzgMHuvtfMqgNfmNn7wbw/ufvE6JUnIl+t2M4tk1NZvX0flw0INdNqUEvNtOR/hXNPUQf2Bk+r\nB18ezaJEBHYfyGX8+0t47Zu1tG9ah9euH8jJndVMS0oW1hUHZlbVzOYDW4CP3f2bYNZ9ZpZqZo+Z\nWbEDeWY2yszmmtncrVu3RqhskcQ2c/Fmznx0Dm98u5brf9SRD357msJcjshCB+BhLmzWCJgC/AbY\nDmwCagATgBXu/ufDrZ+SkuJz584te7UiCW773hzufncR0xZs5JiW9Xnwor70a9co1mVJjJnZPHdP\nOdJypb1J9C4zmwUMdfeHg8k5ZvY88Mcy1CkihC7bn7ZgI3e/u4g9B3L53elduWlQF2pU02X7Er5w\nznJpDuQGYV4bOAN40MyS3D3TQie/DgPSo1yrSELKzNrP7VPSmblkC/3aNeKhi/rSrWX9WJclcSic\nI/Qk4EUzq0pozP0td59uZp8GYW/AfODGKNYpknAKCpzXv1vLAzOWkFdQwO3n9OCaUzpSVZftSxmF\nc5ZLKnBcMdMHR6UikUpg9bZsxk5O5euVOzi5c1PGj+hLctM6sS5L4pyuFBUpR/kFzrNfrOSRj5ZR\no2oVHhgRaqaly/YlEhToIuVk6aY9jJ64gAXrszi9RwvuHdaHVg1rxbosSSAKdJEoy8nL58lZK3hy\ndgb1a1Xn75cdx7l9k3RULhGnQBeJoh/W7mTMpFSWbd7L8OPacMe5PWlSt0asy5IEpUAXiYJ9B/N4\n5KNlPPfvVbRqUIvnrk5hcHc105LoUqCLRNiXGdsYOzmNtTv2ccWJyYwZ2p36aqYl5UCBLhIhWftz\neWDGYt74bh0dmtbhjVEncmKnprEuSyoRBbpIBHy8aDO3v5PG1j053PDjTvz+9G7Uql411mVJJaNA\nFzkK2/bmMG7aQqanZtK9VX2euSqFvm3VTEtiQ4EuUgbuzjvzN3D3u4vYl5PPH8/sxg0/7kz1qmqm\nJbGjQBcppY279nPblDRmLd3KccmNeOjCvnRVMy2pABToImEqKHBe/XYt42cspsDhrvN6ctVJHdRM\nSyoMBbpIGFZty2bMpFS+XbWDU7s044ERfWjXRM20pGJRoIscRl5+Ac9+sYpHP15GzWpVeOiivlx8\nfFtdti8VkgJdpASLNu5mzKRU0jZkcWbPltw7rDctGqiZllRcCnSRInLy8nni0wyemr2CRnWq8+Tl\n/TmrdysdlUuFF84t6GoBc4CawfIT3f0uM+sIvAE0BeYBV7r7wWgWKxJt89aEmmllbNnLiKCZVmM1\n05I4Ec4Reg4w2N33mll14Aszex/4A/CYu79hZk8D1wFPRbFWkajJzsnj4Y+W8sKXq2ndsDYvXHMC\ng45pEeuyREolnFvQObA3eFo9+HJgMPDzYPqLwDgU6BKHPl++lVsmp7F+535GntSePw3tTr2aGo2U\n+BPWuza4QfQ8oAvwD2AFsMvd84JF1gNtSlh3FDAKIDk5+WjrFYmYrH253PveIt6et55Ozevy9o0n\ncUKHJrEuS6TMwgp0d88H+plZI2AK0D3cDbj7BGACQEpKipelSJFI+yA9kzumLmRH9kF+Oagzvx3S\nVc20JO6V6u9Kd99lZrOAk4BGZlYtOEpvC2yIRoEikbRlzwHumrqQ99M30TOpAc9ffQK92zSMdVki\nERHOWS7NgdwgzGsDZwAPArOAiwid6TISmBrNQkWOhrsz6fsN3DN9Eftz8/nTT49h1Gmd1ExLEko4\nR+hJwIvBOHoV4C13n25mi4A3zOxe4Afg2SjWKVJm63fu49Yp6cxZtpWU9o0Zf2FfurSoF+uyRCIu\nnLNcUoHjipm+EhgQjaJEIqGgwHn56zU8+MESAO4+vxdXntieKmqmJQlK52ZJQlqxdS9jJqYyd81O\nTuvWnPuH96ZtYzXTksSmQJeEkptfwIQ5K3l85nJqV6/Kwxcfy4X92+iyfakUFOiSMNI3ZDFmUioL\nN+7mrN6tuPuCXrSor2ZaUnko0CXuHcjN528zl/PPOStpUrcGT1/Rn6G9k2Jdlki5U6BLXPtu9Q7G\nTEpl5dZsLj6+Lbef05OGdarHuiyRmFCgS1zam5PHQx8s4aWv1tC2cW1euW4gp3ZtFuuyRGJKgS5x\n57NlW7l1chobs/ZzzSkd+OOZx1BXzbREFOgSP3ZmH+Se9xYx+fsNdG5el4k3nsTx7dVMS+QQBbpU\neO7O++mbuHNqOrv25fKbwV349eAu1KymZloihSnQpULbsvsAd0xN58OFm+nTpiEvXTuQnq0bxLos\nkQpJgS4Vkrvz9rz13Dt9ETl5BYw9qzu/OLUj1dRMS6RECnSpcNbt2Mctk9P4ImMbAzo2YfyIPnRq\nrmZaIkeiQJcKI7/AefHL1fzlw6VUrWLcO6w3Px+QrGZaImFSoEuFkLFlD6MnpvL92l0MOqY59w/v\nQ+tGtWNdlkhcUaBLTOXmF/DPz1bwt5kZ1K1Zlb/+rB8X9GutZloiZaBAl5hJW5/F6EmpLM7czbl9\nkxh3fi+a1asZ67JE4lY4t6BrB7wEtAQcmODuj5vZOOB6YGuw6K3uPiNahUriOJCbz2OfLOOZOStp\nVq8mE648njN7tYp1WSJxL5wj9DzgZnf/3szqA/PM7ONg3mPu/nD0ypNE8+2qHYydlMrKbdn8LKUd\nt57Tg4a11UxLJBLCuQVdJpAZPN5jZouBNtEuTBLLngO5PPjBEl75ei3tmtTm1V8M5JQuaqYlEkml\nGkM3sw6E7i/6DXAK8GszuwqYS+gofmcx64wCRgEkJycfZbkSj2Yt2cJtU9LI3H2A607tyM1ndqNO\nDX18IxJp5u7hLWhWD/gMuM/dJ5tZS2AboXH1e4Akd7/2cK+RkpLic+fOPcqSJV7syD7IPdMXMeWH\nDXRtUY8HL+pL/+TGsS5LJO6Y2Tx3TznScmEdJplZdWAS8Kq7TwZw982F5j8DTC9jrZJg3J3pqZmM\nm7aQrP25/N+QrvzqJ53VTEskysI5y8WAZ4HF7v5ooelJwfg6wHAgPTolSjzZvPsAt01J55PFm+nb\ntiGvXj+Q7q3UTEukPIRzhH4KcCWQZmbzg2m3ApeZWT9CQy6rgRuiUqHEBXfnze/Wcd+MxRzMK+DW\ns7tz7SlqpiVSnsI5y+ULoLjL9nTOuQCwZns2t0xO48sV2zmxUxPGj+hLh2Z1Y12WSKWjUw2kzPIL\nnOf/vYqHP1pK9SpVuH94Hy49oZ2aaYnEiAJdymTppj2MmZTK/HW7GNK9BfcO701SQzXTEoklBbqU\nysG8Ap6cncE/ZmVQv1Z1Hr+0H+cfq2ZaIhWBAl3CtmDdLsZMSmXJpj1c0K81d57bk6ZqpiVSYSjQ\n5Yj2H8zn0Y+X8uwXq2hRvxb/uiqF03u2jHVZIlKEAl0O68sV2xg7KY21O/bx84HJjD2rOw1qqZmW\nSEWkQJdi7T6QywMzlvD6t2tp37QOr19/Iid1bhrrskTkMBTo8j9mLt7MbVPS2bLnAKNO68TvT+9G\n7Rq6bF+kolOgy39s35vD3e8uYtqCjXRvVZ9/Xnk8x7ZrFOuyRCRMCnTB3Zm2YCPjpi1kb04evzu9\nKzcN6kKNarpsXySeKNArucys/dw+JZ2ZS7bQr10jHrqoL91a1o91WSJSBgr0SqqgwHn9u7U8MGMJ\neQUF3H5OD645pSNVddm+SNxSoFdCq7dlM3ZyKl+v3MHJnZvywIg+tG+qZloi8U6BXonk5Rfw3L9X\n8chHy6hRrQoPXtiHS1La6bJ9kQShQK8klmzazZiJqSxYn8UZPVty77DetGxQK9ZliUgEhXPHonbA\nS0BLQjezmODuj5tZE+BNoAOhG1xcUtxNoiW2cvLy+cesFTw5K4OGtavzxM+P45w+SToqF0lA4Ryh\n5wE3u/v3ZlYfmGdmHwNXAzPdfbyZjQXGAmOiV6qU1g9rdzJmUirLNu9lWL/W3HleL5rUrRHrskQk\nSsK5Y1EmkBk83mNmi4E2wAXAoGCxF4HZKNArhH0H83jko2U89+9VtGpQi+evPoGfdG8R67JEJMpK\nNYZuZh2A44BvgJaFbhK9idCQTHHrjAJGASQnJ5e1TgnTvzO2MXZyKut27OfyoJlWfTXTEqkUwg50\nM6sHTAJ+5+67C4/BurubmRe3nrtPACYApKSkFLuMHL2s/bnc/95i3py7jo7N6vLmqBMZ2EnNtEQq\nk7AC3cyqEwrzV919cjB5s5kluXummSUBW6JVpBzeRws3cfs76WzPPsiNP+7M707vSq3qaqYlUtmE\nc5aLAc8Ci9390UKzpgEjgfHBv1OjUqGUaOueHMa9u5D3UjPpkdSAZ0eeQJ+2DWNdlojESDhH6KcA\nVwJpZjY/mHYroSB/y8yuA9YAl0SnRCnK3Xln/gbufncR+3LyufmMbtw4qDPVq6qZlkhlFs5ZLl8A\nJZ20PCSy5ciRbNi1n9umpDF76Vb6J4eaaXVpoWZaIqIrReNGQYHz6rdrGT9jMQUOd53Xk6tO6qBm\nWiLyHwr0OLBy617GTkrj29U7OLVLMx4Y0Yd2TerEuiwRqWAU6BVYXn4Bz3y+isc+WUatalV46KK+\nXHx8W122LyLFUqBXUIs27mb0pAWkb9jNT3u15J4LetNCzbRE5DAU6BXMgdx8nvg0g6c/W0GjOtV5\n8vL+nN0nKdZliUgcUKBXIPPW7GD0xFRWbM3mwv5tuePcHjSqo2ZaIhIeBXoFkJ2Tx18+XMqLX62m\ndcPavHjtAH7crXmsyxKROKNAj7HPl29l7KQ0Nmbt56oT2/Onod2pV1M/FhEpPSVHjGTty+We9xYx\ncd56OjWvy1s3nMQJHZrEuiwRiWMK9Bj4ID2TO6YuZEf2QW4a1Jn/G6JmWiJy9BTo5WjLngPcNXUh\n76dvolfrBjx/9Qn0bqNmWiISGQr0cuDuTPp+A/dMX8T+3HxGDz2G63/USc20RCSiFOhRtn7nPm6d\nks6cZVtJad+Y8Rf2pUuLerEuS0QSkAI9SgoKnJe/XsODHyzBgD9f0IsrBranipppiUiUKNCjYMXW\nvYyZmMrcNTs5rVtz7h/em7ZPhgnHAAALCUlEQVSN1UxLRKJLgR5BufkFTJizksdnLqd29ao8cvGx\njOjfRs20RKRchHMLuueAc4Et7t47mDYOuB7YGix2q7vPiFaR8SB9QxajJ6ayKHM3Z/dpxd3n96Z5\n/ZqxLktEKpFwjtBfAJ4AXioy/TF3fzjiFcWZA7n5PD5zORPmrKRJ3Ro8fUV/hvZWMy0RKX/h3IJu\njpl1iH4p8ee71TsYMzGVlduyufj4ttx+Tk8a1qke67JEpJI6mjH0X5vZVcBc4GZ331ncQmY2ChgF\nkJycfBSbqzj25uTxlw+W8NLXa2jTqDYvXzeAH3VVMy0Ria2yXtnyFNAZ6AdkAo+UtKC7T3D3FHdP\nad48/kNv9tIt/PSxObz09RpGntSBD393msJcRCqEMh2hu/vmQ4/N7BlgesQqqqB2Zh/knvcWMfn7\nDXRpUY+JN57M8e0bx7osEZH/KFOgm1mSu2cGT4cD6ZErqWJxd95P38SdU9PZtS+X3wzuwq8Hd6Fm\nNTXTEpGKJZzTFl8HBgHNzGw9cBcwyMz6AQ6sBm6IYo0xs2X3Ae6Yms6HCzfTp01DXrp2ID1bN4h1\nWSIixQrnLJfLipn8bBRqqTDcnbfnrufe9xaRk1fAmKHduf5HHammZloiUoHpStEi1u3Yxy2T0/gi\nYxsDOjZh/Ig+dGquZloiUvEp0AP5Bc6LX67mLx8upYrBPcN6c/mAZDXTEpG4oUAHlm/ew+hJqfyw\ndheDjmnOfcP70KZR7ViXJSJSKpU60A/mFfD0Zyt44tMM6tasymM/O5Zh/dRMS0TiU6UN9LT1Wfxp\n4gKWbNrDOX2TuPv8XjSrp2ZaIhK/Kl2gH8jN57FPlvHMnJU0q1eTCVcez5m9WsW6LBGRo1apAv2b\nldsZOzmNVduyuWxAO8ae1YOGtdVMS0QSQ6UI9D0HcnnwgyW88vVakpvU4bVfDOTkLs1iXZaISEQl\nfKDPWrKF26aksWn3AX5xakf+cGY36tRI+G9bRCqhhE22HdkH+fO7C3ln/ka6tqjHpF+ezHHJaqYl\nIokr4QLd3Zmemsm4aQvJ2p/Lb4d05aafdFYzLRFJeAkV6JuyDnD7O+l8sngzx7ZtyKvXD6R7KzXT\nEpHKISEC3d1547t13P/eYnILCrjt7B5ce2pHquqyfRGpROI+0Ndsz2bspDS+WrmdEzs1YfyIvnRo\nVjfWZYmIlLu4DfT8Auf5f6/i4Y+WUr1KFe4f3odLT2inZloiUmnFZaAv3RRqprVg3S6GdG/BvcN7\nk9RQzbREpHIL545FzwHnAlvcvXcwrQnwJtCB0B2LLnH3ndErM+RgXgFPzs7gH7MyqF+rOn+77DjO\n65ukZloiIkA4t+B5ARhaZNpYYKa7dwVmBs+jav66XZz39y/46yfLObtPEh///jTOP7a1wlxEJBDO\nLejmmFmHIpMvIHSfUYAXgdnAmAjW9V/+PnM5j32yjBb1a/HsyBSG9GgZrU2JiMStso6ht3T3zODx\nJqDEhDWzUcAogOTk5DJtLLlpHS4dkMzYs7rToJaaaYmIFMfc/cgLhY7QpxcaQ9/l7o0Kzd/p7ke8\nrj4lJcXnzp1b9mpFRCohM5vn7ilHWq6st7HfbGZJwYaSgC1lfB0REYmQsgb6NGBk8HgkMDUy5YiI\nSFkdMdDN7HXgK+AYM1tvZtcB44EzzGw5cHrwXEREYiics1wuK2HWkAjXIiIiR6GsQy4iIlLBKNBF\nRBKEAl1EJEEo0EVEEkRYFxZFbGNmW4E1ZVy9GbAtguVEiuoqHdVVOqqrdCpqXXB0tbV39+ZHWqhc\nA/1omNnccK6UKm+qq3RUV+mortKpqHVB+dSmIRcRkQShQBcRSRDxFOgTYl1ACVRX6aiu0lFdpVNR\n64JyqC1uxtBFROTw4ukIXUREDkOBLiKSICpUoJvZxWa20MwKzCylyLxbzCzDzJaa2U9LWL+jmX0T\nLPemmdWIQo1vmtn84Gu1mc0vYbnVZpYWLBf1u3qY2Tgz21CotrNLWG5osA8zzCzq94I1s7+Y2RIz\nSzWzKWbWqITlymV/Hen7N7Oawc84I3gvdYhWLYW22c7MZpnZouD9/9tilhlkZlmFfr53RruuYLuH\n/blYyN+C/ZVqZv3LoaZjCu2H+Wa228x+V2SZcttfZvacmW0xs/RC05qY2cdmtjz4t9gbAJnZyGCZ\n5WY2srhlSsXdK8wX0AM4htA9SlMKTe8JLABqAh2BFUDVYtZ/C7g0ePw08Mso1/sIcGcJ81YDzcpx\n340D/niEZaoG+64TUCPYpz2jXNeZQLXg8YPAg7HaX+F8/8BNwNPB40uBN8vhZ5cE9A8e1weWFVPX\nIEJ3DSuX91O4PxfgbOB9wIATgW/Kub6qhG6D2T5W+ws4DegPpBea9hAwNng8trj3PdAEWBn82zh4\n3PhoaqlQR+juvtjdlxYz6wLgDXfPcfdVQAYwoPACZmbAYGBiMOlFYFi0ag22dwnwerS2EQUDgAx3\nX+nuB4E3CO3bqHH3j9w9L3j6NdA2mts7gnC+/wsIvXcg9F4aEvyso8bdM939++DxHmAx0Caa24yg\nC4CXPORroNGhu5mVkyHACncv6xXoR83d5wA7ikwu/D4qKYt+Cnzs7jvcfSfwMTD0aGqpUIF+GG2A\ndYWer+d/3/BNgV2FwqO4ZSLpR8Bmd19ewnwHPjKzecGNssvDr4M/e58r4U+8cPZjNF1L6GiuOOWx\nv8L5/v+zTPBeyiL03ioXwRDPccA3xcw+ycwWmNn7ZtarnEo60s8l1u+pSyn5oCoW++uQlu6eGTze\nBLQsZpmI77sj3uAi0szsE6BVMbNuc/cKcSu7MGu8jMMfnZ/q7hvMrAXwsZktCX6TR6Uu4CngHkL/\nAe8hNBx07dFsLxJ1HdpfZnYbkAe8WsLLRHx/xRszqwdMAn7n7ruLzP6e0LDC3uDzkXeAruVQVoX9\nuQSfkZ0P3FLM7Fjtr//h7m5m5XJ+eLkHurufXobVNgDtCj1vG0wrbDuhP/eqBUdWxS0TkRrNrBow\nAjj+MK+xIfh3i5lNIfTn/lH9Rwh335nZM8D0YmaFsx8jXpeZXQ2cCwzxYPCwmNeI+P4qRjjf/6Fl\n1gc/54aE3ltRZWbVCYX5q+4+uej8wgHv7jPM7Ekza+buUW1EFcbPJSrvqTCdBXzv7puLzojV/ipk\ns5kluXtmMAS1pZhlNhAa6z+kLaHPD8ssXoZcpgGXBmcgdCT0m/bbwgsEQTELuCiYFM2bV58OLHH3\n9cXNNLO6Zlb/0GNCHwymF7dspBQZtxxewva+A7pa6GygGoT+XJ0W5bqGAqOB8919XwnLlNf+Cuf7\nL3wD9IuAT0v6JRQpwRj9s8Bid3+0hGVaHRrLN7MBhP7vRvUXTZg/l2nAVcHZLicCWYWGGqKtxL+S\nY7G/iij8Piopiz4EzjSzxsEQ6ZnBtLIrj0+Bw/0iFETrgRxgM/BhoXm3ETpDYSlwVqHpM4DWweNO\nhII+A3gbqBmlOl8AbiwyrTUwo1AdC4KvhYSGHqK9714G0oDU4M2UVLSu4PnZhM6iWFFOdWUQGiec\nH3w9XbSu8txfxX3/wJ8J/cIBqBW8dzKC91KncthHpxIaKksttJ/OBm489D4Dfh3smwWEPlw+uRzq\nKvbnUqQuA/4R7M80Cp2dFuXa6hIK6IaFpsVkfxH6pZIJ5Ab5dR2hz11mAsuBT4AmwbIpwL8KrXtt\n8F7LAK452lp06b+ISIKIlyEXERE5AgW6iEiCUKCLiCQIBbqISIJQoIuIJAgFuohIglCgi4gkiP8H\n2rJABMFcXO8AAAAASUVORK5CYII=\n", 183 | "text/plain": [ 184 | "
" 185 | ] 186 | }, 187 | "metadata": { 188 | "tags": [] 189 | } 190 | } 191 | ] 192 | }, 193 | { 194 | "cell_type": "markdown", 195 | "metadata": { 196 | "id": "8Phf0lcQ35VE", 197 | "colab_type": "text" 198 | }, 199 | "source": [ 200 | "## প্রথম নিউরাল নেটওয়ার্ক তৈরি \n", 201 | "\n", 202 | "চলুন তৈরি করে ফেলি আমাদের প্রথম নিউরাল নেটওয়ার্ক মডেল। পৃথিবীর সবচেয়ে সোজা মডেল হবে এটা, এটাকে আমরা বলছি ‘ডেন্স’ নেটওয়ার্ক মডেল। ‘ডেন্স’ মানে হচ্ছে একেকটার সাথে স্ট্যাক করে কানেক্টেড মডেল। নিউরাল নেটওয়ার্ক তৈরি করতে আমাদের এটার লেয়ারগুলোর কনফিগারেশন বলতে হবে। লেয়ার ঠিকমতো কনফিগার করা হলে আমরা মডেলকে কম্পাইল করব। \n", 203 | "\n", 204 | "## লেয়ারের কনফিগারেশন \n", 205 | "\n", 206 | "একটা নিউরাল নেটওয়ার্কের বেসিক বিল্ডিং ব্লক হচ্ছে লেয়ার। লেয়ারগুলোতে যে ডেটা ফিড করানো হয় সেখান থেকে সে ডেটার রিপ্রেজেন্টেশনগুলোকে ঠিকমতো এক্সট্রাক্ট করে নিয়ে নেয় একেকটা লেয়ারে। বেশিরভাগ ডিপ লার্নিং মডেলগুলোর লেয়ার একটার সাথে আরেকটার কানেকশন ডেটাগুলোকে ঠিকমতো বুঝতে সাহায্য করে। যেহেতু আমাদের সমস্যাটা খুবই সহজ সে কারণে কিন্তু আমাদের এই নেটওয়ার্কে একটা লেয়ার হলেই চলবে। এই একটা লেয়ারে আমরা একটা নিউরন চালাবো।\n", 207 | "\n", 208 | "এখানে একটা ইনপুট, একটা লেয়ার, একটা নিউরন, এবং একটা আউটপুট। ছবি দেখুন। ইনপুটে একটা ভ্যালু, input_shape=[1], এটা একটা ১ ডাইমেনশনের একটা সংখ্যা। \n", 209 | "\n", 210 | "\n", 211 | " \n", 215 | " \n", 218 | "
\n", 212 | " \"Fashion\n", 214 | "
\n", 216 | " চিত্র ১.০ সবচেয়ে সোজা নিউরাল নেটওয়ার্ক একটা নিউরন
 \n", 217 | "
\n", 219 | "\n", 220 | "\n", 221 | "আমাদের নিউরাল নেটওয়ার্ক ডিফাইন করে নেই। যেহেতু আমরা কেরাস ব্যবহার করছি, সেহেতু সিকুয়েন্সিয়াল লেয়ার ব্যবহার করবো - যা লিনিয়ার লেয়ারগুলোর স্ট্যাক। " 222 | ] 223 | }, 224 | { 225 | "cell_type": "markdown", 226 | "metadata": { 227 | "id": "9U5dxy_6-BTy", 228 | "colab_type": "text" 229 | }, 230 | "source": [ 231 | "\n", 232 | " \n", 236 | " \n", 239 | "
\n", 233 | " \"Fashion\n", 235 | "
\n", 237 | " চিত্র ২.০ সবচেয়ে সোজা নিউরাল নেটওয়ার্ক একটা নিউরন
 \n", 238 | "
\n" 240 | ] 241 | }, 242 | { 243 | "cell_type": "code", 244 | "metadata": { 245 | "id": "ktSD38YW224P", 246 | "colab_type": "code", 247 | "colab": {} 248 | }, 249 | "source": [ 250 | "layer_0 = keras.layers.Dense(units=1, input_shape=[1])\n", 251 | "model = tf.keras.Sequential([layer_0])\n", 252 | "\n", 253 | "# এভাবেও লেখা যায় \n", 254 | "# model = tf.keras.Sequential([keras.layers.Dense(units=1, input_shape=[1])])\n" 255 | ], 256 | "execution_count": 0, 257 | "outputs": [] 258 | }, 259 | { 260 | "cell_type": "markdown", 261 | "metadata": { 262 | "id": "eKxumjtD4ZMt", 263 | "colab_type": "text" 264 | }, 265 | "source": [ 266 | "নিউরাল নেটওয়ার্ককে কম্পাইল করছি " 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "metadata": { 272 | "id": "dqR9BnAS259L", 273 | "colab_type": "code", 274 | "colab": {} 275 | }, 276 | "source": [ 277 | "\n", 278 | "model.compile(optimizer='sgd', loss='mean_squared_error')\n", 279 | "\n" 280 | ], 281 | "execution_count": 0, 282 | "outputs": [] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "metadata": { 287 | "id": "g1I9I8Jx3EnV", 288 | "colab_type": "code", 289 | "colab": {} 290 | }, 291 | "source": [ 292 | "\n", 293 | "model.fit(xs, ys, epochs=100)\n", 294 | "\n" 295 | ], 296 | "execution_count": 0, 297 | "outputs": [] 298 | }, 299 | { 300 | "cell_type": "markdown", 301 | "metadata": { 302 | "id": "8SV1Z68B9n_C", 303 | "colab_type": "text" 304 | }, 305 | "source": [ 306 | "একটা অজানা ভ্যালু প্রেডিক্ট করি " 307 | ] 308 | }, 309 | { 310 | "cell_type": "code", 311 | "metadata": { 312 | "id": "5Ep554if3IlP", 313 | "colab_type": "code", 314 | "colab": {} 315 | }, 316 | "source": [ 317 | "print(model.predict([10.0]))\n", 318 | "\n", 319 | "print(\"We can view the layer variables: {}\".format(layer_0.get_weights()))\n" 320 | ], 321 | "execution_count": 0, 322 | "outputs": [] 323 | } 324 | ] 325 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow2 2 | TensorFlow 2.0 Google Colab Notebook 3 | -------------------------------------------------------------------------------- /Tensorflow_Eager_Exec_1_x_2_0ipynb.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "Tensorflow_Eager_Exec-1.x.-2.0ipynb", 7 | "version": "0.3.2", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "include_colab_link": true 11 | }, 12 | "language_info": { 13 | "codemirror_mode": { 14 | "name": "ipython", 15 | "version": 3 16 | }, 17 | "file_extension": ".py", 18 | "mimetype": "text/x-python", 19 | "name": "python", 20 | "nbconvert_exporter": "python", 21 | "pygments_lexer": "ipython3", 22 | "version": "3.7.3" 23 | }, 24 | "kernelspec": { 25 | "display_name": "Python 3", 26 | "language": "python", 27 | "name": "python3" 28 | } 29 | }, 30 | "cells": [ 31 | { 32 | "cell_type": "markdown", 33 | "metadata": { 34 | "id": "view-in-github", 35 | "colab_type": "text" 36 | }, 37 | "source": [ 38 | "\"Open" 39 | ] 40 | }, 41 | { 42 | "metadata": { 43 | "id": "EM7xU9c7Qo8Z", 44 | "colab_type": "text" 45 | }, 46 | "cell_type": "markdown", 47 | "source": [ 48 | "# টেন্সর-ফ্লো ১.x ভার্সন \n", 49 | "# Tensorflow Eager mode নেই " 50 | ] 51 | }, 52 | { 53 | "metadata": { 54 | "id": "917g8FH5Qo8a", 55 | "colab_type": "code", 56 | "colab": {} 57 | }, 58 | "cell_type": "code", 59 | "source": [ 60 | "import numpy as np\n", 61 | "import tensorflow as tf" 62 | ], 63 | "execution_count": 0, 64 | "outputs": [] 65 | }, 66 | { 67 | "metadata": { 68 | "id": "F_KhZi5BTZpY", 69 | "colab_type": "text" 70 | }, 71 | "cell_type": "markdown", 72 | "source": [ 73 | "# এখন কি আছে?" 74 | ] 75 | }, 76 | { 77 | "metadata": { 78 | "id": "Jg2No-evRzKe", 79 | "colab_type": "code", 80 | "colab": { 81 | "base_uri": "https://localhost:8080/", 82 | "height": 35 83 | }, 84 | "outputId": "5ffe72d3-cc60-4a32-d03a-3f281744e92a" 85 | }, 86 | "cell_type": "code", 87 | "source": [ 88 | "print(tf.__version__)" 89 | ], 90 | "execution_count": 7, 91 | "outputs": [ 92 | { 93 | "output_type": "stream", 94 | "text": [ 95 | "1.13.1\n" 96 | ], 97 | "name": "stdout" 98 | } 99 | ] 100 | }, 101 | { 102 | "metadata": { 103 | "id": "DbKe0HiHQo8e", 104 | "colab_type": "text" 105 | }, 106 | "cell_type": "markdown", 107 | "source": [ 108 | "## Eager Enabled নেই (Default হিসেবে)" 109 | ] 110 | }, 111 | { 112 | "metadata": { 113 | "id": "LaUN8hSzQo8f", 114 | "colab_type": "code", 115 | "outputId": "575c3274-b98a-494f-a726-fe2a79937033", 116 | "colab": { 117 | "base_uri": "https://localhost:8080/", 118 | "height": 34 119 | } 120 | }, 121 | "cell_type": "code", 122 | "source": [ 123 | "tf.executing_eagerly()" 124 | ], 125 | "execution_count": 8, 126 | "outputs": [ 127 | { 128 | "output_type": "execute_result", 129 | "data": { 130 | "text/plain": [ 131 | "False" 132 | ] 133 | }, 134 | "metadata": { 135 | "tags": [] 136 | }, 137 | "execution_count": 8 138 | } 139 | ] 140 | }, 141 | { 142 | "metadata": { 143 | "id": "umwa9KWjSjwi", 144 | "colab_type": "text" 145 | }, 146 | "cell_type": "markdown", 147 | "source": [ 148 | "# tf.matmul: Multiply Two Matricies Using TensorFlow MatMul" 149 | ] 150 | }, 151 | { 152 | "metadata": { 153 | "scrolled": true, 154 | "id": "ZdkL5GI6Qo8i", 155 | "colab_type": "code", 156 | "outputId": "98abf679-cd55-4854-d746-e6c2d57fe2ce", 157 | "colab": { 158 | "base_uri": "https://localhost:8080/", 159 | "height": 34 160 | } 161 | }, 162 | "cell_type": "code", 163 | "source": [ 164 | "x = [[2., 2.],\n", 165 | " [1., 0.]]\n", 166 | "\n", 167 | "m = tf.matmul(x, x)\n", 168 | "\n", 169 | "m" 170 | ], 171 | "execution_count": 9, 172 | "outputs": [ 173 | { 174 | "output_type": "execute_result", 175 | "data": { 176 | "text/plain": [ 177 | "" 178 | ] 179 | }, 180 | "metadata": { 181 | "tags": [] 182 | }, 183 | "execution_count": 9 184 | } 185 | ] 186 | }, 187 | { 188 | "metadata": { 189 | "id": "PpdhriR2T1bY", 190 | "colab_type": "text" 191 | }, 192 | "cell_type": "markdown", 193 | "source": [ 194 | "# টেন্সর-ফ্লো ২.০ ইনস্টলেশন \n", 195 | "\n", 196 | "কিছু জিনিস ঠিকমতো কাজ না করতে পারে, টেস্ট করতে থাকুন " 197 | ] 198 | }, 199 | { 200 | "metadata": { 201 | "id": "1H1iDGiDTpdr", 202 | "colab_type": "code", 203 | "colab": { 204 | "base_uri": "https://localhost:8080/", 205 | "height": 616 206 | }, 207 | "outputId": "1de8cd03-d374-4dba-9147-a6a672f65013" 208 | }, 209 | "cell_type": "code", 210 | "source": [ 211 | "!pip install --upgrade tensorflow-gpu==2.0.0-alpha0" 212 | ], 213 | "execution_count": 10, 214 | "outputs": [ 215 | { 216 | "output_type": "stream", 217 | "text": [ 218 | "Collecting tensorflow-gpu==2.0.0-alpha0\n", 219 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/1a/66/32cffad095253219d53f6b6c2a436637bbe45ac4e7be0244557210dc3918/tensorflow_gpu-2.0.0a0-cp36-cp36m-manylinux1_x86_64.whl (332.1MB)\n", 220 | "\u001b[K |████████████████████████████████| 332.1MB 62kB/s \n", 221 | "\u001b[?25hRequirement already satisfied, skipping upgrade: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.2.2)\n", 222 | "Requirement already satisfied, skipping upgrade: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.12.0)\n", 223 | "Requirement already satisfied, skipping upgrade: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.7)\n", 224 | "Collecting tf-estimator-nightly<1.14.0.dev2019030116,>=1.14.0.dev2019030115 (from tensorflow-gpu==2.0.0-alpha0)\n", 225 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/13/82/f16063b4eed210dc2ab057930ac1da4fbe1e91b7b051a6c8370b401e6ae7/tf_estimator_nightly-1.14.0.dev2019030115-py2.py3-none-any.whl (411kB)\n", 226 | "\u001b[K |████████████████████████████████| 419kB 40.7MB/s \n", 227 | "\u001b[?25hRequirement already satisfied, skipping upgrade: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (3.7.1)\n", 228 | "Requirement already satisfied, skipping upgrade: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)\n", 229 | "Requirement already satisfied, skipping upgrade: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.1.0)\n", 230 | "Requirement already satisfied, skipping upgrade: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)\n", 231 | "Requirement already satisfied, skipping upgrade: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.33.1)\n", 232 | "Collecting tb-nightly<1.14.0a20190302,>=1.14.0a20190301 (from tensorflow-gpu==2.0.0-alpha0)\n", 233 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/a9/51/aa1d756644bf4624c03844115e4ac4058eff77acd786b26315f051a4b195/tb_nightly-1.14.0a20190301-py3-none-any.whl (3.0MB)\n", 234 | "\u001b[K |████████████████████████████████| 3.0MB 31.4MB/s \n", 235 | "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.16.3)\n", 236 | "Requirement already satisfied, skipping upgrade: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.15.0)\n", 237 | "Requirement already satisfied, skipping upgrade: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.9)\n", 238 | "Collecting google-pasta>=0.1.2 (from tensorflow-gpu==2.0.0-alpha0)\n", 239 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/64/bb/f1bbc131d6294baa6085a222d29abadd012696b73dcbf8cf1bf56b9f082a/google_pasta-0.1.5-py3-none-any.whl (51kB)\n", 240 | "\u001b[K |████████████████████████████████| 61kB 21.6MB/s \n", 241 | "\u001b[?25hRequirement already satisfied, skipping upgrade: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-alpha0) (2.8.0)\n", 242 | "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0.0-alpha0) (41.0.1)\n", 243 | "Requirement already satisfied, skipping upgrade: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (0.15.2)\n", 244 | "Requirement already satisfied, skipping upgrade: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (3.1)\n", 245 | "Installing collected packages: tf-estimator-nightly, tb-nightly, google-pasta, tensorflow-gpu\n", 246 | "Successfully installed google-pasta-0.1.5 tb-nightly-1.14.0a20190301 tensorflow-gpu-2.0.0a0 tf-estimator-nightly-1.14.0.dev2019030115\n" 247 | ], 248 | "name": "stdout" 249 | }, 250 | { 251 | "output_type": "display_data", 252 | "data": { 253 | "application/vnd.colab-display-data+json": { 254 | "pip_warning": { 255 | "packages": [ 256 | "tensorflow", 257 | "tensorflow_estimator" 258 | ] 259 | } 260 | } 261 | }, 262 | "metadata": { 263 | "tags": [] 264 | } 265 | } 266 | ] 267 | }, 268 | { 269 | "metadata": { 270 | "id": "sqqaXABD53ND", 271 | "colab_type": "code", 272 | "colab": {} 273 | }, 274 | "cell_type": "code", 275 | "source": [ 276 | "import numpy as np\n", 277 | "import tensorflow as tf" 278 | ], 279 | "execution_count": 0, 280 | "outputs": [] 281 | }, 282 | { 283 | "metadata": { 284 | "id": "m1V3nWne55Aa", 285 | "colab_type": "code", 286 | "colab": { 287 | "base_uri": "https://localhost:8080/", 288 | "height": 34 289 | }, 290 | "outputId": "42af5bc0-7f5d-47e6-bbc2-ecdd98c6aee2" 291 | }, 292 | "cell_type": "code", 293 | "source": [ 294 | "tf.executing_eagerly()" 295 | ], 296 | "execution_count": 3, 297 | "outputs": [ 298 | { 299 | "output_type": "execute_result", 300 | "data": { 301 | "text/plain": [ 302 | "True" 303 | ] 304 | }, 305 | "metadata": { 306 | "tags": [] 307 | }, 308 | "execution_count": 3 309 | } 310 | ] 311 | }, 312 | { 313 | "metadata": { 314 | "id": "nelIcfzlT9Hj", 315 | "colab_type": "code", 316 | "colab": { 317 | "base_uri": "https://localhost:8080/", 318 | "height": 34 319 | }, 320 | "outputId": "ace48694-d669-4f6a-d883-edd2741183bc" 321 | }, 322 | "cell_type": "code", 323 | "source": [ 324 | "tf.__version__" 325 | ], 326 | "execution_count": 4, 327 | "outputs": [ 328 | { 329 | "output_type": "execute_result", 330 | "data": { 331 | "text/plain": [ 332 | "'2.0.0-alpha0'" 333 | ] 334 | }, 335 | "metadata": { 336 | "tags": [] 337 | }, 338 | "execution_count": 4 339 | } 340 | ] 341 | }, 342 | { 343 | "metadata": { 344 | "id": "pmGNj0YNUGVz", 345 | "colab_type": "code", 346 | "colab": { 347 | "base_uri": "https://localhost:8080/", 348 | "height": 68 349 | }, 350 | "outputId": "00ae5cc7-83bd-4e03-f527-3188fc33fad8" 351 | }, 352 | "cell_type": "code", 353 | "source": [ 354 | "x = [[2., 2.],\n", 355 | " [1., 0.]]\n", 356 | "\n", 357 | "m = tf.matmul(x, x)\n", 358 | "\n", 359 | "m" 360 | ], 361 | "execution_count": 5, 362 | "outputs": [ 363 | { 364 | "output_type": "execute_result", 365 | "data": { 366 | "text/plain": [ 367 | "" 370 | ] 371 | }, 372 | "metadata": { 373 | "tags": [] 374 | }, 375 | "execution_count": 5 376 | } 377 | ] 378 | }, 379 | { 380 | "metadata": { 381 | "id": "XNWuHyXwQo8l", 382 | "colab_type": "code", 383 | "colab": { 384 | "base_uri": "https://localhost:8080/", 385 | "height": 51 386 | }, 387 | "outputId": "9ee0696a-cfc2-4a21-f879-90ca04f2a78c" 388 | }, 389 | "cell_type": "code", 390 | "source": [ 391 | "m.numpy()" 392 | ], 393 | "execution_count": 6, 394 | "outputs": [ 395 | { 396 | "output_type": "execute_result", 397 | "data": { 398 | "text/plain": [ 399 | "array([[6., 4.],\n", 400 | " [2., 2.]], dtype=float32)" 401 | ] 402 | }, 403 | "metadata": { 404 | "tags": [] 405 | }, 406 | "execution_count": 6 407 | } 408 | ] 409 | } 410 | ] 411 | } -------------------------------------------------------------------------------- /assets/curl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/curl.png -------------------------------------------------------------------------------- /assets/life.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/life.PNG -------------------------------------------------------------------------------- /assets/nn0.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/nn0.PNG -------------------------------------------------------------------------------- /assets/nn01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/nn01.png -------------------------------------------------------------------------------- /assets/nn1.1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/nn1.1.PNG -------------------------------------------------------------------------------- /assets/nn1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/nn1.png -------------------------------------------------------------------------------- /assets/nn2.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/nn2.1.png -------------------------------------------------------------------------------- /assets/nn2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/nn2.png -------------------------------------------------------------------------------- /assets/nn3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/nn3.png -------------------------------------------------------------------------------- /assets/nn5.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/nn5.PNG -------------------------------------------------------------------------------- /assets/nn6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/nn6.png -------------------------------------------------------------------------------- /assets/nn_d.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/nn_d.jpg -------------------------------------------------------------------------------- /assets/readme.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /assets/tensorboad1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/tensorboad1.PNG -------------------------------------------------------------------------------- /assets/training-test2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/assets/training-test2.jpg -------------------------------------------------------------------------------- /datasets/readme.md: -------------------------------------------------------------------------------- 1 | Keeping all the datasets 2 | -------------------------------------------------------------------------------- /jax.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "jax.ipynb", 7 | "provenance": [], 8 | "authorship_tag": "ABX9TyOYdS4nexFssHMoS1G29yM3", 9 | "include_colab_link": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "accelerator": "GPU" 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "view-in-github", 22 | "colab_type": "text" 23 | }, 24 | "source": [ 25 | "\"Open" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "metadata": { 31 | "id": "Ea4uD9__QFkn" 32 | }, 33 | "source": [ 34 | "import random\r\n", 35 | "import itertools\r\n", 36 | "\r\n", 37 | "import jax\r\n", 38 | "import jax.numpy as np\r\n", 39 | "# Current convention is to import original numpy as \"onp\"\r\n", 40 | "import numpy as onp" 41 | ], 42 | "execution_count": null, 43 | "outputs": [] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "metadata": { 48 | "id": "plzFK-tzQQlr" 49 | }, 50 | "source": [ 51 | "# Sigmoid nonlinearity\r\n", 52 | "def sigmoid(x):\r\n", 53 | " return 1 / (1 + np.exp(-x))\r\n", 54 | "\r\n", 55 | "# Computes our network's output\r\n", 56 | "def net(params, x):\r\n", 57 | " w1, b1, w2, b2 = params\r\n", 58 | " hidden = np.tanh(np.dot(w1, x) + b1)\r\n", 59 | " return sigmoid(np.dot(w2, hidden) + b2)\r\n", 60 | "\r\n", 61 | "# Cross-entropy loss\r\n", 62 | "def loss(params, x, y):\r\n", 63 | " out = net(params, x)\r\n", 64 | " cross_entropy = -y * np.log(out) - (1 - y)*np.log(1 - out)\r\n", 65 | " return cross_entropy\r\n", 66 | "\r\n", 67 | "# Utility function for testing whether the net produces the correct\r\n", 68 | "# output for all possible inputs\r\n", 69 | "def test_all_inputs(inputs, params):\r\n", 70 | " predictions = [int(net(params, inp) > 0.5) for inp in inputs]\r\n", 71 | " for inp, out in zip(inputs, predictions):\r\n", 72 | " print(inp, '->', out)\r\n", 73 | " return (predictions == [onp.bitwise_xor(*inp) for inp in inputs])" 74 | ], 75 | "execution_count": null, 76 | "outputs": [] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "metadata": { 81 | "id": "bQwLurZiQXL0" 82 | }, 83 | "source": [ 84 | "def initial_params():\r\n", 85 | " return [\r\n", 86 | " onp.random.randn(3, 2), # w1\r\n", 87 | " onp.random.randn(3), # b1\r\n", 88 | " onp.random.randn(3), # w2\r\n", 89 | " onp.random.randn(), #b2\r\n", 90 | " ]" 91 | ], 92 | "execution_count": null, 93 | "outputs": [] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "metadata": { 98 | "colab": { 99 | "base_uri": "https://localhost:8080/" 100 | }, 101 | "id": "74AYe7w2QbA9", 102 | "outputId": "6cc776cb-cff5-4aac-ce61-8daa2d1224f9" 103 | }, 104 | "source": [ 105 | "loss_grad = jax.grad(loss)\r\n", 106 | "\r\n", 107 | "# Stochastic gradient descent learning rate\r\n", 108 | "learning_rate = 1.\r\n", 109 | "# All possible inputs\r\n", 110 | "inputs = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])\r\n", 111 | "\r\n", 112 | "# Initialize parameters randomly\r\n", 113 | "params = initial_params()\r\n", 114 | "\r\n", 115 | "for n in itertools.count():\r\n", 116 | " # Grab a single random input\r\n", 117 | " x = inputs[onp.random.choice(inputs.shape[0])]\r\n", 118 | " # Compute the target output\r\n", 119 | " y = onp.bitwise_xor(*x)\r\n", 120 | " # Get the gradient of the loss for this input/output pair\r\n", 121 | " grads = loss_grad(params, x, y)\r\n", 122 | " # Update parameters via gradient descent\r\n", 123 | " params = [param - learning_rate * grad\r\n", 124 | " for param, grad in zip(params, grads)]\r\n", 125 | " # Every 100 iterations, check whether we've solved XOR\r\n", 126 | " if not n % 100:\r\n", 127 | " print('Iteration {}'.format(n))\r\n", 128 | " if test_all_inputs(inputs, params):\r\n", 129 | " break" 130 | ], 131 | "execution_count": null, 132 | "outputs": [ 133 | { 134 | "output_type": "stream", 135 | "text": [ 136 | "Iteration 0\n", 137 | "[0 0] -> 0\n", 138 | "[0 1] -> 0\n", 139 | "[1 0] -> 0\n", 140 | "[1 1] -> 0\n", 141 | "Iteration 100\n", 142 | "[0 0] -> 0\n", 143 | "[0 1] -> 1\n", 144 | "[1 0] -> 1\n", 145 | "[1 1] -> 1\n", 146 | "Iteration 200\n", 147 | "[0 0] -> 0\n", 148 | "[0 1] -> 1\n", 149 | "[1 0] -> 1\n", 150 | "[1 1] -> 0\n" 151 | ], 152 | "name": "stdout" 153 | } 154 | ] 155 | }, 156 | { 157 | "cell_type": "code", 158 | "metadata": { 159 | "id": "Smn6o3apQf5u" 160 | }, 161 | "source": [ 162 | "import jax.numpy as jnp\r\n", 163 | "from jax import grad, jit, vmap\r\n", 164 | "from jax import random" 165 | ], 166 | "execution_count": 7, 167 | "outputs": [] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "metadata": { 172 | "colab": { 173 | "base_uri": "https://localhost:8080/" 174 | }, 175 | "id": "eUGug3oiX7Q8", 176 | "outputId": "027506c5-36a2-4d95-d435-22a834af51f4" 177 | }, 178 | "source": [ 179 | "key = random.PRNGKey(0)\r\n", 180 | "x = random.normal(key, (10,))\r\n", 181 | "print(x)\r\n" 182 | ], 183 | "execution_count": 8, 184 | "outputs": [ 185 | { 186 | "output_type": "stream", 187 | "text": [ 188 | "[-0.3721109 0.26423115 -0.18252768 -0.7368197 -0.44030377 -0.1521442\n", 189 | " -0.67135346 -0.5908641 0.73168886 0.5673026 ]\n" 190 | ], 191 | "name": "stdout" 192 | } 193 | ] 194 | }, 195 | { 196 | "cell_type": "code", 197 | "metadata": { 198 | "colab": { 199 | "base_uri": "https://localhost:8080/" 200 | }, 201 | "id": "Ra-UbQlGX9S6", 202 | "outputId": "e426e152-7bf2-4f27-cc01-04cdf59d2a02" 203 | }, 204 | "source": [ 205 | "size = 3000\r\n", 206 | "x = random.normal(key, (size, size), dtype=jnp.float32)\r\n", 207 | "%timeit jnp.dot(x, x.T).block_until_ready() # runs on the GPU" 208 | ], 209 | "execution_count": 7, 210 | "outputs": [ 211 | { 212 | "output_type": "stream", 213 | "text": [ 214 | "1 loop, best of 3: 931 ms per loop\n" 215 | ], 216 | "name": "stdout" 217 | } 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "metadata": { 223 | "colab": { 224 | "base_uri": "https://localhost:8080/" 225 | }, 226 | "id": "R0wdkJr7YKMd", 227 | "outputId": "abc62879-4ff8-42fb-cc65-67d92b99dea1" 228 | }, 229 | "source": [ 230 | "import numpy as np\r\n", 231 | "x = np.random.normal(size=(size, size)).astype(np.float32)\r\n", 232 | "%timeit jnp.dot(x, x.T).block_until_ready()" 233 | ], 234 | "execution_count": 8, 235 | "outputs": [ 236 | { 237 | "output_type": "stream", 238 | "text": [ 239 | "1 loop, best of 3: 938 ms per loop\n" 240 | ], 241 | "name": "stdout" 242 | } 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "metadata": { 248 | "colab": { 249 | "base_uri": "https://localhost:8080/" 250 | }, 251 | "id": "nO2oQqSvYRz4", 252 | "outputId": "b09af564-0389-4ad0-d6b9-e5ea6da86e3a" 253 | }, 254 | "source": [ 255 | "from jax import device_put\r\n", 256 | "\r\n", 257 | "x = np.random.normal(size=(size, size)).astype(np.float32)\r\n", 258 | "x = device_put(x)\r\n", 259 | "%timeit jnp.dot(x, x.T).block_until_ready()" 260 | ], 261 | "execution_count": 9, 262 | "outputs": [ 263 | { 264 | "output_type": "stream", 265 | "text": [ 266 | "1 loop, best of 3: 903 ms per loop\n" 267 | ], 268 | "name": "stdout" 269 | } 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "metadata": { 275 | "colab": { 276 | "base_uri": "https://localhost:8080/" 277 | }, 278 | "id": "5k9sx9_IYZub", 279 | "outputId": "b1dd7e02-177c-4fe6-9dc4-8619efd0925f" 280 | }, 281 | "source": [ 282 | "x = np.random.normal(size=(size, size)).astype(np.float32)\r\n", 283 | "%timeit np.dot(x, x.T)" 284 | ], 285 | "execution_count": 10, 286 | "outputs": [ 287 | { 288 | "output_type": "stream", 289 | "text": [ 290 | "1 loop, best of 3: 452 ms per loop\n" 291 | ], 292 | "name": "stdout" 293 | } 294 | ] 295 | }, 296 | { 297 | "cell_type": "code", 298 | "metadata": { 299 | "colab": { 300 | "base_uri": "https://localhost:8080/" 301 | }, 302 | "id": "C8REjj8eYeRD", 303 | "outputId": "03d82285-9d14-495c-bfbf-5b8d3603c380" 304 | }, 305 | "source": [ 306 | "size = 3000\r\n", 307 | "x = random.normal(key, (size, size), dtype=jnp.float32)\r\n", 308 | "%timeit jnp.dot(x, x.T).block_until_ready() # runs on the GPU" 309 | ], 310 | "execution_count": 9, 311 | "outputs": [ 312 | { 313 | "output_type": "stream", 314 | "text": [ 315 | "The slowest run took 227.61 times longer than the fastest. This could mean that an intermediate result is being cached.\n", 316 | "1 loop, best of 3: 10.1 ms per loop\n" 317 | ], 318 | "name": "stdout" 319 | } 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "metadata": { 325 | "colab": { 326 | "base_uri": "https://localhost:8080/" 327 | }, 328 | "id": "OAAFywxgYo4-", 329 | "outputId": "560293e7-a3ed-4c39-d0fe-6b8c57012fc5" 330 | }, 331 | "source": [ 332 | "import numpy as np\r\n", 333 | "x = np.random.normal(size=(size, size)).astype(np.float32)\r\n", 334 | "%timeit jnp.dot(x, x.T).block_until_ready()" 335 | ], 336 | "execution_count": 10, 337 | "outputs": [ 338 | { 339 | "output_type": "stream", 340 | "text": [ 341 | "10 loops, best of 3: 34.9 ms per loop\n" 342 | ], 343 | "name": "stdout" 344 | } 345 | ] 346 | }, 347 | { 348 | "cell_type": "code", 349 | "metadata": { 350 | "colab": { 351 | "base_uri": "https://localhost:8080/" 352 | }, 353 | "id": "1l4tV68nY2wJ", 354 | "outputId": "8dd24500-56de-43ab-a170-8525b447cbf5" 355 | }, 356 | "source": [ 357 | "from jax import device_put\r\n", 358 | "\r\n", 359 | "x = np.random.normal(size=(size, size)).astype(np.float32)\r\n", 360 | "x = device_put(x)\r\n", 361 | "%timeit jnp.dot(x, x.T).block_until_ready()" 362 | ], 363 | "execution_count": 11, 364 | "outputs": [ 365 | { 366 | "output_type": "stream", 367 | "text": [ 368 | "10 loops, best of 3: 20.3 ms per loop\n" 369 | ], 370 | "name": "stdout" 371 | } 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "metadata": { 377 | "colab": { 378 | "base_uri": "https://localhost:8080/" 379 | }, 380 | "id": "vky4LhZOY8Jo", 381 | "outputId": "ba11a9cc-82b6-4a0e-ab1b-11944dd31a19" 382 | }, 383 | "source": [ 384 | "x = np.random.normal(size=(size, size)).astype(np.float32)\r\n", 385 | "%timeit np.dot(x, x.T)" 386 | ], 387 | "execution_count": 12, 388 | "outputs": [ 389 | { 390 | "output_type": "stream", 391 | "text": [ 392 | "1 loop, best of 3: 324 ms per loop\n" 393 | ], 394 | "name": "stdout" 395 | } 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "metadata": { 401 | "colab": { 402 | "base_uri": "https://localhost:8080/" 403 | }, 404 | "id": "hxk0Xg3IZACW", 405 | "outputId": "a181e205-71df-460f-ebc5-6dd60275912e" 406 | }, 407 | "source": [ 408 | "def selu(x, alpha=1.67, lmbda=1.05):\r\n", 409 | " return lmbda * jnp.where(x > 0, x, alpha * jnp.exp(x) - alpha)\r\n", 410 | "\r\n", 411 | "x = random.normal(key, (1000000,))\r\n", 412 | "%timeit selu(x).block_until_ready()" 413 | ], 414 | "execution_count": 13, 415 | "outputs": [ 416 | { 417 | "output_type": "stream", 418 | "text": [ 419 | "The slowest run took 159.08 times longer than the fastest. This could mean that an intermediate result is being cached.\n", 420 | "1 loop, best of 3: 2.36 ms per loop\n" 421 | ], 422 | "name": "stdout" 423 | } 424 | ] 425 | }, 426 | { 427 | "cell_type": "code", 428 | "metadata": { 429 | "colab": { 430 | "base_uri": "https://localhost:8080/" 431 | }, 432 | "id": "wzO4yKtmZMO1", 433 | "outputId": "56df94c1-3ec4-4ed9-b205-93fa1a9e2eca" 434 | }, 435 | "source": [ 436 | "selu_jit = jit(selu)\r\n", 437 | "%timeit selu_jit(x).block_until_ready()" 438 | ], 439 | "execution_count": 14, 440 | "outputs": [ 441 | { 442 | "output_type": "stream", 443 | "text": [ 444 | "The slowest run took 433.81 times longer than the fastest. This could mean that an intermediate result is being cached.\n", 445 | "1000 loops, best of 3: 270 µs per loop\n" 446 | ], 447 | "name": "stdout" 448 | } 449 | ] 450 | }, 451 | { 452 | "cell_type": "code", 453 | "metadata": { 454 | "colab": { 455 | "base_uri": "https://localhost:8080/" 456 | }, 457 | "id": "ydt0Ryo6ZeNA", 458 | "outputId": "7ef2b5c2-63b9-402e-94a6-37016800b68d" 459 | }, 460 | "source": [ 461 | "x" 462 | ], 463 | "execution_count": 15, 464 | "outputs": [ 465 | { 466 | "output_type": "execute_result", 467 | "data": { 468 | "text/plain": [ 469 | "DeviceArray([ 1.99376 , 0.20781846, -0.34406224, ..., 0.03467206,\n", 470 | " 0.7103182 , 0.1965587 ], dtype=float32)" 471 | ] 472 | }, 473 | "metadata": { 474 | "tags": [] 475 | }, 476 | "execution_count": 15 477 | } 478 | ] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "metadata": { 483 | "id": "6ZAVEg5vZTYi" 484 | }, 485 | "source": [ 486 | "" 487 | ], 488 | "execution_count": null, 489 | "outputs": [] 490 | } 491 | ] 492 | } -------------------------------------------------------------------------------- /object.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/object.jpg -------------------------------------------------------------------------------- /scratch_model_weight_changes_affect_accuracy.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "scratch_model_weight_changes_affect_accuracy.ipynb", 7 | "provenance": [], 8 | "authorship_tag": "ABX9TyPdI1D7PI0pSgwlrQULZj1P", 9 | "include_colab_link": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "markdown", 19 | "metadata": { 20 | "id": "view-in-github", 21 | "colab_type": "text" 22 | }, 23 | "source": [ 24 | "\"Open" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "metadata": { 30 | "id": "HomHZrXgZHtm", 31 | "colab_type": "code", 32 | "colab": {} 33 | }, 34 | "source": [ 35 | "import numpy as np\n", 36 | "\n", 37 | "# আমাদের প্রেডিকশন করার জন্য ডেটা পয়েন্ট, ছবির সাথে মিলিয়ে দেখুন\n", 38 | "input_data = np.array([2, 3])\n", 39 | "\n", 40 | "# আমাদের ডিকশনারী\n", 41 | "weights = {'node_0': np.array([1, 1]),\n", 42 | " 'node_1': np.array([-1, 1]),\n", 43 | " 'output': np.array([2, -1])\n", 44 | " }" 45 | ], 46 | "execution_count": 0, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "metadata": { 52 | "id": "v1b80TK6ZSWL", 53 | "colab_type": "code", 54 | "outputId": "7ac3581a-e48e-41f4-a39c-856b6603f0b0", 55 | "colab": { 56 | "base_uri": "https://localhost:8080/", 57 | "height": 52 58 | } 59 | }, 60 | "source": [ 61 | "# node 0 এর ভ্যালু ক্যালকুলেট করি: node_0_value\n", 62 | "node_0_value = (input_data * weights['node_0']).sum()\n", 63 | "\n", 64 | "# node ১ এর ভ্যালু ক্যালকুলেট করি: node_1_value\n", 65 | "node_1_value = (input_data * weights['node_1']).sum()\n", 66 | "\n", 67 | "# নোডগুলোর ভ্য়ালুগুলোকে অ্যারেতে রাখি: hidden_layer_outputs\n", 68 | "hidden_layer_outputs = np.array([node_0_value, node_1_value])\n", 69 | "\n", 70 | "# আউটপুট ক্যালকুলেট করি: output\n", 71 | "output = (hidden_layer_outputs * weights['output']).sum()\n", 72 | "\n", 73 | "# আউটপুট প্রিন্ট করে দেখি\n", 74 | "print(hidden_layer_outputs)\n", 75 | "print(output)" 76 | ], 77 | "execution_count": 5, 78 | "outputs": [ 79 | { 80 | "output_type": "stream", 81 | "text": [ 82 | "[5 1]\n", 83 | "9\n" 84 | ], 85 | "name": "stdout" 86 | } 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "metadata": { 92 | "id": "kNrRxANoZiet", 93 | "colab_type": "code", 94 | "outputId": "5af736d6-45ae-43d8-f7eb-f369a55213ca", 95 | "colab": { 96 | "base_uri": "https://localhost:8080/", 97 | "height": 35 98 | } 99 | }, 100 | "source": [ 101 | "# নতুন ওয়েট এবং ইনপুট ডেটা \n", 102 | "weights = np.array([1, 2])\n", 103 | "input_data = np.array([3, 4]) \n", 104 | "\n", 105 | "# প্রেডিকশন ক্যাল্কুলেট করি: preds\n", 106 | "preds = (weights * input_data).sum()\n", 107 | "\n", 108 | "# ধরি আমাদের টার্গেট ৬\n", 109 | "target = 6\n", 110 | "\n", 111 | "# এরর ক্যালকুলেট করি: error\n", 112 | "error = preds - target\n", 113 | "\n", 114 | "# স্লোপ ক্যালকুলেট করি: slope\n", 115 | "slope = 2 * input_data * error\n", 116 | "\n", 117 | "# স্লোপ প্রিন্ট করি \n", 118 | "print(slope)\n" 119 | ], 120 | "execution_count": 6, 121 | "outputs": [ 122 | { 123 | "output_type": "stream", 124 | "text": [ 125 | "[30 40]\n" 126 | ], 127 | "name": "stdout" 128 | } 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "metadata": { 134 | "id": "B_U_OVOkaJXp", 135 | "colab_type": "code", 136 | "outputId": "038ce0a4-e45f-4e6e-ab0d-a5bb886c9bd8", 137 | "colab": { 138 | "base_uri": "https://localhost:8080/", 139 | "height": 52 140 | } 141 | }, 142 | "source": [ 143 | "# লার্নিং রেট ঠিক করি: learning_rate\n", 144 | "learning_rate = 0.01\n", 145 | "\n", 146 | "# স্লোপ/গ্রেডিয়েন্ট ক্যালকুলেট করি: gradient\n", 147 | "gradient = 2 * input_data * error\n", 148 | "\n", 149 | "# ওয়েট আপডেট করি: weights_updated\n", 150 | "weights_updated = weights - learning_rate * gradient\n", 151 | "\n", 152 | "# প্রেডিকশন আপডেট নেই : preds_updated\n", 153 | "preds_updated = (weights_updated * input_data).sum()\n", 154 | "\n", 155 | "# এররের আপডেট নেই: error_updated\n", 156 | "error_updated = preds_updated - target\n", 157 | "\n", 158 | "# শুরুর এরর প্রিন্ট করি \n", 159 | "print(error)\n", 160 | "\n", 161 | "# নতুন এরর প্রিন্ট করি \n", 162 | "print(error_updated)" 163 | ], 164 | "execution_count": 7, 165 | "outputs": [ 166 | { 167 | "output_type": "stream", 168 | "text": [ 169 | "5\n", 170 | "2.5\n" 171 | ], 172 | "name": "stdout" 173 | } 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "metadata": { 179 | "id": "sBmSDPfm7B__", 180 | "colab_type": "code", 181 | "colab": { 182 | "base_uri": "https://localhost:8080/", 183 | "height": 87 184 | }, 185 | "outputId": "9d4c1158-9f32-4476-fdf0-9c6368226f49" 186 | }, 187 | "source": [ 188 | "import numpy as np\n", 189 | "\n", 190 | "# আমাদের প্রেডিকশন করার জন্য ডেটা পয়েন্ট, ছবির সাথে মিলিয়ে দেখুন\n", 191 | "input_data = np.array([0, 3])\n", 192 | "\n", 193 | "# স্যাম্পল ওয়েট যা পাল্টে দিয়েছি আমরা \n", 194 | "weights_0 = {'node_0': [2, 1],\n", 195 | " 'node_1': [1, 2],\n", 196 | " 'output': [1, 1]\n", 197 | " }\n", 198 | "\n", 199 | "# আসল টার্গেট ভ্যালু, এরর বের করার জন্য লাগবে \n", 200 | "target_actual = 3\n", 201 | "\n", 202 | "# দুটো মেথড ডিফাইন করি \n", 203 | "def relu(input):\n", 204 | " output = max(0, input)\n", 205 | " return output\n", 206 | "\n", 207 | "def predict_with_network(input_data_row, weights):\n", 208 | " node_0_input = (input_data_row * weights['node_0']).sum()\n", 209 | " # print(node_0_input)\n", 210 | " node_0_output = relu(node_0_input)\n", 211 | " # print(node_0_output)\n", 212 | "\n", 213 | " node_1_input = (input_data_row * weights['node_1']).sum()\n", 214 | " node_1_output = relu(node_1_input)\n", 215 | " \n", 216 | " hidden_layer_outputs = np.array([node_0_output, node_1_output])\n", 217 | " \n", 218 | " input_to_final_layer = (hidden_layer_outputs * weights['output']).sum()\n", 219 | " model_output = relu(input_to_final_layer)\n", 220 | " return model_output\n", 221 | " \n", 222 | "\n", 223 | "# শুরুর ওয়েট দিয়ে প্রেডিকশন করি \n", 224 | "model_output_0 = predict_with_network(input_data, weights_0)\n", 225 | "\n", 226 | "# এরর ক্যালকুলেট করি: error_0\n", 227 | "error_0 = model_output_0 - target_actual\n", 228 | "\n", 229 | "# নতুন ওয়েট দেই যাতে টার্গেট প্রেডিকশন (3) ধরতে পারে: weights_1\n", 230 | "weights_1 = {'node_0': [2, 1],\n", 231 | " 'node_1': [1, 2],\n", 232 | " 'output': [1, 0]\n", 233 | " }\n", 234 | "\n", 235 | "# নতুন ওয়েট দিয়ে প্রেডিকশন: model_output_1\n", 236 | "model_output_1 = predict_with_network(input_data, weights_1)\n", 237 | "\n", 238 | "# আবার এরর ক্যালকুলেট করি: error_1\n", 239 | "error_1 = model_output_1 - target_actual\n", 240 | "\n", 241 | "# সবকিছু প্রিন্ট করে দেখি \n", 242 | "\n", 243 | "print(model_output_0)\n", 244 | "print(model_output_1)\n", 245 | "print(error_0)\n", 246 | "print(error_1)" 247 | ], 248 | "execution_count": 8, 249 | "outputs": [ 250 | { 251 | "output_type": "stream", 252 | "text": [ 253 | "9\n", 254 | "3\n", 255 | "6\n", 256 | "0\n" 257 | ], 258 | "name": "stdout" 259 | } 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "metadata": { 265 | "id": "bNaR0dxG9MFW", 266 | "colab_type": "code", 267 | "colab": { 268 | "base_uri": "https://localhost:8080/", 269 | "height": 87 270 | }, 271 | "outputId": "82ce5d31-8242-48f5-a935-eb50f7096f29" 272 | }, 273 | "source": [ 274 | "import numpy as np\n", 275 | "\n", 276 | "# আমাদের প্রেডিকশন করার জন্য ডেটা পয়েন্ট, ছবির সাথে মিলিয়ে দেখুন\n", 277 | "input_data = np.array([-1, 2])\n", 278 | "\n", 279 | "# আমাদের ডিকশনারী\n", 280 | "weights = {'node_0': np.array([3, 3]),\n", 281 | " 'node_1': np.array([1, 5]),\n", 282 | " 'output': np.array([2, -1])\n", 283 | " }\n", 284 | "\n", 285 | "def relu(input):\n", 286 | " '''রেল্যু ফাংশনকে ডিফাইন করে দিচ্ছি এখানে'''\n", 287 | " # ইনপুটে যা পাবো সেটাকে ম্যাক্সিমাম যা আসবে, অথবা ঋনাত্বক আসলে \"০\" : output\n", 288 | " output = max(0, input)\n", 289 | " \n", 290 | " # Return the value just calculated\n", 291 | " return(output)\n", 292 | "\n", 293 | "# নোড ১ এর ভ্যালু ক্যালকুলেট করি: node_0_output\n", 294 | "node_0_input = (input_data * weights['node_0']).sum()\n", 295 | "node_0_output = relu(node_0_input)\n", 296 | "\n", 297 | "# নোড ২ এর ভ্যালু ক্যালকুলেট করি: node_1_output\n", 298 | "node_1_input = (input_data * weights['node_1']).sum()\n", 299 | "node_1_output = relu(node_1_input)\n", 300 | "\n", 301 | "# নতুন ভ্যালুগুলোকে অ্যারেতে বসাই: hidden_layer_outputs\n", 302 | "hidden_layer_outputs = np.array([node_0_output, node_1_output])\n", 303 | "\n", 304 | "# মডেলের আউটপুট ক্যালকুলেট করি (রেল্যুকে সরাসরি ব্যবহার না করে)\n", 305 | "model_output = (hidden_layer_outputs * weights['output']).sum()\n", 306 | "\n", 307 | "# Print model output\n", 308 | "print(node_0_output)\n", 309 | "print(node_1_output)\n", 310 | "print(hidden_layer_outputs)\n", 311 | "print(model_output)" 312 | ], 313 | "execution_count": 9, 314 | "outputs": [ 315 | { 316 | "output_type": "stream", 317 | "text": [ 318 | "3\n", 319 | "9\n", 320 | "[3 9]\n", 321 | "-3\n" 322 | ], 323 | "name": "stdout" 324 | } 325 | ] 326 | } 327 | ] 328 | } -------------------------------------------------------------------------------- /temp/word_embeddings.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "accelerator": "GPU", 6 | "colab": { 7 | "name": "word_embeddings.ipynb", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "include_colab_link": true 11 | }, 12 | "kernelspec": { 13 | "name": "python3", 14 | "display_name": "Python 3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "view-in-github", 22 | "colab_type": "text" 23 | }, 24 | "source": [ 25 | "\"Open" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "metadata": { 31 | "id": "o2-QgXTj3mw_", 32 | "colab_type": "code", 33 | "colab": { 34 | "base_uri": "https://localhost:8080/", 35 | "height": 34 36 | }, 37 | "outputId": "d5c97527-91ca-4d73-b949-7c4201e4f33f" 38 | }, 39 | "source": [ 40 | "try:\n", 41 | " # শুধুমাত্র টেন্সর-ফ্লো ২.x ব্যবহার করবো \n", 42 | " %tensorflow_version 2.x\n", 43 | "except Exception:\n", 44 | " pass\n", 45 | "\n", 46 | "import tensorflow as tf\n", 47 | "from tensorflow import keras\n", 48 | "from tensorflow.keras import layers" 49 | ], 50 | "execution_count": 1, 51 | "outputs": [ 52 | { 53 | "output_type": "stream", 54 | "text": [ 55 | "TensorFlow 2.x selected.\n" 56 | ], 57 | "name": "stdout" 58 | } 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "metadata": { 64 | "id": "HEaBBaCg4iaM", 65 | "colab_type": "code", 66 | "colab": {} 67 | }, 68 | "source": [ 69 | "embedding_layer = layers.Embedding(1000, 5)" 70 | ], 71 | "execution_count": 0, 72 | "outputs": [] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "metadata": { 77 | "colab_type": "code", 78 | "id": "0YUjPgP7w0PO", 79 | "colab": { 80 | "base_uri": "https://localhost:8080/", 81 | "height": 85 82 | }, 83 | "outputId": "4245ebcc-d86c-4658-f34a-a6bf9055cfcb" 84 | }, 85 | "source": [ 86 | "result = embedding_layer(tf.constant([1,2,3]))\n", 87 | "result.numpy()" 88 | ], 89 | "execution_count": 3, 90 | "outputs": [ 91 | { 92 | "output_type": "execute_result", 93 | "data": { 94 | "text/plain": [ 95 | "array([[-0.03846053, -0.00674887, -0.01975296, -0.02325729, 0.04202225],\n", 96 | " [ 0.02986335, 0.01062097, -0.03452694, -0.0296132 , 0.00191165],\n", 97 | " [ 0.02632796, -0.04222443, -0.03406482, -0.048172 , -0.01869417]],\n", 98 | " dtype=float32)" 99 | ] 100 | }, 101 | "metadata": { 102 | "tags": [] 103 | }, 104 | "execution_count": 3 105 | } 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "metadata": { 111 | "id": "DBKDbyfvy2Pm", 112 | "colab_type": "code", 113 | "colab": { 114 | "base_uri": "https://localhost:8080/", 115 | "height": 129 116 | }, 117 | "outputId": "cffbcec0-6c57-4a95-df61-721e9abed070" 118 | }, 119 | "source": [ 120 | "result = embedding_layer(tf.constant([১,২,৩]))\n", 121 | "result.numpy()" 122 | ], 123 | "execution_count": 4, 124 | "outputs": [ 125 | { 126 | "output_type": "error", 127 | "ename": "SyntaxError", 128 | "evalue": "ignored", 129 | "traceback": [ 130 | "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m result = embedding_layer(tf.constant([১,২,৩]))\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid character in identifier\n" 131 | ] 132 | } 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "metadata": { 138 | "id": "Gl97oboX0ydP", 139 | "colab_type": "code", 140 | "colab": { 141 | "base_uri": "https://localhost:8080/", 142 | "height": 289 143 | }, 144 | "outputId": "cc8ac26c-7b4c-4bcb-d6b9-07954a4671bd" 145 | }, 146 | "source": [ 147 | "!pip install spacy" 148 | ], 149 | "execution_count": 5, 150 | "outputs": [ 151 | { 152 | "output_type": "stream", 153 | "text": [ 154 | "Requirement already satisfied: spacy in /usr/local/lib/python3.6/dist-packages (2.1.8)\n", 155 | "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.2)\n", 156 | "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.21.0)\n", 157 | "Requirement already satisfied: thinc<7.1.0,>=7.0.8 in /usr/local/lib/python3.6/dist-packages (from spacy) (7.0.8)\n", 158 | "Requirement already satisfied: srsly<1.1.0,>=0.0.6 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.1.0)\n", 159 | "Requirement already satisfied: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.9.6)\n", 160 | "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.2)\n", 161 | "Requirement already satisfied: wasabi<1.1.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.2.2)\n", 162 | "Requirement already satisfied: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.1)\n", 163 | "Requirement already satisfied: blis<0.3.0,>=0.2.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.2.4)\n", 164 | "Requirement already satisfied: numpy>=1.15.0 in /tensorflow-2.0.0/python3.6 (from spacy) (1.17.3)\n", 165 | "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n", 166 | "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2019.9.11)\n", 167 | "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.24.3)\n", 168 | "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.8)\n", 169 | "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<7.1.0,>=7.0.8->spacy) (4.28.1)\n" 170 | ], 171 | "name": "stdout" 172 | } 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "metadata": { 178 | "id": "C_u-D5y41dxW", 179 | "colab_type": "code", 180 | "colab": { 181 | "base_uri": "https://localhost:8080/", 182 | "height": 68 183 | }, 184 | "outputId": "e2bf5ea2-cdf4-4245-94b6-5b1a415cd59d" 185 | }, 186 | "source": [ 187 | "!python -m spacy download bn_core_news_sm" 188 | ], 189 | "execution_count": 6, 190 | "outputs": [ 191 | { 192 | "output_type": "stream", 193 | "text": [ 194 | "\n", 195 | "\u001b[38;5;1m✘ No compatible model found for 'bn_core_news_sm' (spaCy v2.1.8).\u001b[0m\n", 196 | "\n" 197 | ], 198 | "name": "stdout" 199 | } 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "metadata": { 205 | "id": "PV-iFPP82axg", 206 | "colab_type": "code", 207 | "colab": { 208 | "base_uri": "https://localhost:8080/", 209 | "height": 479 210 | }, 211 | "outputId": "b5d84b00-082a-455b-b37b-d17d97ee007b" 212 | }, 213 | "source": [ 214 | "pip install https://github.com/banglakit/spacy-models/releases/download/bn_core_news_sm-2.0.0/bn_core_news_sm-2.0.0.tar.gz" 215 | ], 216 | "execution_count": 7, 217 | "outputs": [ 218 | { 219 | "output_type": "stream", 220 | "text": [ 221 | "Collecting https://github.com/banglakit/spacy-models/releases/download/bn_core_news_sm-2.0.0/bn_core_news_sm-2.0.0.tar.gz\n", 222 | "\u001b[?25l Downloading https://github.com/banglakit/spacy-models/releases/download/bn_core_news_sm-2.0.0/bn_core_news_sm-2.0.0.tar.gz (40.5MB)\n", 223 | "\u001b[K |████████████████████████████████| 40.5MB 223kB/s \n", 224 | "\u001b[?25hRequirement already satisfied: spacy>=>=2.0.12 in /usr/local/lib/python3.6/dist-packages (from bn-core-news-sm==2.0.0) (2.1.8)\n", 225 | "Requirement already satisfied: wasabi<1.1.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (0.2.2)\n", 226 | "Requirement already satisfied: thinc<7.1.0,>=7.0.8 in /usr/local/lib/python3.6/dist-packages (from spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (7.0.8)\n", 227 | "Requirement already satisfied: blis<0.3.0,>=0.2.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (0.2.4)\n", 228 | "Requirement already satisfied: numpy>=1.15.0 in /tensorflow-2.0.0/python3.6 (from spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (1.17.3)\n", 229 | "Requirement already satisfied: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (0.9.6)\n", 230 | "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (2.0.2)\n", 231 | "Requirement already satisfied: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (2.0.1)\n", 232 | "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (2.21.0)\n", 233 | "Requirement already satisfied: srsly<1.1.0,>=0.0.6 in /usr/local/lib/python3.6/dist-packages (from spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (0.1.0)\n", 234 | "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (1.0.2)\n", 235 | "Requirement already satisfied: tqdm<5.0.0,>=4.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<7.1.0,>=7.0.8->spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (4.28.1)\n", 236 | "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (3.0.4)\n", 237 | "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (2.8)\n", 238 | "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (1.24.3)\n", 239 | "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=>=2.0.12->bn-core-news-sm==2.0.0) (2019.9.11)\n", 240 | "Building wheels for collected packages: bn-core-news-sm\n", 241 | " Building wheel for bn-core-news-sm (setup.py) ... \u001b[?25l\u001b[?25hdone\n", 242 | " Created wheel for bn-core-news-sm: filename=bn_core_news_sm-2.0.0-cp36-none-any.whl size=41254297 sha256=99fcea737cd7f45d9aeed8e4be6bb94237c779869ef398c0da06d342c5e32384\n", 243 | " Stored in directory: /root/.cache/pip/wheels/92/ef/60/567e70071236fd8535ae8aa080933500dc0a0455bdc0287e4d\n", 244 | "Successfully built bn-core-news-sm\n", 245 | "Installing collected packages: bn-core-news-sm\n", 246 | "Successfully installed bn-core-news-sm-2.0.0\n" 247 | ], 248 | "name": "stdout" 249 | } 250 | ] 251 | }, 252 | { 253 | "cell_type": "code", 254 | "metadata": { 255 | "id": "W-XdY2eG0dgZ", 256 | "colab_type": "code", 257 | "colab": { 258 | "base_uri": "https://localhost:8080/", 259 | "height": 377 260 | }, 261 | "outputId": "c4b81ec4-2c61-447e-8ce6-9e0faf8249a3" 262 | }, 263 | "source": [ 264 | "import spacy\n", 265 | "import bn_core_news_sm\n", 266 | "\n", 267 | "nlp = bn_core_news_sm.load()\n", 268 | "doc = nlp(u'আমি বাংলায় গান গাই। তুমি কি গাও?')" 269 | ], 270 | "execution_count": 14, 271 | "outputs": [ 272 | { 273 | "output_type": "error", 274 | "ename": "KeyError", 275 | "evalue": "ignored", 276 | "traceback": [ 277 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 278 | "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", 279 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbn_core_news_sm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mnlp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbn_core_news_sm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mu'আমি বাংলায় গান গাই। তুমি কি গাও?'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 280 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/bn_core_news_sm/__init__.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(**overrides)\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m \u001b[0mlang\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_model_from_init_py\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m__file__\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 50\u001b[0m \u001b[0mlang\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfactories\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'sbd'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mSentenceSegmenter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[0mlang\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfactories\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'sentencizer'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mSentenceSegmenter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 281 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/spacy/util.py\u001b[0m in \u001b[0;36mload_model_from_init_py\u001b[0;34m(init_file, **overrides)\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mmodel_path\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE052\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath2str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mload_model_from_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 282 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/spacy/util.py\u001b[0m in \u001b[0;36mload_model_from_path\u001b[0;34m(model_path, meta, **overrides)\u001b[0m\n\u001b[1;32m 175\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdisable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 176\u001b[0m \u001b[0mconfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"pipeline_args\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m \u001b[0mcomponent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_pipe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 178\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_pipe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomponent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_disk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 283 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/spacy/language.py\u001b[0m in \u001b[0;36mcreate_pipe\u001b[0;34m(self, name, config)\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfactories\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 262\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"sbd\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 263\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE108\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 264\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 265\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE002\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 284 | "\u001b[0;31mKeyError\u001b[0m: \"[E108] As of spaCy v2.1, the pipe name `sbd` has been deprecated in favor of the pipe name `sentencizer`, which does the same thing. For example, use `nlp.create_pipeline('sentencizer')`\"" 285 | ] 286 | } 287 | ] 288 | }, 289 | { 290 | "cell_type": "code", 291 | "metadata": { 292 | "id": "2w43jPG496qk", 293 | "colab_type": "code", 294 | "colab": { 295 | "base_uri": "https://localhost:8080/", 296 | "height": 394 297 | }, 298 | "outputId": "cc74f8df-50ae-47bb-e061-c61dc3f9f515" 299 | }, 300 | "source": [ 301 | "import spacy\n", 302 | "import bn_core_news_sm\n", 303 | "\n", 304 | "nlp = bn_core_news_sm.load()\n", 305 | "doc = nlp(u'আমি বাংলায় গান গাই। তুমি কি গাও?')\n", 306 | "\n", 307 | "# Create list of word tokens\n", 308 | "token_list = []\n", 309 | "for token in doc:\n", 310 | " token_list.append(token.text)\n", 311 | "print(token_list)" 312 | ], 313 | "execution_count": 11, 314 | "outputs": [ 315 | { 316 | "output_type": "error", 317 | "ename": "KeyError", 318 | "evalue": "ignored", 319 | "traceback": [ 320 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 321 | "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", 322 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbn_core_news_sm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mnlp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbn_core_news_sm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mu'আমি বাংলায় গান গাই। তুমি কি গাও?'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 323 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/bn_core_news_sm/__init__.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(**overrides)\u001b[0m\n\u001b[1;32m 47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 48\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m \u001b[0mlang\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_model_from_init_py\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m__file__\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 50\u001b[0m \u001b[0mlang\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfactories\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'sbd'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mSentenceSegmenter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[0mlang\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfactories\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'sentencizer'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mSentenceSegmenter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcfg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 324 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/spacy/util.py\u001b[0m in \u001b[0;36mload_model_from_init_py\u001b[0;34m(init_file, **overrides)\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mmodel_path\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE052\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath2str\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mload_model_from_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 325 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/spacy/util.py\u001b[0m in \u001b[0;36mload_model_from_path\u001b[0;34m(model_path, meta, **overrides)\u001b[0m\n\u001b[1;32m 175\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdisable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 176\u001b[0m \u001b[0mconfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmeta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"pipeline_args\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m \u001b[0mcomponent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_pipe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 178\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_pipe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomponent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_disk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 326 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/spacy/language.py\u001b[0m in \u001b[0;36mcreate_pipe\u001b[0;34m(self, name, config)\u001b[0m\n\u001b[1;32m 261\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfactories\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 262\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"sbd\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 263\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE108\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 264\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 265\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE002\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 327 | "\u001b[0;31mKeyError\u001b[0m: \"[E108] As of spaCy v2.1, the pipe name `sbd` has been deprecated in favor of the pipe name `sentencizer`, which does the same thing. For example, use `nlp.create_pipeline('sentencizer')`\"" 328 | ] 329 | } 330 | ] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "metadata": { 335 | "id": "SFDpZXCr_yWz", 336 | "colab_type": "code", 337 | "colab": { 338 | "base_uri": "https://localhost:8080/", 339 | "height": 374 340 | }, 341 | "outputId": "f262f437-267f-4476-83b7-13735de6388c" 342 | }, 343 | "source": [ 344 | "import spacy\n", 345 | "\n", 346 | "nlp = spacy.load(\"bn_core_news_sm\")\n", 347 | "doc = nlp(u'আমি বাংলায় গান গাই। তুমি কি গাও?')\n", 348 | "for token in doc:\n", 349 | " print(token.text, token.pos_, token.dep_)" 350 | ], 351 | "execution_count": 13, 352 | "outputs": [ 353 | { 354 | "output_type": "error", 355 | "ename": "OSError", 356 | "evalue": "ignored", 357 | "traceback": [ 358 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 359 | "\u001b[0;31mOSError\u001b[0m Traceback (most recent call last)", 360 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mspacy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mnlp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspacy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"bn_core_news_sm\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mu'আমি বাংলায় গান গাই। তুমি কি গাও?'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 361 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/spacy/__init__.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(name, **overrides)\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdepr_path\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0mdeprecation_warning\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mWarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mW001\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepr_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 362 | "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/spacy/util.py\u001b[0m in \u001b[0;36mload_model\u001b[0;34m(name, **overrides)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"exists\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# Path or Path-like to model data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 138\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mload_model_from_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE050\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 363 | "\u001b[0;31mOSError\u001b[0m: [E050] Can't find model 'bn_core_news_sm'. It doesn't seem to be a shortcut link, a Python package or a valid path to a data directory." 364 | ] 365 | } 366 | ] 367 | } 368 | ] 369 | } -------------------------------------------------------------------------------- /tensor_numpy_shape_batch.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "tensor_numpy_shape_batch.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "include_colab_link": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "markdown", 19 | "metadata": { 20 | "id": "view-in-github", 21 | "colab_type": "text" 22 | }, 23 | "source": [ 24 | "\"Open" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": { 30 | "id": "yjx49CwPUtSP", 31 | "colab_type": "text" 32 | }, 33 | "source": [ 34 | "## নিউরাল নেটওয়ার্কে ডাটা কিভাবে থাকে?\n", 35 | "(খসড়া)\n", 36 | "\n", 37 | "নিউরাল নেটওয়ার্কে কিভাবে ডাটা রিপ্রেজেন্টেশন হয় সেটা হাতেকলমে দেখলে ব্যাপারটা আরো পরিস্কার হবে। “শূন্য থেকে পাইথন মেশিন লার্নিং” বইটাতে এই ব্যাপারে ধারণা দিলেও সেটাকে একটু অন্যভাবে এখানে আলাপ করতে চাচ্ছি। নিউরাল নেটওয়ার্কে যখন ডাটাকে স্টোর করতে গিয়ে যে অনেক ডাইমেনশনের ‘নামপাই’ অথবা ‘টেন্সরফ্লো’তে n সংখ্যক অ্যারে ব্যবহার করছি সেটাকে আমরা ‘টেন্সর’ বলতে পারি। এই ‘টেন্সর’ হচ্ছে ডিপ লার্নিং এর একটা বেসিক অংশ যাকে গুগল ‘টেন্সর’ এর ‘ফ্লো’ মানে টেন্সরের গতিধারা দিয়ে 'টেন্সরফ্লো' বলে একটা ডিপ লার্নিং লাইব্রেরি তৈরি করেছে। আবারো বলছি, টেন্সর এর ফ্লো। টেন্সরফ্লো এবং নামপাই এর ভেতরের অপারেশন কিভাবে এই টেন্সর দিয়ে হচ্ছে সেটাই দেখাবো এখানে।\n", 38 | "\n", 39 | "আমাদের সবচেয়ে ছোট ডাটা কন্টেইনার হচ্ছে এই ‘টেন্সর’ যা প্রায় সব সময় সংখ্যা নিয়ে কাজ করে। কম্পিউটার তো আর সংখ্যা ছাড়া কিছু চেনে না। কেরাস এর জনক ‘ফ্রাঁসোয়া শোলে’ এর একটা অসাধারণ বই আছে এই ধারণাগুলোকে নিয়ে। বর্তমান ‘টেন্সরফ্লো’ এর নতুন ফিচারগুলো এই বইটাতে কাভার না করলেও অনেক বেসিক জিনিস জানতে আমি প্রায় ফেরত যাই ওই “ডিপ লার্নিং উইথ পাইথন” বইটাতে। কিছুটা পুরনো তবে এখনো কাজে লাগে। সেখান থেকে কিছু ধারনা নিয়েছি এখানে।" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": { 45 | "id": "-wfD_5eOxMVG", 46 | "colab_type": "text" 47 | }, 48 | "source": [ 49 | "‘নামপাই’ এবং ‘টেন্সরফ্লো’ এর ‘টেন্সর’ প্রায় একই জিনিস। দুটোই N ডাইমেনশন অ্যারে লাইব্রেরি। নামপাই স্বয়ংক্রিয়ভাবে ডেরিভেটিভ কম্পিউট করতে পারে না। এর পাশাপাশি জিপিইউ ব্যবহার করতে পারে না। তাছাড়া দুটোর কাজ প্রায় একই। যেদুটো কাজ পারছেনা নামপাই, সেগুলো আমাদের কাজে এগুলো এমুহুর্তে লাগবে না। এছাড়া ‘টেন্সরফ্লো’তে নামপাই বলে আলাদা মেথড আছে যা দেখাবো সামনে। \n", 50 | "\n", 51 | "### ১. স্কেলার (০ ডাইমেনশনের ‘টেন্সর’) \n", 52 | "\n", 53 | "যেই টেন্সরে একটা মাত্র সংখ্যা থাকে সেটাকে আমরা স্কেলার টেন্সর বলতে পারি। একে ০ ডাইমেনশন টেন্সর বলা হচ্ছে যেখানে ০ এক্সিস মানে ০ দিকে এর ডাইমেনশন আছে। আমরা নামপাই টেন্সরে এক্সিস দেখতে পারি ndim অ্যাট্রিবিউট দিয়ে। একটা টেন্সরে এক্সিসের সংখ্যাকে আমরা অনেক সময় রেঙ্ক “rank” বলি। " 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "metadata": { 59 | "id": "jdWuhV9_Un1s", 60 | "colab_type": "code", 61 | "outputId": "4f964b69-131a-4e4c-bb49-7aa73eb48bd3", 62 | "colab": { 63 | "base_uri": "https://localhost:8080/", 64 | "height": 35 65 | } 66 | }, 67 | "source": [ 68 | "import numpy as np\n", 69 | "ক = np.array(12)\n", 70 | "ক" 71 | ], 72 | "execution_count": 1, 73 | "outputs": [ 74 | { 75 | "output_type": "execute_result", 76 | "data": { 77 | "text/plain": [ 78 | "array(12)" 79 | ] 80 | }, 81 | "metadata": { 82 | "tags": [] 83 | }, 84 | "execution_count": 1 85 | } 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "metadata": { 91 | "id": "dMkmYaHOVQrR", 92 | "colab_type": "code", 93 | "outputId": "92d8edc1-ed84-4217-881b-ee0804260c00", 94 | "colab": { 95 | "base_uri": "https://localhost:8080/", 96 | "height": 35 97 | } 98 | }, 99 | "source": [ 100 | "ক.ndim" 101 | ], 102 | "execution_count": 2, 103 | "outputs": [ 104 | { 105 | "output_type": "execute_result", 106 | "data": { 107 | "text/plain": [ 108 | "0" 109 | ] 110 | }, 111 | "metadata": { 112 | "tags": [] 113 | }, 114 | "execution_count": 2 115 | } 116 | ] 117 | }, 118 | { 119 | "cell_type": "markdown", 120 | "metadata": { 121 | "id": "iJ77_EG_q4Yo", 122 | "colab_type": "text" 123 | }, 124 | "source": [ 125 | "### ২. ভেক্টর (১ ডাইমেনশনের ‘টেন্সর’) \n", 126 | "\n", 127 | "সংখ্যার অ্যারেকে আমরা ভেক্টর বা ১ ডাইমেনশন টেন্সর বলি। এখানে আমরা চারটা সংখ্যার অ্যারেকে নিয়ে একটা এক ডাইমেনশনের ‘টেন্সর’ তৈরি করেছি।" 128 | ] 129 | }, 130 | { 131 | "cell_type": "code", 132 | "metadata": { 133 | "id": "R8VLwZNoyEGi", 134 | "colab_type": "code", 135 | "outputId": "2dfff752-f4af-421d-e353-c99fa77f16fa", 136 | "colab": { 137 | "base_uri": "https://localhost:8080/", 138 | "height": 35 139 | } 140 | }, 141 | "source": [ 142 | "খ = np.array([12, 3, 6, 14])\n", 143 | "খ" 144 | ], 145 | "execution_count": 3, 146 | "outputs": [ 147 | { 148 | "output_type": "execute_result", 149 | "data": { 150 | "text/plain": [ 151 | "array([12, 3, 6, 14])" 152 | ] 153 | }, 154 | "metadata": { 155 | "tags": [] 156 | }, 157 | "execution_count": 3 158 | } 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "metadata": { 164 | "id": "vV8d5Xc7yLVc", 165 | "colab_type": "code", 166 | "outputId": "15f1272c-59c6-4aea-a88c-500f66276225", 167 | "colab": { 168 | "base_uri": "https://localhost:8080/", 169 | "height": 35 170 | } 171 | }, 172 | "source": [ 173 | "খ.ndim" 174 | ], 175 | "execution_count": 4, 176 | "outputs": [ 177 | { 178 | "output_type": "execute_result", 179 | "data": { 180 | "text/plain": [ 181 | "1" 182 | ] 183 | }, 184 | "metadata": { 185 | "tags": [] 186 | }, 187 | "execution_count": 4 188 | } 189 | ] 190 | }, 191 | { 192 | "cell_type": "markdown", 193 | "metadata": { 194 | "id": "4Z7IWRudyWMd", 195 | "colab_type": "text" 196 | }, 197 | "source": [ 198 | "### ৩. ম্যাট্রিক্স, দুই ডাইমেনশন এর ‘টেন্সর’\n", 199 | "\n", 200 | "আগে আমরা যেই ভেক্টর নিয়ে আলাপ করেছিলাম সেই ভেক্টরের অ্যারেকে আমরা ম্যাট্রিক্স বলতে পারি। এটা দুই ডাইমেনশন এর ‘টেন্সর’। একটা ম্যাট্রিক্সে দুটো এক্সিস থাকে যাকে আমরা ‘সারি’ এবং ‘কলাম’ দিয়ে ডিফাইন করতে পারি। নিচের উদাহরণটা দেখুন, এখানে প্রথম এক্সিস হচ্ছে ‘সারি’ আর পরের এক্সিসটা হচ্ছে ‘কলাম’।" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "metadata": { 206 | "id": "tZzug4icq5AF", 207 | "colab_type": "code", 208 | "outputId": "5fa7bfb9-7799-41ae-eab1-099698a55cba", 209 | "colab": { 210 | "base_uri": "https://localhost:8080/", 211 | "height": 69 212 | } 213 | }, 214 | "source": [ 215 | "গ = np.array([[5, 78, 2, 34, 0],\n", 216 | " [6, 79, 3, 35, 1],\n", 217 | " [7, 80, 4, 36, 2]])\n", 218 | "গ" 219 | ], 220 | "execution_count": 5, 221 | "outputs": [ 222 | { 223 | "output_type": "execute_result", 224 | "data": { 225 | "text/plain": [ 226 | "array([[ 5, 78, 2, 34, 0],\n", 227 | " [ 6, 79, 3, 35, 1],\n", 228 | " [ 7, 80, 4, 36, 2]])" 229 | ] 230 | }, 231 | "metadata": { 232 | "tags": [] 233 | }, 234 | "execution_count": 5 235 | } 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "metadata": { 241 | "id": "SkJZks0frJAa", 242 | "colab_type": "code", 243 | "outputId": "32729989-f58c-4b38-cb67-37f8642469b3", 244 | "colab": { 245 | "base_uri": "https://localhost:8080/", 246 | "height": 35 247 | } 248 | }, 249 | "source": [ 250 | "গ.ndim" 251 | ], 252 | "execution_count": 6, 253 | "outputs": [ 254 | { 255 | "output_type": "execute_result", 256 | "data": { 257 | "text/plain": [ 258 | "2" 259 | ] 260 | }, 261 | "metadata": { 262 | "tags": [] 263 | }, 264 | "execution_count": 6 265 | } 266 | ] 267 | }, 268 | { 269 | "cell_type": "markdown", 270 | "metadata": { 271 | "id": "8WSyYBUjwEvD", 272 | "colab_type": "text" 273 | }, 274 | "source": [ 275 | "## পাশাপাশি একটু সরাসরি টেন্সরফ্লোতে দেখি \n", 276 | "\n", 277 | "টেন্সরফ্লোতে সবচেয়ে ছোট অপারেশন শুরু করা যায় কয়েকটা কনস্ট্যান্ট এবং ভ্যারিয়েবল তৈরি করে। সেটা আমরা করবো tf.constant এবং tf.Variable ফাংশন কল করে যাতে সেটা অ্যারে বানাতে পারে।" 278 | ] 279 | }, 280 | { 281 | "cell_type": "code", 282 | "metadata": { 283 | "id": "pp_Wp_ptBwNn", 284 | "colab_type": "code", 285 | "outputId": "8ed2dfbd-d1bc-482d-af14-b93a72d49eaa", 286 | "colab": { 287 | "base_uri": "https://localhost:8080/", 288 | "height": 52 289 | } 290 | }, 291 | "source": [ 292 | "# টেন্সরফ্লো ২.x সিলেক্ট করি\n", 293 | "%tensorflow_version 2.x\n", 294 | "import tensorflow as tf\n", 295 | "# একটা কনস্ট্যান্ট টেন্সর বানাই, যেটা পাল্টাবে না\n", 296 | "ঘ = tf.constant([[3, 2],\n", 297 | " [5, 2]])\n", 298 | "\n", 299 | "# ভ্যারিয়েবল টেন্সর, যেটা পাল্টাবে, মানে .Variable\n", 300 | "ঙ = tf.Variable([[3, 2],\n", 301 | " [5, 2],\n", 302 | " [5, 2]])\n", 303 | "# tf.shape(ঘ)\n", 304 | "# আমরা শেপ দেখি\n", 305 | "tf.shape(ঙ)" 306 | ], 307 | "execution_count": 7, 308 | "outputs": [ 309 | { 310 | "output_type": "stream", 311 | "text": [ 312 | "TensorFlow 2.x selected.\n" 313 | ], 314 | "name": "stdout" 315 | }, 316 | { 317 | "output_type": "execute_result", 318 | "data": { 319 | "text/plain": [ 320 | "" 321 | ] 322 | }, 323 | "metadata": { 324 | "tags": [] 325 | }, 326 | "execution_count": 7 327 | } 328 | ] 329 | }, 330 | { 331 | "cell_type": "code", 332 | "metadata": { 333 | "id": "il_dIA2MFBK3", 334 | "colab_type": "code", 335 | "outputId": "623e0f08-6309-4662-a942-248b4b555092", 336 | "colab": { 337 | "base_uri": "https://localhost:8080/", 338 | "height": 52 339 | } 340 | }, 341 | "source": [ 342 | "# আরেকটা উদাহরণ\n", 343 | "চ = tf.constant([[1.0, 2.0, 3.0], \n", 344 | " [4.0, 5.0, 6.0]])\n", 345 | "print(চ.get_shape())\n", 346 | "print(চ.shape.dims)" 347 | ], 348 | "execution_count": 8, 349 | "outputs": [ 350 | { 351 | "output_type": "stream", 352 | "text": [ 353 | "(2, 3)\n", 354 | "[Dimension(2), Dimension(3)]\n" 355 | ], 356 | "name": "stdout" 357 | } 358 | ] 359 | }, 360 | { 361 | "cell_type": "markdown", 362 | "metadata": { 363 | "id": "Mky3clnidodE", 364 | "colab_type": "text" 365 | }, 366 | "source": [ 367 | "## টেন্সরফ্লো ২.x এ কিভাবে টেন্সর অবজেক্টকে নামপাই অ্যারেতে কনভার্ট করা যায়?\n", 368 | "\n", 369 | "যেহেতু ইগার একজিকিউশন যেহেতু ডিফল্ট হিসেবে চালু থাকে, সেকারণে .numpy()কে টেন্সর অবজেক্টে কল করলেই সেটা পাওয়া যাবে। তারা একই মেমরি শেয়ার করে। একটা পাল্টালে আরেকটা পাল্টাবে।" 370 | ] 371 | }, 372 | { 373 | "cell_type": "code", 374 | "metadata": { 375 | "id": "V2Q2rPqFfHio", 376 | "colab_type": "code", 377 | "outputId": "98a5b960-a54f-42f2-faa6-2fdb6058094a", 378 | "colab": { 379 | "base_uri": "https://localhost:8080/", 380 | "height": 52 381 | } 382 | }, 383 | "source": [ 384 | "ল = tf.constant([[1, 2], [3, 4]]) \n", 385 | "র = tf.add(ল, 1)\n", 386 | "\n", 387 | "ল.numpy()" 388 | ], 389 | "execution_count": 9, 390 | "outputs": [ 391 | { 392 | "output_type": "execute_result", 393 | "data": { 394 | "text/plain": [ 395 | "array([[1, 2],\n", 396 | " [3, 4]], dtype=int32)" 397 | ] 398 | }, 399 | "metadata": { 400 | "tags": [] 401 | }, 402 | "execution_count": 9 403 | } 404 | ] 405 | }, 406 | { 407 | "cell_type": "code", 408 | "metadata": { 409 | "id": "rZDpizQHfMjc", 410 | "colab_type": "code", 411 | "outputId": "5148b847-559f-4667-b940-8a282d639466", 412 | "colab": { 413 | "base_uri": "https://localhost:8080/", 414 | "height": 52 415 | } 416 | }, 417 | "source": [ 418 | "র.numpy()" 419 | ], 420 | "execution_count": 10, 421 | "outputs": [ 422 | { 423 | "output_type": "execute_result", 424 | "data": { 425 | "text/plain": [ 426 | "array([[2, 3],\n", 427 | " [4, 5]], dtype=int32)" 428 | ] 429 | }, 430 | "metadata": { 431 | "tags": [] 432 | }, 433 | "execution_count": 10 434 | } 435 | ] 436 | }, 437 | { 438 | "cell_type": "code", 439 | "metadata": { 440 | "id": "FDVke7glfQ0z", 441 | "colab_type": "code", 442 | "outputId": "11c258aa-1374-4098-ea0b-20a0cf990596", 443 | "colab": { 444 | "base_uri": "https://localhost:8080/", 445 | "height": 52 446 | } 447 | }, 448 | "source": [ 449 | "tf.multiply(ল, র).numpy()\n" 450 | ], 451 | "execution_count": 11, 452 | "outputs": [ 453 | { 454 | "output_type": "execute_result", 455 | "data": { 456 | "text/plain": [ 457 | "array([[ 2, 6],\n", 458 | " [12, 20]], dtype=int32)" 459 | ] 460 | }, 461 | "metadata": { 462 | "tags": [] 463 | }, 464 | "execution_count": 11 465 | } 466 | ] 467 | }, 468 | { 469 | "cell_type": "markdown", 470 | "metadata": { 471 | "id": "mLN2MVpayme4", 472 | "colab_type": "text" 473 | }, 474 | "source": [ 475 | "### ৪. ৩ ডাইমেনশন এবং তার বেশি ডাইমেনশনের ‘টেন্সর’\n", 476 | "\n", 477 | "আমরা যখন এরকম কয়েকটা ম্যাট্রিক্সকে আগের মতো একটা অ্যারেতে ফেলতে চাই তখন সেটা তিন ডাইমেনশনের টেন্সর হয়ে যায়। এ ধরনের ছবিগুলো আমরা দেখেছিলাম “শূন্য থেকে পাইথন মেশিন লার্নিং” বইটাতে। এর এক্সিস তিনটা। এভাবে আমরা চার ডাইমেনশনের টেন্সর তৈরি করতে পারি। বিশেষ করে আমাদের ‘কম্পিউটার ভিশন’ রিলেটেড সমস্যাগুলো যা ছবি নিয়ে কাজ করে - সেগুলো চার ডাইমেনশন এর হয়ে থাকে। ডিপ লার্নিং এ আমরা সাধারণত: শূন্য থেকে চার ডাইমেনশন নিয়ে কথা বলব। যখন সেটা ভিডিও হবে তখন সেটা পাঁচ ডাইমেনশনে চলে যাবে।\n" 478 | ] 479 | }, 480 | { 481 | "cell_type": "code", 482 | "metadata": { 483 | "id": "dI2AdtrbrRWm", 484 | "colab_type": "code", 485 | "colab": {} 486 | }, 487 | "source": [ 488 | "ছ = np.array([[[5, 78, 2, 34, 0],\n", 489 | " [6, 79, 3, 35, 1],\n", 490 | " [7, 80, 4, 36, 2]],\n", 491 | " \n", 492 | " [[5, 78, 2, 34, 0],\n", 493 | " [6, 79, 3, 35, 1],\n", 494 | " [7, 80, 4, 36, 2]],\n", 495 | " \n", 496 | " [[5, 78, 2, 34, 0],\n", 497 | " [6, 79, 3, 35, 1],\n", 498 | " [7, 80, 4, 36, 2]]])" 499 | ], 500 | "execution_count": 0, 501 | "outputs": [] 502 | }, 503 | { 504 | "cell_type": "code", 505 | "metadata": { 506 | "id": "9ztmJYzVrlIb", 507 | "colab_type": "code", 508 | "outputId": "ad4dd5b7-43c6-4785-ff16-4f4584a6375f", 509 | "colab": { 510 | "base_uri": "https://localhost:8080/", 511 | "height": 208 512 | } 513 | }, 514 | "source": [ 515 | "ছ" 516 | ], 517 | "execution_count": 13, 518 | "outputs": [ 519 | { 520 | "output_type": "execute_result", 521 | "data": { 522 | "text/plain": [ 523 | "array([[[ 5, 78, 2, 34, 0],\n", 524 | " [ 6, 79, 3, 35, 1],\n", 525 | " [ 7, 80, 4, 36, 2]],\n", 526 | "\n", 527 | " [[ 5, 78, 2, 34, 0],\n", 528 | " [ 6, 79, 3, 35, 1],\n", 529 | " [ 7, 80, 4, 36, 2]],\n", 530 | "\n", 531 | " [[ 5, 78, 2, 34, 0],\n", 532 | " [ 6, 79, 3, 35, 1],\n", 533 | " [ 7, 80, 4, 36, 2]]])" 534 | ] 535 | }, 536 | "metadata": { 537 | "tags": [] 538 | }, 539 | "execution_count": 13 540 | } 541 | ] 542 | }, 543 | { 544 | "cell_type": "code", 545 | "metadata": { 546 | "id": "XNkWIJqIrwa7", 547 | "colab_type": "code", 548 | "outputId": "95f145bd-2a4a-48c3-e4e3-365c5f47d657", 549 | "colab": { 550 | "base_uri": "https://localhost:8080/", 551 | "height": 35 552 | } 553 | }, 554 | "source": [ 555 | "ছ.ndim" 556 | ], 557 | "execution_count": 14, 558 | "outputs": [ 559 | { 560 | "output_type": "execute_result", 561 | "data": { 562 | "text/plain": [ 563 | "3" 564 | ] 565 | }, 566 | "metadata": { 567 | "tags": [] 568 | }, 569 | "execution_count": 14 570 | } 571 | ] 572 | }, 573 | { 574 | "cell_type": "markdown", 575 | "metadata": { 576 | "id": "E1nxd_MZywsk", 577 | "colab_type": "text" 578 | }, 579 | "source": [ 580 | "### শেপ (shape)\n", 581 | "\n", 582 | "এটা ইন্টিজার দিয়ে তৈরি করা একটা “টুপল” (tuple) যা বলে দেয় একটা টেন্সরের প্রতিটা এক্সিসে কত ডাইমেনশন আছে। আগের উদাহরণগুলোতে ভেক্টরে শেপ ছিল (৫, ), ম্যাট্রিক্স উদাহরণে শেপ (৩, ৫) পাশাপাশি তিন ডাইমেনশনের এর টেন্সরে শেপ ছিল (৩,৩,৫) আর স্কেলার (), মানে খালি শেপ। \n", 583 | "\n", 584 | "(১) ভেক্টর ডাটা - (samples, features)\n", 585 | "\n", 586 | "(২) টাইমসিরিজ/ সিকোয়েন্স ডাটা ৩D - (samples, timesteps, features)\n", 587 | "\n", 588 | "(৩) ইমেজ ডাটা, যেমন এমনিস্ট ৪D - (samples, height, width, channels) অথবা (samples, channels, height, width) \n", 589 | "\n", 590 | "(৪) ভিডিও ডাটা ৫D - (samples, frames, height, width, channels) অথবা (samples, frames, channels, height, width)\n", 591 | "\n", 592 | "### ইমেজ ডাটা শেপ\n", 593 | "\n", 594 | "সাধারণতঃ ইমেজে তিনটা ডাইমেনশন হয়, দৈর্ঘ্য, প্রস্থ এবং কালার চ্যানেল। আমাদের এমনিস্ট ডাটাসেট নিয়ে কাজ করতে গেলে একটা মজার জিনিস হয়। যেহেতু এটা গ্রে-স্কেল এর মানে হচ্ছে কালার চ্যানেল একটা দিয়েই হয়ে যায়। এর মানে ২D টেন্সরে হয়ে যাবার কথা। তবে কনভেনশন অনুযায়ী ইমেজ যেহেতু ৩D, সেখানে ১ ডাইমেনশন কালার চ্যানেল ব্যবহার হবে গ্রে-স্কেল ইমেজের জন্য। কালারের জন্য ৩। \n", 595 | "\n", 596 | "ধরুন, একটা ব্যাচে ১২৮টা ২৮ × ২৮ পিক্সেলের গ্রে-স্কেল ইমেজ স্টোর করা যাবে যেই টেন্সরে তার শেপ (১২৮, ২৮, ২৮, ১) অথবা কালারের জন্য শেপ গিয়ে দাড়াবে (১২৮, ২৮, ২৮, ৩)তে।\n" 597 | ] 598 | }, 599 | { 600 | "cell_type": "markdown", 601 | "metadata": { 602 | "id": "sRxp7sChGrzp", 603 | "colab_type": "text" 604 | }, 605 | "source": [ 606 | " চিত্রঃ ৪D ইমেজ ডাটা টেন্সর এর উদাহরণ" 607 | ] 608 | }, 609 | { 610 | "cell_type": "code", 611 | "metadata": { 612 | "id": "9mJ3lq5BGsoc", 613 | "colab_type": "code", 614 | "outputId": "60829873-ecee-495d-dbd0-e3956b5b51f6", 615 | "colab": { 616 | "base_uri": "https://localhost:8080/", 617 | "height": 156 618 | } 619 | }, 620 | "source": [ 621 | "# টেন্সর শেপকে পাল্টে অন্য শেপে আনা, লাগবে এমনিস্ট ডাটাসেটে\n", 622 | "টেন্সর = tf.constant([[3, 2],\n", 623 | " [5, 2],\n", 624 | " [9, 5],\n", 625 | " [1, 3]])\n", 626 | "\n", 627 | "# টেন্সরকে সারি, কলাম ধরে পাল্টাই: shape = [rows, columns]\n", 628 | "শেপ_টেন্সর = tf.reshape(tensor = টেন্সর,\n", 629 | " shape = [1, 8])\n", 630 | "\n", 631 | "print(('টেন্সর রিশেপিং এর আগে:\\n{0}').format(\n", 632 | " টেন্সর.numpy()\n", 633 | "))\n", 634 | "print(('\\nটেন্সর রিশেপিং এর পরে:\\n{0}').format(\n", 635 | " শেপ_টেন্সর.numpy()\n", 636 | "))" 637 | ], 638 | "execution_count": 15, 639 | "outputs": [ 640 | { 641 | "output_type": "stream", 642 | "text": [ 643 | "টেন্সর রিশেপিং এর আগে:\n", 644 | "[[3 2]\n", 645 | " [5 2]\n", 646 | " [9 5]\n", 647 | " [1 3]]\n", 648 | "\n", 649 | "টেন্সর রিশেপিং এর পরে:\n", 650 | "[[3 2 5 2 9 5 1 3]]\n" 651 | ], 652 | "name": "stdout" 653 | } 654 | ] 655 | }, 656 | { 657 | "cell_type": "markdown", 658 | "metadata": { 659 | "id": "pIBMqXdrHeEr", 660 | "colab_type": "text" 661 | }, 662 | "source": [ 663 | "## টেন্সরের ম্যাট্রিক্স মাল্টিপ্লিকেশন\n", 664 | "\n", 665 | "মেশিন/ডিপ লার্নিং এর হৃদয়ে আছে টেন্সরগুলোর ম্যাট্রিক্স মাল্টিপ্লিকেশন, সেটার একটা উদাহরণ দেখাচ্ছি দুটো ম্যাট্রিক্স (এর মধ্যে একটা ভেক্টর) এবং tf.matmul ফাংশন ব্যবহার করে।" 666 | ] 667 | }, 668 | { 669 | "cell_type": "code", 670 | "metadata": { 671 | "id": "SUbh5-Y4H5ei", 672 | "colab_type": "code", 673 | "outputId": "074af5b4-ffa5-4a4b-f881-0c05d4dc4efa", 674 | "colab": { 675 | "base_uri": "https://localhost:8080/", 676 | "height": 69 677 | } 678 | }, 679 | "source": [ 680 | "# উদাহরণের জন্য ম্যাট্রিক্স \"ঝ\"\n", 681 | "ঝ = tf.constant([[3, 7],\n", 682 | " [1, 9]])\n", 683 | "\n", 684 | "# উদাহরণের জন্য ভেক্টর \"ঞ\" \n", 685 | "ঞ = tf.constant([[5],\n", 686 | " [2]])\n", 687 | "\n", 688 | "# ম্যাট্রিক্স মাল্টিপ্লিকেশন \"ঝ\" এবং \"ঞ\"\n", 689 | "ঝঞ = tf.matmul(ঝ, ঞ)\n", 690 | "\n", 691 | "print(('\"ঝ\" এবং \"ঞ\" এর ম্যাট্রিক্স মাল্টিপ্লিকেশন দিয়ে নতুন টেন্সর:\\n{0}').format(\n", 692 | " ঝঞ\n", 693 | "))" 694 | ], 695 | "execution_count": 16, 696 | "outputs": [ 697 | { 698 | "output_type": "stream", 699 | "text": [ 700 | "\"ঝ\" এবং \"ঞ\" এর ম্যাট্রিক্স মাল্টিপ্লিকেশন দিয়ে নতুন টেন্সর:\n", 701 | "[[29]\n", 702 | " [23]]\n" 703 | ], 704 | "name": "stdout" 705 | } 706 | ] 707 | }, 708 | { 709 | "cell_type": "markdown", 710 | "metadata": { 711 | "id": "s4PI7G7RN4ar", 712 | "colab_type": "text" 713 | }, 714 | "source": [ 715 | "## আমাদের ফ্যাশন এমনিস্ট ডাটাসেটের ডাইমেনশন, অ্যারের শেপ \n", 716 | "\n", 717 | "এমনিস্ট ডাটাসেটে ট্রেনিং_ইমেজ, ট্রেনিং_লেবেল, টেস্ট_ইমেজ, টেস্ট_লেবেল যে চারটা অংশ আছে সেটার শেপ এবং এক্সিস দেখি এখন। শুরুতে একটা দেখলেই হবে। " 718 | ] 719 | }, 720 | { 721 | "cell_type": "code", 722 | "metadata": { 723 | "id": "yH6NekvOsAY2", 724 | "colab_type": "code", 725 | "colab": { 726 | "base_uri": "https://localhost:8080/", 727 | "height": 176 728 | }, 729 | "outputId": "397251fc-4ab4-4b05-da24-a190b5289fbb" 730 | }, 731 | "source": [ 732 | "import tensorflow as tf\n", 733 | "from tensorflow import keras\n", 734 | "\n", 735 | "ফ্যাশন_এমনিস্ট = tf.keras.datasets.fashion_mnist\n", 736 | "(ট্রেনিং_ইমেজ, ট্রেনিং_লেবেল), (টেস্ট_ইমেজ, টেস্ট_লেবেল) = ফ্যাশন_এমনিস্ট.load_data()" 737 | ], 738 | "execution_count": 17, 739 | "outputs": [ 740 | { 741 | "output_type": "stream", 742 | "text": [ 743 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n", 744 | "32768/29515 [=================================] - 0s 0us/step\n", 745 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n", 746 | "26427392/26421880 [==============================] - 0s 0us/step\n", 747 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n", 748 | "8192/5148 [===============================================] - 0s 0us/step\n", 749 | "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n", 750 | "4423680/4422102 [==============================] - 0s 0us/step\n" 751 | ], 752 | "name": "stdout" 753 | } 754 | ] 755 | }, 756 | { 757 | "cell_type": "code", 758 | "metadata": { 759 | "id": "OjXjO5t7udXj", 760 | "colab_type": "code", 761 | "outputId": "36630e01-4ba7-4be8-8b70-2a4f20556a3c", 762 | "colab": { 763 | "base_uri": "https://localhost:8080/", 764 | "height": 35 765 | } 766 | }, 767 | "source": [ 768 | "print(ট্রেনিং_ইমেজ.ndim)" 769 | ], 770 | "execution_count": 18, 771 | "outputs": [ 772 | { 773 | "output_type": "stream", 774 | "text": [ 775 | "3\n" 776 | ], 777 | "name": "stdout" 778 | } 779 | ] 780 | }, 781 | { 782 | "cell_type": "markdown", 783 | "metadata": { 784 | "id": "oZhWY_pBhNmw", 785 | "colab_type": "text" 786 | }, 787 | "source": [ 788 | "তিনটা ডাইমেনশন: (ইমেজ সংখ্যা, দৈর্ঘ্য় মানে পিক্সেল সংখ্যা, প্রস্থ মানে পিক্সেল সংখ্যা)" 789 | ] 790 | }, 791 | { 792 | "cell_type": "code", 793 | "metadata": { 794 | "id": "CO5-XC64UBBX", 795 | "colab_type": "code", 796 | "outputId": "68864e9b-80c1-4e13-8b61-19a1c30899dc", 797 | "colab": { 798 | "base_uri": "https://localhost:8080/", 799 | "height": 35 800 | } 801 | }, 802 | "source": [ 803 | "print(ট্রেনিং_ইমেজ.shape)" 804 | ], 805 | "execution_count": 19, 806 | "outputs": [ 807 | { 808 | "output_type": "stream", 809 | "text": [ 810 | "(60000, 28, 28)\n" 811 | ], 812 | "name": "stdout" 813 | } 814 | ] 815 | }, 816 | { 817 | "cell_type": "code", 818 | "metadata": { 819 | "id": "k9YB3pul8L6q", 820 | "colab_type": "code", 821 | "colab": { 822 | "base_uri": "https://localhost:8080/", 823 | "height": 35 824 | }, 825 | "outputId": "aeaf1e4d-f159-4b6a-8d18-259e38b99d6c" 826 | }, 827 | "source": [ 828 | "print(টেস্ট_ইমেজ.shape)" 829 | ], 830 | "execution_count": 20, 831 | "outputs": [ 832 | { 833 | "output_type": "stream", 834 | "text": [ 835 | "(10000, 28, 28)\n" 836 | ], 837 | "name": "stdout" 838 | } 839 | ] 840 | }, 841 | { 842 | "cell_type": "markdown", 843 | "metadata": { 844 | "id": "QrxSbJtNOnf8", 845 | "colab_type": "text" 846 | }, 847 | "source": [ 848 | "শুরু এবং শেষ ইনডেক্স দিয়ে স্লাইসের ভেতরে আগের মতো দেখা। ১০ থেকে ১০০তম, তবে ১০০তম ইমেজ বাদে। আমাদের শেপের অ্যারে কি হতে পারে?\n" 849 | ] 850 | }, 851 | { 852 | "cell_type": "code", 853 | "metadata": { 854 | "id": "4LSleoeAUY1w", 855 | "colab_type": "code", 856 | "outputId": "2d4ea6dc-3ea2-43a7-9e48-fa0a620498a0", 857 | "colab": { 858 | "base_uri": "https://localhost:8080/", 859 | "height": 35 860 | } 861 | }, 862 | "source": [ 863 | "পরিক্ষা_ইমেজ১ = ট্রেনিং_ইমেজ[10:100]\n", 864 | "print(পরিক্ষা_ইমেজ১.shape)\n" 865 | ], 866 | "execution_count": 21, 867 | "outputs": [ 868 | { 869 | "output_type": "stream", 870 | "text": [ 871 | "(90, 28, 28)\n" 872 | ], 873 | "name": "stdout" 874 | } 875 | ] 876 | }, 877 | { 878 | "cell_type": "code", 879 | "metadata": { 880 | "id": "4gs_XIAcW68M", 881 | "colab_type": "code", 882 | "outputId": "59e45a5a-9ba6-4678-db1c-1f168507afda", 883 | "colab": { 884 | "base_uri": "https://localhost:8080/", 885 | "height": 35 886 | } 887 | }, 888 | "source": [ 889 | "পরিক্ষা_ইমেজ২ = ট্রেনিং_ইমেজ[10:100, :, :]\n", 890 | "পরিক্ষা_ইমেজ২.shape" 891 | ], 892 | "execution_count": 22, 893 | "outputs": [ 894 | { 895 | "output_type": "execute_result", 896 | "data": { 897 | "text/plain": [ 898 | "(90, 28, 28)" 899 | ] 900 | }, 901 | "metadata": { 902 | "tags": [] 903 | }, 904 | "execution_count": 22 905 | } 906 | ] 907 | }, 908 | { 909 | "cell_type": "code", 910 | "metadata": { 911 | "id": "qoGs_e_YXbXC", 912 | "colab_type": "code", 913 | "outputId": "f34329bd-d864-42c8-b0e4-900e65e33f0f", 914 | "colab": { 915 | "base_uri": "https://localhost:8080/", 916 | "height": 35 917 | } 918 | }, 919 | "source": [ 920 | "পরিক্ষা_ইমেজ৩ = ট্রেনিং_ইমেজ[10:100, 0:28, 0:28]\n", 921 | "পরিক্ষা_ইমেজ৩.shape" 922 | ], 923 | "execution_count": 23, 924 | "outputs": [ 925 | { 926 | "output_type": "execute_result", 927 | "data": { 928 | "text/plain": [ 929 | "(90, 28, 28)" 930 | ] 931 | }, 932 | "metadata": { 933 | "tags": [] 934 | }, 935 | "execution_count": 23 936 | } 937 | ] 938 | }, 939 | { 940 | "cell_type": "markdown", 941 | "metadata": { 942 | "id": "Qqej7aPdELvR", 943 | "colab_type": "text" 944 | }, 945 | "source": [ 946 | "### ডাটা ব্যাচ\n", 947 | "\n", 948 | "আমরা যখন ডিপ লার্নিংয়ে ডাটা টেন্সরের প্রথম এক্সিস (যাকে বলছি এক্সিস ০, কারণ ইনডেক্সিং শুরু হয় ০ থেকে) নিয়ে কাজ করি সেটাকে samples এক্সিস অথবা samples ডাইমেনশন বলি। আমাদের ফ্যাশন এমনিস্ট ডাটাসেটে এই samples হচ্ছে ড্রেস/জুতার ছবিগুলো।\n", 949 | "\n", 950 | "তবে পাইপলাইন অপটিমাইজেশনের কারণে ডিপ লার্নিং পুরো ডাটাসেট নিয়ে কাজ না করে সেটাকে ছোট ছোট ব্যাচে ভাগ করে ট্রেনিং করে।\n", 951 | "\n", 952 | "আমাদের এমনিস্ট উদাহরণে যদি একটা ব্যাচ সাইজ ১২৮টা করে ইমেজ হয়, তাহলে এভাবে হতে পারে:" 953 | ] 954 | }, 955 | { 956 | "cell_type": "code", 957 | "metadata": { 958 | "id": "o7-7PqigTCNb", 959 | "colab_type": "code", 960 | "colab": {} 961 | }, 962 | "source": [ 963 | "ব্যাচ = ট্রেনিং_ইমেজ[:128]" 964 | ], 965 | "execution_count": 0, 966 | "outputs": [] 967 | }, 968 | { 969 | "cell_type": "markdown", 970 | "metadata": { 971 | "id": "bfs6egrjTPWq", 972 | "colab_type": "text" 973 | }, 974 | "source": [ 975 | "পরের ব্যাচ" 976 | ] 977 | }, 978 | { 979 | "cell_type": "code", 980 | "metadata": { 981 | "id": "SECs9ASiTSbn", 982 | "colab_type": "code", 983 | "colab": {} 984 | }, 985 | "source": [ 986 | "ব্যাচ = ট্রেনিং_ইমেজ[128:256]" 987 | ], 988 | "execution_count": 0, 989 | "outputs": [] 990 | }, 991 | { 992 | "cell_type": "code", 993 | "metadata": { 994 | "id": "-77fvK7PT_OS", 995 | "colab_type": "code", 996 | "outputId": "a786d53d-3e2d-4f42-8b24-c7ac9bca7fbd", 997 | "colab": { 998 | "base_uri": "https://localhost:8080/", 999 | "height": 867 1000 | } 1001 | }, 1002 | "source": [ 1003 | "ব্যাচ" 1004 | ], 1005 | "execution_count": 26, 1006 | "outputs": [ 1007 | { 1008 | "output_type": "execute_result", 1009 | "data": { 1010 | "text/plain": [ 1011 | "array([[[0, 0, 0, ..., 0, 0, 0],\n", 1012 | " [0, 0, 0, ..., 0, 0, 0],\n", 1013 | " [0, 0, 0, ..., 0, 0, 0],\n", 1014 | " ...,\n", 1015 | " [0, 0, 0, ..., 0, 0, 0],\n", 1016 | " [0, 0, 0, ..., 0, 0, 0],\n", 1017 | " [0, 0, 0, ..., 0, 0, 0]],\n", 1018 | "\n", 1019 | " [[0, 0, 0, ..., 0, 0, 0],\n", 1020 | " [0, 0, 0, ..., 0, 0, 0],\n", 1021 | " [0, 0, 0, ..., 0, 0, 0],\n", 1022 | " ...,\n", 1023 | " [0, 0, 0, ..., 0, 0, 0],\n", 1024 | " [0, 0, 0, ..., 0, 0, 0],\n", 1025 | " [0, 0, 0, ..., 0, 0, 0]],\n", 1026 | "\n", 1027 | " [[0, 0, 0, ..., 0, 0, 0],\n", 1028 | " [0, 0, 0, ..., 0, 0, 0],\n", 1029 | " [0, 0, 0, ..., 0, 0, 0],\n", 1030 | " ...,\n", 1031 | " [0, 0, 0, ..., 0, 0, 0],\n", 1032 | " [0, 0, 0, ..., 0, 0, 0],\n", 1033 | " [0, 0, 0, ..., 0, 0, 0]],\n", 1034 | "\n", 1035 | " ...,\n", 1036 | "\n", 1037 | " [[0, 0, 0, ..., 0, 0, 0],\n", 1038 | " [0, 0, 0, ..., 0, 0, 0],\n", 1039 | " [0, 0, 0, ..., 0, 0, 0],\n", 1040 | " ...,\n", 1041 | " [0, 0, 0, ..., 0, 0, 0],\n", 1042 | " [0, 0, 0, ..., 0, 0, 0],\n", 1043 | " [0, 0, 0, ..., 0, 0, 0]],\n", 1044 | "\n", 1045 | " [[0, 0, 0, ..., 0, 0, 0],\n", 1046 | " [0, 0, 0, ..., 0, 0, 0],\n", 1047 | " [0, 0, 0, ..., 0, 0, 0],\n", 1048 | " ...,\n", 1049 | " [0, 0, 0, ..., 0, 0, 0],\n", 1050 | " [0, 0, 0, ..., 0, 0, 0],\n", 1051 | " [0, 0, 0, ..., 0, 0, 0]],\n", 1052 | "\n", 1053 | " [[0, 0, 0, ..., 0, 0, 0],\n", 1054 | " [0, 0, 0, ..., 0, 0, 0],\n", 1055 | " [0, 0, 0, ..., 0, 0, 0],\n", 1056 | " ...,\n", 1057 | " [0, 0, 0, ..., 0, 0, 0],\n", 1058 | " [0, 0, 0, ..., 0, 0, 0],\n", 1059 | " [0, 0, 0, ..., 0, 0, 0]]], dtype=uint8)" 1060 | ] 1061 | }, 1062 | "metadata": { 1063 | "tags": [] 1064 | }, 1065 | "execution_count": 26 1066 | } 1067 | ] 1068 | }, 1069 | { 1070 | "cell_type": "markdown", 1071 | "metadata": { 1072 | "id": "iAZwumqkTckt", 1073 | "colab_type": "text" 1074 | }, 1075 | "source": [ 1076 | "nতম ব্যাচ হলে;\n", 1077 | "\n", 1078 | "ব্যাচ = ট্রেনিং_ইমেজ[128 * n:128 * (n + 1)]\n", 1079 | "\n", 1080 | "আমরা যখন একটা ব্যাচ টেন্সর নিয়ে কাজ করবো, সেখানে প্রথম এক্সিস (এক্সিস ০)কে বলি \"ব্যাচ এক্সিস\" বা \"ব্যাচ ডাইমেনশন\"।" 1081 | ] 1082 | } 1083 | ] 1084 | } -------------------------------------------------------------------------------- /tensorflow_2_0.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "tensorflow-2.0.ipynb", 7 | "version": "0.3.2", 8 | "provenance": [], 9 | "collapsed_sections": [], 10 | "include_colab_link": true 11 | }, 12 | "kernelspec": { 13 | "name": "python3", 14 | "display_name": "Python 3" 15 | }, 16 | "accelerator": "GPU" 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "markdown", 21 | "metadata": { 22 | "id": "view-in-github", 23 | "colab_type": "text" 24 | }, 25 | "source": [ 26 | "\"Open" 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": { 32 | "colab_type": "text", 33 | "id": "2QQJJyDzqGRb" 34 | }, 35 | "source": [ 36 | "# টেন্সর-ফ্লো ২.০ ইনস্টলেশন \n", 37 | "\n", 38 | "কিছু জিনিস ঠিকমতো কাজ না করতে পারে, টেস্ট করতে থাকুন \n" 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": { 44 | "colab_type": "text", 45 | "id": "RBAeIwOMrYk8" 46 | }, 47 | "source": [ 48 | "## এনভায়রনমেন্ট ঠিক করে নেই " 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "metadata": { 54 | "attributes": { 55 | "classes": [ 56 | "py" 57 | ], 58 | "id": "" 59 | }, 60 | "colab_type": "code", 61 | "id": "7aFsD8csqEff", 62 | "colab": {} 63 | }, 64 | "source": [ 65 | "from __future__ import absolute_import, division, print_function, unicode_literals\n", 66 | "\n", 67 | "import tensorflow as tf\n", 68 | "\n", 69 | "import numpy as np\n", 70 | "\n", 71 | "# tf.enable_eager_execution()" 72 | ], 73 | "execution_count": 0, 74 | "outputs": [] 75 | }, 76 | { 77 | "cell_type": "markdown", 78 | "metadata": { 79 | "id": "KIAL4s0UT0_4", 80 | "colab_type": "text" 81 | }, 82 | "source": [ 83 | "এখন কি আছে?" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "metadata": { 89 | "id": "Q2A2OwZUT4Ub", 90 | "colab_type": "code", 91 | "outputId": "8c17c232-2a6e-436a-aad2-882e3477a009", 92 | "colab": { 93 | "base_uri": "https://localhost:8080/", 94 | "height": 34 95 | } 96 | }, 97 | "source": [ 98 | "tf.__version__" 99 | ], 100 | "execution_count": 0, 101 | "outputs": [ 102 | { 103 | "output_type": "execute_result", 104 | "data": { 105 | "text/plain": [ 106 | "'2.0.0-alpha0'" 107 | ] 108 | }, 109 | "metadata": { 110 | "tags": [] 111 | }, 112 | "execution_count": 3 113 | } 114 | ] 115 | }, 116 | { 117 | "cell_type": "markdown", 118 | "metadata": { 119 | "id": "zrGTAgw3UIDx", 120 | "colab_type": "text" 121 | }, 122 | "source": [ 123 | "টেন্সর-ফ্লো ২.০ ইনস্টল করে নেই, জিপিইউ ভার্সন " 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "metadata": { 129 | "id": "Hd9aatIiTvK2", 130 | "colab_type": "code", 131 | "outputId": "ad5ad8df-2dbe-49a0-fb64-71a423917c19", 132 | "colab": { 133 | "base_uri": "https://localhost:8080/", 134 | "height": 530 135 | } 136 | }, 137 | "source": [ 138 | "#!pip install tensorflow==2.0.0-alpha0\n", 139 | "!pip install --upgrade tensorflow-gpu==2.0.0-alpha0" 140 | ], 141 | "execution_count": 0, 142 | "outputs": [ 143 | { 144 | "output_type": "stream", 145 | "text": [ 146 | "Collecting tensorflow-gpu==2.0.0-alpha0\n", 147 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/1a/66/32cffad095253219d53f6b6c2a436637bbe45ac4e7be0244557210dc3918/tensorflow_gpu-2.0.0a0-cp36-cp36m-manylinux1_x86_64.whl (332.1MB)\n", 148 | "\u001b[K |████████████████████████████████| 332.1MB 73kB/s \n", 149 | "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.16.3)\n", 150 | "Requirement already satisfied, skipping upgrade: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.33.1)\n", 151 | "Requirement already satisfied, skipping upgrade: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)\n", 152 | "Requirement already satisfied, skipping upgrade: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)\n", 153 | "Collecting tb-nightly<1.14.0a20190302,>=1.14.0a20190301 (from tensorflow-gpu==2.0.0-alpha0)\n", 154 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/a9/51/aa1d756644bf4624c03844115e4ac4058eff77acd786b26315f051a4b195/tb_nightly-1.14.0a20190301-py3-none-any.whl (3.0MB)\n", 155 | "\u001b[K |████████████████████████████████| 3.0MB 44.4MB/s \n", 156 | "\u001b[?25hRequirement already satisfied, skipping upgrade: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.1.0)\n", 157 | "Requirement already satisfied, skipping upgrade: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (3.7.1)\n", 158 | "Requirement already satisfied, skipping upgrade: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.15.0)\n", 159 | "Requirement already satisfied, skipping upgrade: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.12.0)\n", 160 | "Requirement already satisfied, skipping upgrade: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.2.2)\n", 161 | "Collecting google-pasta>=0.1.2 (from tensorflow-gpu==2.0.0-alpha0)\n", 162 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/64/bb/f1bbc131d6294baa6085a222d29abadd012696b73dcbf8cf1bf56b9f082a/google_pasta-0.1.5-py3-none-any.whl (51kB)\n", 163 | "\u001b[K |████████████████████████████████| 61kB 30.9MB/s \n", 164 | "\u001b[?25hRequirement already satisfied, skipping upgrade: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.7)\n", 165 | "Requirement already satisfied, skipping upgrade: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.9)\n", 166 | "Collecting tf-estimator-nightly<1.14.0.dev2019030116,>=1.14.0.dev2019030115 (from tensorflow-gpu==2.0.0-alpha0)\n", 167 | "\u001b[?25l Downloading https://files.pythonhosted.org/packages/13/82/f16063b4eed210dc2ab057930ac1da4fbe1e91b7b051a6c8370b401e6ae7/tf_estimator_nightly-1.14.0.dev2019030115-py2.py3-none-any.whl (411kB)\n", 168 | "\u001b[K |████████████████████████████████| 419kB 55.4MB/s \n", 169 | "\u001b[?25hRequirement already satisfied, skipping upgrade: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (0.15.2)\n", 170 | "Requirement already satisfied, skipping upgrade: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (3.1)\n", 171 | "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0.0-alpha0) (41.0.1)\n", 172 | "Requirement already satisfied, skipping upgrade: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-alpha0) (2.8.0)\n", 173 | "Installing collected packages: tb-nightly, google-pasta, tf-estimator-nightly, tensorflow-gpu\n", 174 | "Successfully installed google-pasta-0.1.5 tb-nightly-1.14.0a20190301 tensorflow-gpu-2.0.0a0 tf-estimator-nightly-1.14.0.dev2019030115\n" 175 | ], 176 | "name": "stdout" 177 | } 178 | ] 179 | }, 180 | { 181 | "cell_type": "markdown", 182 | "metadata": { 183 | "colab_type": "text", 184 | "id": "x_G1zZT5qEfh" 185 | }, 186 | "source": [ 187 | "# tf.Session() কোথায় গেল?" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "metadata": { 193 | "id": "1WsmrcZZUdJw", 194 | "colab_type": "code", 195 | "outputId": "61b9e106-d8df-4b58-a6e9-0e908ac256eb", 196 | "colab": { 197 | "base_uri": "https://localhost:8080/", 198 | "height": 51 199 | } 200 | }, 201 | "source": [ 202 | "hello = tf.constant(\"Hello World from Bangladesh\")\n", 203 | "print(\"Tensor:\", hello)\n", 204 | "print(\"Value :\", hello.numpy())" 205 | ], 206 | "execution_count": 0, 207 | "outputs": [ 208 | { 209 | "output_type": "stream", 210 | "text": [ 211 | "Tensor: tf.Tensor(b'Hello World from Bangladesh', shape=(), dtype=string)\n", 212 | "Value : b'Hello World from Bangladesh'\n" 213 | ], 214 | "name": "stdout" 215 | } 216 | ] 217 | }, 218 | { 219 | "cell_type": "code", 220 | "metadata": { 221 | "colab_type": "code", 222 | "id": "5hien2IEqwLQ", 223 | "outputId": "2af79bbf-1021-4a4d-8d40-1819b702f785", 224 | "colab": { 225 | "base_uri": "https://localhost:8080/", 226 | "height": 34 227 | } 228 | }, 229 | "source": [ 230 | "tf.executing_eagerly()" 231 | ], 232 | "execution_count": 0, 233 | "outputs": [ 234 | { 235 | "output_type": "execute_result", 236 | "data": { 237 | "text/plain": [ 238 | "True" 239 | ] 240 | }, 241 | "metadata": { 242 | "tags": [] 243 | }, 244 | "execution_count": 5 245 | } 246 | ] 247 | }, 248 | { 249 | "cell_type": "markdown", 250 | "metadata": { 251 | "id": "DzclV2ebaJMT", 252 | "colab_type": "text" 253 | }, 254 | "source": [ 255 | "## ইগার এক্সিকিউশন দিয়ে " 256 | ] 257 | }, 258 | { 259 | "cell_type": "code", 260 | "metadata": { 261 | "id": "TuVTcnr3YJTN", 262 | "colab_type": "code", 263 | "outputId": "2be2dad7-f5ea-4765-a656-5932c589d37c", 264 | "colab": { 265 | "base_uri": "https://localhost:8080/", 266 | "height": 34 267 | } 268 | }, 269 | "source": [ 270 | "import tensorflow as tf\n", 271 | "\n", 272 | "x = [[2.]]\n", 273 | "y = [[1.]]\n", 274 | "k = tf.add(x, y)\n", 275 | "print(k) # k is the Tensor [[3.]]" 276 | ], 277 | "execution_count": 0, 278 | "outputs": [ 279 | { 280 | "output_type": "stream", 281 | "text": [ 282 | "tf.Tensor([[3.]], shape=(1, 1), dtype=float32)\n" 283 | ], 284 | "name": "stdout" 285 | } 286 | ] 287 | }, 288 | { 289 | "cell_type": "markdown", 290 | "metadata": { 291 | "id": "L3tROI00X5su", 292 | "colab_type": "text" 293 | }, 294 | "source": [ 295 | "## আগে টেন্সর-ফ্লো কোর দিয়ে যদি করতাম " 296 | ] 297 | }, 298 | { 299 | "cell_type": "code", 300 | "metadata": { 301 | "id": "TrlHs93pX_lX", 302 | "colab_type": "code", 303 | "outputId": "a692c7df-ccb7-4038-e4e2-74fcf5f4dbf2", 304 | "colab": { 305 | "base_uri": "https://localhost:8080/", 306 | "height": 34 307 | } 308 | }, 309 | "source": [ 310 | "# এটা করা ঠিক না \n", 311 | "import tensorflow.compat.v1 as tf\n", 312 | "tf.disable_v2_behavior()\n", 313 | "\n", 314 | "x = tf.placeholder(tf.float32, shape=[1, 1])\n", 315 | "y = tf.placeholder(tf.float32, shape=[1, 1])\n", 316 | "k = tf.add(x, y) # k is a Tensor, but has no value assigned yet\n", 317 | "with tf.Session() as sess:\n", 318 | " k = sess.run(k, feed_dict={x: [[2.]], y: [[1.]]})\n", 319 | "print(k) # k is 3." 320 | ], 321 | "execution_count": 0, 322 | "outputs": [ 323 | { 324 | "output_type": "stream", 325 | "text": [ 326 | "[[3.]]\n" 327 | ], 328 | "name": "stdout" 329 | } 330 | ] 331 | } 332 | ] 333 | } -------------------------------------------------------------------------------- /tensorflow_serving/README.md: -------------------------------------------------------------------------------- 1 | Test for the tensorflow serving 2 | -------------------------------------------------------------------------------- /tensorflow_serving/curl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raqueeb/TensorFlow2/e9698ed7e7339554ae80cd4b33924a6ac33e8456/tensorflow_serving/curl.png -------------------------------------------------------------------------------- /tensorflow_serving/docker_curl.md: -------------------------------------------------------------------------------- 1 | PS C:\Users\Test> docker pull tensorflow/serving 2 | 3 | Using default tag: latest 4 | latest: Pulling from tensorflow/serving 5 | 22e816666fd6: Pull complete 6 | 079b6d2a1e53: Pull complete 7 | 11048ebae908: Pull complete 8 | c58094023a2e: Pull complete 9 | 4dee0153a839: Pull complete 10 | 90850b98765d: Pull complete 11 | de0a35913cb5: Pull complete 12 | 91e09abfcd7f: Pull complete 13 | Digest: sha256:091c1d0440815e250114a6d0232ad3cb1d320c64b1ebb75ed8a80184fc25482d 14 | Status: Downloaded newer image for tensorflow/serving:latest 15 | docker.io/tensorflow/serving:latest 16 | 17 | 18 | 19 | $ git clone https://github.com/tensorflow/serving 20 | 21 | Cloning into 'serving'... 22 | remote: Enumerating objects: 54, done. 23 | remote: Counting objects: 100% (54/54), done. 24 | remote: Compressing objects: 100% (52/52), done. 25 | Receiving objects: 48% (9475/19738), 3.61 MiB | 195.00 KiB/s 26 | remote: Total 19738 (delta 36), reused 20 (delta 2), pack-reused 19684 27 | Receiving objects: 100% (19738/19738), 5.22 MiB | 178.00 KiB/s, done. 28 | Resolving deltas: 100% (14964/14964), done. 29 | 30 | 31 | 32 | PS E:\git_portable> docker run -t --rm -p 8501:8501 -v "E:\git_portable\serving\tensorflow_serving\servables\tensorflow\testdata\saved_model_half_plus_two_cpu:/models/half_plus_two" -e MODEL_NAME=half_plus_two tensorflow/serving 33 | 34 | 2019-11-10 07:11:17.037045: I tensorflow_serving/model_servers/server.cc:85] Building single TensorFlow model file config: model_name: half_plus_two model_base_path: /models/half_plus_two 35 | 2019-11-10 07:11:17.037797: I tensorflow_serving/model_servers/server_core.cc:462] Adding/updating models. 36 | 2019-11-10 07:11:17.037861: I tensorflow_serving/model_servers/server_core.cc:573] (Re-)adding model: half_plus_two 37 | 2019-11-10 07:11:17.158245: I tensorflow_serving/core/basic_manager.cc:739] Successfully reserved resources to load servable {name: half_plus_two version: 123} 38 | 2019-11-10 07:11:17.158435: I tensorflow_serving/core/loader_harness.cc:66] Approving load for servable version {name: half_plus_two version: 123} 39 | 2019-11-10 07:11:17.158496: I tensorflow_serving/core/loader_harness.cc:74] Loading servable version {name: half_plus_two version: 123} 40 | 2019-11-10 07:11:17.158573: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:31] Reading SavedModel from: /models/half_plus_two/00000123 41 | 2019-11-10 07:11:17.170610: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:54] Reading meta graph with tags { serve } 42 | 2019-11-10 07:11:17.172642: I external/org_tensorflow/tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA 43 | 2019-11-10 07:11:17.212202: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:202] Restoring SavedModel bundle. 44 | 2019-11-10 07:11:17.230431: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:151] Running initialization op on SavedModel bundle at path: /models/half_plus_two/00000123 45 | 2019-11-10 07:11:17.236016: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:311] SavedModel load for tags { serve }; Status: success. Took 77445 microseconds. 46 | 2019-11-10 07:11:17.237262: I tensorflow_serving/servables/tensorflow/saved_model_warmup.cc:105] No warmup data file found at /models/half_plus_two/00000123/assets.extra/tf_serving_warmup_requests 47 | 2019-11-10 07:11:17.247605: I tensorflow_serving/core/loader_harness.cc:87] Successfully loaded servable version {name: half_plus_two version: 123} 48 | 2019-11-10 07:11:17.250931: I tensorflow_serving/model_servers/server.cc:353] Running gRPC ModelServer at 0.0.0.0:8500 ... 49 | [warn] getaddrinfo: address family for nodename not supported 50 | 2019-11-10 07:11:17.252948: I tensorflow_serving/model_servers/server.cc:373] Exporting HTTP/REST API at:localhost:8501 ... 51 | 52 | 53 | ![Curl](../assets/curl.png "Curl call") 54 | 55 | 56 | PS E:\git_portable> docker ps 57 | 58 | CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 59 | c6e0b94c10b9 tensorflow/tensorflow:2.0.0-py3-jupyter-pandas-sklearn "bash -c 'source /et…" 2 hours ago Up 2 hours 0.0.0.0:8888->8888/tcp tensorflow2 60 | c0a477f441fb tensorflow/serving "/usr/bin/tf_serving…" 2 hours ago Up 2 hours 8500/tcp, 0.0.0.0:8501->8501/tcp hungry_robinson 61 | 62 | 63 | PS E:\git_portable> curl -d '{"instances": [1.0, 2.0, 5.0]}' -X POST http://localhost:8501/v1/models/half_plus_two:predict 64 | 65 | Invoke-WebRequest : A parameter cannot be found that matches parameter name 'X'. 66 | At line:1 char:42 67 | + curl -d '{"instances": [1.0, 2.0, 5.0]}' -X POST http://localhost:850 ... 68 | + CategoryInfo : InvalidArgument: (:) [Invoke-WebRequest], ParameterBindingException 69 | + FullyQualifiedErrorId : NamedParameterNotFound,Microsoft.PowerShell.Commands.InvokeWebRequestCommand 70 | 71 | 72 | tensorflow / tensorflow : 2.0.0rc0-py3-jupyter 73 | 74 | docker pull tensorflow/tensorflow:2.0.0-py3-jupyter 75 | 76 | Working 77 | 78 | docker run -it -p 8888:8888 -v "c:/users/test/google drive/dlbook:/tf" --rm --name tensorflow2 tensorflow/tensorflow:2.0.0-py3-jupyter-pandas 79 | 80 | docker run -it -p 8888:8888 -v "c:/users/test/google drive:/tf" --rm --name tensorflow2 tensorflow/tensorflow:2.0.0-py3-jupyter-pandas 81 | 82 | docker run -it -p 8888:8888 -v "c:/users/test/google drive:/tf" --rm --name tensorflow2 tensorflow/tensorflow:2.0.0-py3-jupyter 83 | 84 | docker run -it -p 8888:8888 -v "c:/users/test/google drive/dlbook:/tf" --rm tensorflow/tensorflow:2.0.0-py3-jupyter --restart unless-stopped 85 | 86 | docker run -it -p 8888:8888 -v "c:/users/test/google drive/dlbook:/tf" tensorflow/tensorflow:2.0.0-py3-jupyter 87 | 88 | 89 | docker run -it -p 8888:8888 -v e:/posts:/tf --rm tensorflow/tensorflow:2.0.0-py3-jupyter 90 | 91 | docker run -it -p 8888:8888 -v ${PWD}:/tf --rm tensorflow/tensorflow:2.0.0-py3-jupyter 92 | 93 | -v ${PWD} 94 | 95 | docker run -it -p 8888:8888 -v C:/Users/Test/Google Drive/DLbook:/tf --rm tensorflow/tensorflow:2.0.0-py3-jupyter 96 | 97 | "C:\Users\Test\Google Drive\DLbook" 98 | 99 | docker run -it -p 8888:8888 -v e:/posts:/tf --rm tensorflow/tensorflow:2.0.0-py3-jupyter 100 | 101 | docker run -it -p 8888:8888 tensorflow/tensorflow:2.0.0-py3-jupyter 102 | 103 | docker run -it -p 8888:8888 tensorflow/tensorflow:2.0.0rc0-py3-jupyter 104 | 105 | docker run -it -p 8888:8888 -v C:\Users\Mr. raqueeb\Documents:/tmp --rm --name tensorflow2 tensorflow/tensorflow:2.0.0rc0-py3-jupyter 106 | 107 | docker run -it -p 8888:8888 -v C:\ProgramData\Docker:/tmp --rm --name tensorflow2 tensorflow/tensorflow:2.0.0rc0-py3-jupyter 108 | 109 | docker run -it -p 8888:8888 -v C:\ProgramData\Docker:/notebook --rm --name tensorflow2 tensorflow/tensorflow:2.0.0rc0-py3-jupyter 110 | 111 | docker run -it p 8888:8888 --volume //e/posts:/tf -rm --name tensorflow2 tensorflow/tensorflow:2.0.0-py3-jupyter 112 | 113 | docker create -t -i -v i:/project/web01:/mnt/test fedora /bin/bash 114 | 115 | docker run -it p 8888:8888 -v e:/posts:/tf -rm --name tensorflow2 tensorflow/tensorflow:2.0.0-py3-jupyter 116 | 117 | C:\ProgramData\Docker 118 | 119 | docker pull tensorflow/tensorflow:latest-gpu-jupyter 120 | 121 | docker container ls -a 122 | 123 | docker container rm 7f6a0963b4fc 124 | 125 | docker container stop $(docker container ls -aq) 126 | 127 | docker run -it tensorflow/tensorflow:2.0.0-py3-jupyter /bin/bash 128 | pip install pandas 129 | 130 | docker export --output="tensorflow.tar" fc8da732849a 131 | 132 | docker run tensorflow/tensorflow:2.0.0-py3-jupyter bash -c "apt-get update && apt-get install -y pandas" 133 | 134 | PS C:\Users\Test> docker pull tensorflow/serving 135 | Using default tag: latest 136 | latest: Pulling from tensorflow/serving 137 | 22e816666fd6: Pull complete 079b6d2a1e53: Pull complete 11048ebae908: Pull complete c58094023a2e: Pull complete 4dee0153a839: Pull complete 90850b98765d: Pull complete de0a35913cb5: Pull complete 91e09abfcd7f: Pull complete Digest: sha256:091c1d0440815e250114a6d0232ad3cb1d320c64b1ebb75ed8a80184fc25482d 138 | Status: Downloaded newer image for tensorflow/serving:latest 139 | docker.io/tensorflow/serving:latest 140 | 141 | 142 | curl -d '{"instances": [1.0, 2.0, 5.0]}' -X POST http://localhost:8501/v1/models/half_plus_two:predict 143 | 144 | curl -d '{"instances": [1.0, 2.0, 5.0]}' -X POST http://localhost:8501/v1/models/half_plus_two:predict 145 | 146 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0, 5.0, 9.0]}" -X POST http://127.0.0.1:8501/v1/models/half_plus_two:predict 147 | { 148 | "predictions": [2.5, 4.5, 6.5 149 | ] 150 | } 151 | 152 | E:\git_portable\serving\tensorflow_serving\servables\tensorflow\testdata\saved_model_half_plus_three 153 | 154 | 155 | docker run -t --rm -p 8507:8507 -v "E:\git_portable\serving\tensorflow_serving\servables\tensorflow\testdata\saved_model_half_plus_three:/models/half_plus_three" -e MODEL_NAME=half_plus_three tensorflow/serving 156 | 157 | 158 | curl -d "{\"instances\": [1.0,2.0,5.0]}" -X POST http://localhost:8507/v1/models/saved_model_half_plus_three:predict 159 | 160 | 161 | PS E:\git_portable> docker run -t --rm -p 8507:8507 -v "E:\git_portable\serving\tensorflow_serving\servables\tensorflow\testdata\saved_model_half_plus_three:/models/half_plus_three" -e MODEL_NAME=half_plus_three tensorflow/serving 162 | 2019-11-10 16:32:41.006983: I tensorflow_serving/model_servers/server.cc:85] Building single TensorFlow model file config: model_name: half_plus_three model_base_path: /models/half_plus_three 163 | 2019-11-10 16:32:41.007173: I tensorflow_serving/model_servers/server_core.cc:462] Adding/updating models. 164 | 2019-11-10 16:32:41.007190: I tensorflow_serving/model_servers/server_core.cc:573] (Re-)adding model: half_plus_three 165 | 2019-11-10 16:32:41.112369: I tensorflow_serving/core/basic_manager.cc:739] Successfully reserved resources to load servable {name: half_plus_three version: 123} 166 | 2019-11-10 16:32:41.112428: I tensorflow_serving/core/loader_harness.cc:66] Approving load for servable version {name: half_plus_three version: 123} 167 | 2019-11-10 16:32:41.112460: I tensorflow_serving/core/loader_harness.cc:74] Loading servable version {name: half_plus_three version: 123} 168 | 2019-11-10 16:32:41.112493: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:31] Reading SavedModel from: /models/half_plus_three/00000123 169 | 2019-11-10 16:32:41.190027: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:54] Reading meta graph with tags { serve } 170 | 2019-11-10 16:32:41.190727: I external/org_tensorflow/tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA 171 | 2019-11-10 16:32:41.210597: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:202] Restoring SavedModel bundle. 172 | 2019-11-10 16:32:41.241783: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:151] Running initialization op on SavedModel bundle at path: /models/half_plus_three/00000123 173 | 2019-11-10 16:32:41.247985: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:311] SavedModel load for tags { serve }; Status: success. Took 135489 microseconds. 174 | 2019-11-10 16:32:41.249075: I tensorflow_serving/servables/tensorflow/saved_model_warmup.cc:105] No warmup data file found at /models/half_plus_three/00000123/assets.extra/tf_serving_warmup_requests 175 | 2019-11-10 16:32:41.257380: I tensorflow_serving/core/loader_harness.cc:87] Successfully loaded servable version {name: half_plus_three version: 123} 176 | 2019-11-10 16:32:41.261126: I tensorflow_serving/model_servers/server.cc:353] Running gRPC ModelServer at 0.0.0.0:8500 ... 177 | [warn] getaddrinfo: address family for nodename not supported 178 | 2019-11-10 16:32:41.262500: I tensorflow_serving/model_servers/server.cc:373] Exporting HTTP/REST API at:localhost:8501 ... 179 | [evhttp_server.cc : 238] NET_LOG: Entering the event loop ... 180 | PS E:\git_portable> 181 | PS E:\git_portable> docker ps 182 | CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 183 | 5f0bb96c40ed tensorflow/serving "/usr/bin/tf_serving…" 7 minutes ago Up 7 minutes 8500-8501/tcp, 0.0.0.0:8507->8507/tcp epic_mendel 184 | c6e0b94c10b9 tensorflow/tensorflow:2.0.0-py3-jupyter-pandas-sklearn "bash -c 'source /et…" 9 hours ago Up 9 hours 0.0.0.0:8888->8888/tcp tensorflow2 185 | c0a477f441fb tensorflow/serving "/usr/bin/tf_serving…" 9 hours ago Up 9 hours 8500/tcp, 0.0.0.0:8501->8501/tcp hungry_robinson 186 | PS E:\git_portable> 187 | PS E:\git_portable> docker container stop $(docker container ls -aq) 188 | 5f0bb96c40ed 189 | c6e0b94c10b9 190 | c0a477f441fb 191 | 192 | 193 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0, 5.0, 9.0]}" -X POST http://127.0.0.1:8501/v1/models/half_plus_two:predict 194 | { 195 | "predictions": [2.5, 4.5, 6.5 196 | ] 197 | } 198 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0, 5.0, 9.0]}" -X POST http://localhost:8507/v1/models/saved_model_half_plus_three:predict 199 | curl: (52) Empty reply from server 200 | 201 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0, 5.0, 9.0]}" -X POST http://localhost:8501/v1/models/saved_model_half_plus_three:predict 202 | { "error": "Servable not found for request: Latest(saved_model_half_plus_three)" } 203 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0, 5.0, 9.0]}" -X POST http://localhost:8507/v1/models/saved_model_half_plus_three:predict 204 | curl: (52) Empty reply from server 205 | 206 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0, 5.0, 9.0]}" -X POST http://localhost:8507/v1/models/saved_model_half_plus_three:predict 207 | curl: (52) Empty reply from server 208 | 209 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0, 5.0, 9.0]}" -X POST http://localhost:8507/v1/models/saved_model_half_plus_three:predict 210 | curl: (52) Empty reply from server 211 | 212 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0,2.0,5.0]}" -X POST http://localhost:8507/v1/models/saved_model_half_plus_three:predict 213 | curl: (52) Empty reply from server 214 | 215 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0,2.0,5.0]}" -X POST http://localhost:8500/v1/models/saved_model_half_plus_three:predict 216 | curl: (7) Failed to connect to localhost port 8500: Connection refused 217 | 218 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0,2.0,5.0]}" -X POST http://localhost:8501/v1/models/saved_model_half_plus_three:predict 219 | curl: (7) Failed to connect to localhost port 8501: Connection refused 220 | 221 | C:\WINDOWS\system32>curl -d "{\"instances\": [1.0,2.0,5.0]}" -X POST http://localhost:8507/v1/models/saved_model_half_plus_three:predict 222 | curl: (52) Empty reply from server 223 | 224 | C:\WINDOWS\system32> 225 | -------------------------------------------------------------------------------- /text_classification.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "text_classification.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyOSr9zFdQ4SSOAJ/mO4lVfv", 10 | "include_colab_link": true 11 | }, 12 | "kernelspec": { 13 | "name": "python3", 14 | "display_name": "Python 3" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "view-in-github", 22 | "colab_type": "text" 23 | }, 24 | "source": [ 25 | "\"Open" 26 | ] 27 | }, 28 | { 29 | "cell_type": "markdown", 30 | "metadata": { 31 | "id": "tpO_fxIyVySA", 32 | "colab_type": "text" 33 | }, 34 | "source": [ 35 | "## আমাদের কর্পাস\n", 36 | "\n", 37 | "বাংলা নাচারাল ল্যাঙ্গুয়েজ প্রসেসিং\n", 38 | "\n", 39 | "১. টোকেনাইজেশন\n", 40 | "\n", 41 | "২. ভেক্টরাইজেশন" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "metadata": { 47 | "id": "D3yDo75TdgkK", 48 | "colab_type": "code", 49 | "colab": {} 50 | }, 51 | "source": [ 52 | "sentences = ['আমি মাঝে মধ্যেই ফিরে যাই পুরানো কিছু ক্লাসিক বইয়ে', 'বিশেষ করে বেসিক ঝালাই করার জন্য']" 53 | ], 54 | "execution_count": 0, 55 | "outputs": [] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "metadata": { 60 | "id": "qnhM3rgr25jd", 61 | "colab_type": "code", 62 | "outputId": "547f317e-1c19-4bd4-a522-ca98aed2d4c0", 63 | "colab": { 64 | "base_uri": "https://localhost:8080/", 65 | "height": 69 66 | } 67 | }, 68 | "source": [ 69 | "import nltk\n", 70 | "nltk.download('punkt')" 71 | ], 72 | "execution_count": 55, 73 | "outputs": [ 74 | { 75 | "output_type": "stream", 76 | "text": [ 77 | "[nltk_data] Downloading package punkt to /root/nltk_data...\n", 78 | "[nltk_data] Package punkt is already up-to-date!\n" 79 | ], 80 | "name": "stdout" 81 | }, 82 | { 83 | "output_type": "execute_result", 84 | "data": { 85 | "text/plain": [ 86 | "True" 87 | ] 88 | }, 89 | "metadata": { 90 | "tags": [] 91 | }, 92 | "execution_count": 55 93 | } 94 | ] 95 | }, 96 | { 97 | "cell_type": "code", 98 | "metadata": { 99 | "id": "mOGXQnKRWGYP", 100 | "colab_type": "code", 101 | "outputId": "5ff83c04-4ac8-4214-a085-9e86b66a7632", 102 | "colab": { 103 | "base_uri": "https://localhost:8080/", 104 | "height": 35 105 | } 106 | }, 107 | "source": [ 108 | "from sklearn.feature_extraction.text import CountVectorizer\n", 109 | "\n", 110 | "# ট্রান্সফরমেশন তৈরি করি\n", 111 | "vectorizer = CountVectorizer()\n", 112 | "\n", 113 | "# টোকেনাইজ এবং ভোকাবুলারি তৈরি করি\n", 114 | "vectorizer.fit(sentences)\n", 115 | "\n", 116 | "# সামারি দেখি\n", 117 | "vectorizer.vocabulary_" 118 | ], 119 | "execution_count": 56, 120 | "outputs": [ 121 | { 122 | "output_type": "execute_result", 123 | "data": { 124 | "text/plain": [ 125 | "{'আম': 0, 'কর': 1, 'জন': 2, 'বইয়': 3, 'মধ': 4}" 126 | ] 127 | }, 128 | "metadata": { 129 | "tags": [] 130 | }, 131 | "execution_count": 56 132 | } 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "metadata": { 138 | "id": "PrzmLNq0W7Dw", 139 | "colab_type": "code", 140 | "colab": {} 141 | }, 142 | "source": [ 143 | "import warnings\n", 144 | "warnings.filterwarnings(\"ignore\")" 145 | ], 146 | "execution_count": 0, 147 | "outputs": [] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "metadata": { 152 | "id": "B7YCjhUbdlWL", 153 | "colab_type": "code", 154 | "outputId": "b28d89eb-10d8-4e53-c08e-693d83d7927b", 155 | "colab": { 156 | "base_uri": "https://localhost:8080/", 157 | "height": 277 158 | } 159 | }, 160 | "source": [ 161 | "# ইউনিকোডে দেখুন নিচের লিঙ্কে\n", 162 | "# https://jrgraphix.net/r/Unicode/0980-09FF\n", 163 | "\n", 164 | "from nltk import word_tokenize\n", 165 | "\n", 166 | "vectorizer = CountVectorizer(encoding='utf-8', tokenizer=word_tokenize)\n", 167 | "\n", 168 | "vectorizer.fit(sentences)\n", 169 | "vectorizer.vocabulary_" 170 | ], 171 | "execution_count": 58, 172 | "outputs": [ 173 | { 174 | "output_type": "execute_result", 175 | "data": { 176 | "text/plain": [ 177 | "{'আমি': 0,\n", 178 | " 'করার': 1,\n", 179 | " 'করে': 2,\n", 180 | " 'কিছু': 3,\n", 181 | " 'ক্লাসিক': 4,\n", 182 | " 'জন্য': 5,\n", 183 | " 'ঝালাই': 6,\n", 184 | " 'পুরানো': 7,\n", 185 | " 'ফিরে': 8,\n", 186 | " 'বইয়ে': 9,\n", 187 | " 'বিশেষ': 10,\n", 188 | " 'বেসিক': 11,\n", 189 | " 'মধ্যেই': 12,\n", 190 | " 'মাঝে': 13,\n", 191 | " 'যাই': 14}" 192 | ] 193 | }, 194 | "metadata": { 195 | "tags": [] 196 | }, 197 | "execution_count": 58 198 | } 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "metadata": { 204 | "id": "3aRbrXdvU-kB", 205 | "colab_type": "code", 206 | "outputId": "c0319153-7390-4980-cb26-9a5b0c34d773", 207 | "colab": { 208 | "base_uri": "https://localhost:8080/", 209 | "height": 55 210 | } 211 | }, 212 | "source": [ 213 | "print(vectorizer.vocabulary_)" 214 | ], 215 | "execution_count": 59, 216 | "outputs": [ 217 | { 218 | "output_type": "stream", 219 | "text": [ 220 | "{'আমি': 0, 'মাঝে': 13, 'মধ্যেই': 12, 'ফিরে': 8, 'যাই': 14, 'পুরানো': 7, 'কিছু': 3, 'ক্লাসিক': 4, 'বইয়ে': 9, 'বিশেষ': 10, 'করে': 2, 'বেসিক': 11, 'ঝালাই': 6, 'করার': 1, 'জন্য': 5}\n" 221 | ], 222 | "name": "stdout" 223 | } 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "metadata": { 229 | "id": "vp-68i2y4-p6", 230 | "colab_type": "code", 231 | "outputId": "b1744d15-ff45-4dde-bb96-581f92de9943", 232 | "colab": { 233 | "base_uri": "https://localhost:8080/", 234 | "height": 52 235 | } 236 | }, 237 | "source": [ 238 | "vectorizer.transform(sentences).toarray()" 239 | ], 240 | "execution_count": 60, 241 | "outputs": [ 242 | { 243 | "output_type": "execute_result", 244 | "data": { 245 | "text/plain": [ 246 | "array([[1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1],\n", 247 | " [0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0]])" 248 | ] 249 | }, 250 | "metadata": { 251 | "tags": [] 252 | }, 253 | "execution_count": 60 254 | } 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "metadata": { 260 | "id": "Wdb8FgzeOE0-", 261 | "colab_type": "code", 262 | "outputId": "ce5dce6e-aa6a-42bb-c4f1-ae9e316fef90", 263 | "colab": { 264 | "base_uri": "https://localhost:8080/", 265 | "height": 52 266 | } 267 | }, 268 | "source": [ 269 | "vec = CountVectorizer()\n", 270 | "x = vec.fit_transform(sentences).toarray()\n", 271 | "print(x.shape)\n", 272 | "print(vec.get_feature_names())" 273 | ], 274 | "execution_count": 61, 275 | "outputs": [ 276 | { 277 | "output_type": "stream", 278 | "text": [ 279 | "(2, 5)\n", 280 | "['আম', 'কর', 'জন', 'বইয়', 'মধ']\n" 281 | ], 282 | "name": "stdout" 283 | } 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "metadata": { 289 | "id": "43MkcTpZ6klp", 290 | "colab_type": "code", 291 | "colab": { 292 | "base_uri": "https://localhost:8080/", 293 | "height": 277 294 | }, 295 | "outputId": "055f7f1f-51fa-4ebc-82c7-539401057eea" 296 | }, 297 | "source": [ 298 | "# ইউনিকোডে দেখুন নিচের লিঙ্কে\n", 299 | "# https://jrgraphix.net/r/Unicode/0980-09FF\n", 300 | "\n", 301 | "vectorizer = CountVectorizer(encoding='utf-8', token_pattern=r'[\\u0980-\\u09ff]+')\n", 302 | "vectorizer.fit(sentences)\n", 303 | "vectorizer.vocabulary_" 304 | ], 305 | "execution_count": 62, 306 | "outputs": [ 307 | { 308 | "output_type": "execute_result", 309 | "data": { 310 | "text/plain": [ 311 | "{'আমি': 0,\n", 312 | " 'করার': 1,\n", 313 | " 'করে': 2,\n", 314 | " 'কিছু': 3,\n", 315 | " 'ক্লাসিক': 4,\n", 316 | " 'জন্য': 5,\n", 317 | " 'ঝালাই': 6,\n", 318 | " 'পুরানো': 7,\n", 319 | " 'ফিরে': 8,\n", 320 | " 'বইয়ে': 9,\n", 321 | " 'বিশেষ': 10,\n", 322 | " 'বেসিক': 11,\n", 323 | " 'মধ্যেই': 12,\n", 324 | " 'মাঝে': 13,\n", 325 | " 'যাই': 14}" 326 | ] 327 | }, 328 | "metadata": { 329 | "tags": [] 330 | }, 331 | "execution_count": 62 332 | } 333 | ] 334 | }, 335 | { 336 | "cell_type": "code", 337 | "metadata": { 338 | "id": "F3UJHipiOma3", 339 | "colab_type": "code", 340 | "outputId": "57419cba-72a5-4202-f640-7594a8e1bddc", 341 | "colab": { 342 | "base_uri": "https://localhost:8080/", 343 | "height": 277 344 | } 345 | }, 346 | "source": [ 347 | "from sklearn.feature_extraction.text import TfidfVectorizer\n", 348 | "\n", 349 | "vectorizer =TfidfVectorizer(encoding='utf-8', tokenizer=word_tokenize)\n", 350 | "\n", 351 | "vectorizer.fit(sentences)\n", 352 | "vectorizer.vocabulary_" 353 | ], 354 | "execution_count": 63, 355 | "outputs": [ 356 | { 357 | "output_type": "execute_result", 358 | "data": { 359 | "text/plain": [ 360 | "{'আমি': 0,\n", 361 | " 'করার': 1,\n", 362 | " 'করে': 2,\n", 363 | " 'কিছু': 3,\n", 364 | " 'ক্লাসিক': 4,\n", 365 | " 'জন্য': 5,\n", 366 | " 'ঝালাই': 6,\n", 367 | " 'পুরানো': 7,\n", 368 | " 'ফিরে': 8,\n", 369 | " 'বইয়ে': 9,\n", 370 | " 'বিশেষ': 10,\n", 371 | " 'বেসিক': 11,\n", 372 | " 'মধ্যেই': 12,\n", 373 | " 'মাঝে': 13,\n", 374 | " 'যাই': 14}" 375 | ] 376 | }, 377 | "metadata": { 378 | "tags": [] 379 | }, 380 | "execution_count": 63 381 | } 382 | ] 383 | }, 384 | { 385 | "cell_type": "code", 386 | "metadata": { 387 | "id": "Dsr8-tmwdp0O", 388 | "colab_type": "code", 389 | "outputId": "4bd0b4eb-abd9-44b8-ce30-28392851df49", 390 | "colab": { 391 | "base_uri": "https://localhost:8080/", 392 | "height": 121 393 | } 394 | }, 395 | "source": [ 396 | "vectorizer.transform(sentences).toarray()" 397 | ], 398 | "execution_count": 64, 399 | "outputs": [ 400 | { 401 | "output_type": "execute_result", 402 | "data": { 403 | "text/plain": [ 404 | "array([[0.33333333, 0. , 0. , 0.33333333, 0.33333333,\n", 405 | " 0. , 0. , 0.33333333, 0.33333333, 0.33333333,\n", 406 | " 0. , 0. , 0.33333333, 0.33333333, 0.33333333],\n", 407 | " [0. , 0.40824829, 0.40824829, 0. , 0. ,\n", 408 | " 0.40824829, 0.40824829, 0. , 0. , 0. ,\n", 409 | " 0.40824829, 0.40824829, 0. , 0. , 0. ]])" 410 | ] 411 | }, 412 | "metadata": { 413 | "tags": [] 414 | }, 415 | "execution_count": 64 416 | } 417 | ] 418 | }, 419 | { 420 | "cell_type": "code", 421 | "metadata": { 422 | "id": "qNEsv14BYRIQ", 423 | "colab_type": "code", 424 | "outputId": "5fee99ef-094e-4011-f4d6-bc50f96c75b4", 425 | "colab": { 426 | "base_uri": "https://localhost:8080/", 427 | "height": 69 428 | } 429 | }, 430 | "source": [ 431 | "print(vectorizer.idf_)" 432 | ], 433 | "execution_count": 65, 434 | "outputs": [ 435 | { 436 | "output_type": "stream", 437 | "text": [ 438 | "[1.40546511 1.40546511 1.40546511 1.40546511 1.40546511 1.40546511\n", 439 | " 1.40546511 1.40546511 1.40546511 1.40546511 1.40546511 1.40546511\n", 440 | " 1.40546511 1.40546511 1.40546511]\n" 441 | ], 442 | "name": "stdout" 443 | } 444 | ] 445 | }, 446 | { 447 | "cell_type": "code", 448 | "metadata": { 449 | "id": "5vt20FLIYpoW", 450 | "colab_type": "code", 451 | "outputId": "01e1f0a3-c306-47e8-8031-f553259eec74", 452 | "colab": { 453 | "base_uri": "https://localhost:8080/", 454 | "height": 87 455 | } 456 | }, 457 | "source": [ 458 | "vector = vectorizer.transform([sentences[0]])\n", 459 | "# এনকোডেড ভেক্টরকে সামারাইজ করি\n", 460 | "print(vector.shape)\n", 461 | "print(vector.toarray())" 462 | ], 463 | "execution_count": 66, 464 | "outputs": [ 465 | { 466 | "output_type": "stream", 467 | "text": [ 468 | "(1, 15)\n", 469 | "[[0.33333333 0. 0. 0.33333333 0.33333333 0.\n", 470 | " 0. 0.33333333 0.33333333 0.33333333 0. 0.\n", 471 | " 0.33333333 0.33333333 0.33333333]]\n" 472 | ], 473 | "name": "stdout" 474 | } 475 | ] 476 | }, 477 | { 478 | "cell_type": "code", 479 | "metadata": { 480 | "id": "Sx0aljIreVvg", 481 | "colab_type": "code", 482 | "outputId": "17918534-dc76-492e-a78c-89b9199e2e87", 483 | "colab": { 484 | "base_uri": "https://localhost:8080/", 485 | "height": 35 486 | } 487 | }, 488 | "source": [ 489 | "cities = ['ঢাকা', 'বার্লিন', 'কুমিল্লা', 'শিকাগো', 'সিঙ্গাপুর']\n", 490 | "cities" 491 | ], 492 | "execution_count": 67, 493 | "outputs": [ 494 | { 495 | "output_type": "execute_result", 496 | "data": { 497 | "text/plain": [ 498 | "['ঢাকা', 'বার্লিন', 'কুমিল্লা', 'শিকাগো', 'সিঙ্গাপুর']" 499 | ] 500 | }, 501 | "metadata": { 502 | "tags": [] 503 | }, 504 | "execution_count": 67 505 | } 506 | ] 507 | }, 508 | { 509 | "cell_type": "code", 510 | "metadata": { 511 | "id": "uaK8h7hgfbuW", 512 | "colab_type": "code", 513 | "colab": {} 514 | }, 515 | "source": [ 516 | "from sklearn.preprocessing import LabelEncoder" 517 | ], 518 | "execution_count": 0, 519 | "outputs": [] 520 | }, 521 | { 522 | "cell_type": "code", 523 | "metadata": { 524 | "id": "8WYvhQsEffDV", 525 | "colab_type": "code", 526 | "outputId": "9c623882-50b3-4133-b132-bb39dd3a6ae9", 527 | "colab": { 528 | "base_uri": "https://localhost:8080/", 529 | "height": 35 530 | } 531 | }, 532 | "source": [ 533 | "encoder = LabelEncoder()\n", 534 | "city_labels = encoder.fit_transform(cities)\n", 535 | "city_labels" 536 | ], 537 | "execution_count": 69, 538 | "outputs": [ 539 | { 540 | "output_type": "execute_result", 541 | "data": { 542 | "text/plain": [ 543 | "array([1, 2, 0, 3, 4])" 544 | ] 545 | }, 546 | "metadata": { 547 | "tags": [] 548 | }, 549 | "execution_count": 69 550 | } 551 | ] 552 | }, 553 | { 554 | "cell_type": "code", 555 | "metadata": { 556 | "id": "GK4o6WGIhnxh", 557 | "colab_type": "code", 558 | "outputId": "861b1f05-e1ef-4e50-f09e-0f7d3193d942", 559 | "colab": { 560 | "base_uri": "https://localhost:8080/", 561 | "height": 104 562 | } 563 | }, 564 | "source": [ 565 | "from sklearn.preprocessing import OneHotEncoder\n", 566 | "\n", 567 | "encoder = OneHotEncoder(sparse=False)\n", 568 | "city_labels = city_labels.reshape((5, 1))\n", 569 | "encoder.fit_transform(city_labels)" 570 | ], 571 | "execution_count": 70, 572 | "outputs": [ 573 | { 574 | "output_type": "execute_result", 575 | "data": { 576 | "text/plain": [ 577 | "array([[0., 1., 0., 0., 0.],\n", 578 | " [0., 0., 1., 0., 0.],\n", 579 | " [1., 0., 0., 0., 0.],\n", 580 | " [0., 0., 0., 1., 0.],\n", 581 | " [0., 0., 0., 0., 1.]])" 582 | ] 583 | }, 584 | "metadata": { 585 | "tags": [] 586 | }, 587 | "execution_count": 70 588 | } 589 | ] 590 | } 591 | ] 592 | } -------------------------------------------------------------------------------- /wikipedia_embeddings.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "wikipedia_embeddings.ipynb", 7 | "provenance": [], 8 | "include_colab_link": true 9 | }, 10 | "kernelspec": { 11 | "display_name": "Python 3", 12 | "language": "python", 13 | "name": "python3" 14 | }, 15 | "language_info": { 16 | "codemirror_mode": { 17 | "name": "ipython", 18 | "version": 3 19 | }, 20 | "file_extension": ".py", 21 | "mimetype": "text/x-python", 22 | "name": "python", 23 | "nbconvert_exporter": "python", 24 | "pygments_lexer": "ipython3", 25 | "version": "3.6.8" 26 | } 27 | }, 28 | "cells": [ 29 | { 30 | "cell_type": "markdown", 31 | "metadata": { 32 | "id": "view-in-github", 33 | "colab_type": "text" 34 | }, 35 | "source": [ 36 | "\"Open" 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": { 42 | "id": "QEx9dAerBzOm", 43 | "colab_type": "text" 44 | }, 45 | "source": [ 46 | "" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": { 52 | "id": "Ew82tJs67G1R", 53 | "colab_type": "text" 54 | }, 55 | "source": [ 56 | "আমাদের এই টেক্সট ফাইলটা দেখে নিতে পারেন। প্রতিটা লাইনে একটা করে বাংলা বাক্য আছে। বাংলা উইকিপিডিয়া থেকে নেয়া। প্রি-প্রসেসিং করে নিয়েছি আগেই। এখানে সহায়তা দিয়েছেন তারেক আল মুনতাসির। আমাদের এই টেক্সট ফাইলটা দেখে নিতে পারেন। প্রতিটা লাইনে একটা করে বাংলা বাক্য আছে। বাংলা উইকিপিডিয়া থেকে নেয়া। প্রি-প্রসেসিং করে নিয়েছি আগেই। এখানে সহায়তা দিয়েছেন তারেক আল মুনতাসির। আঅ" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "metadata": { 62 | "colab_type": "code", 63 | "id": "css35ZIlIesJ", 64 | "colab": {} 65 | }, 66 | "source": [ 67 | "import re\n", 68 | "import os\n", 69 | "import glob\n", 70 | "import string\n", 71 | "\n", 72 | "## দুটো মডেল দেখি 'জেনসিম' ফ্রেমওয়ার্ক দিয়ে\n", 73 | "from gensim.models import Word2Vec\n", 74 | "from gensim.models import FastText" 75 | ], 76 | "execution_count": 0, 77 | "outputs": [] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "metadata": { 82 | "id": "EeyJuXl_KAzj", 83 | "colab_type": "code", 84 | "outputId": "34082019-c63f-4a57-ebd2-d4fa9f80aac3", 85 | "colab": { 86 | "base_uri": "https://localhost:8080/", 87 | "height": 204 88 | } 89 | }, 90 | "source": [ 91 | "!wget https://media.githubusercontent.com/media/raqueeb/datasets/master/bnwiki-texts.zip" 92 | ], 93 | "execution_count": 2, 94 | "outputs": [ 95 | { 96 | "output_type": "stream", 97 | "text": [ 98 | "--2019-11-02 11:30:58-- https://media.githubusercontent.com/media/raqueeb/datasets/master/bnwiki-texts.zip\n", 99 | "Resolving media.githubusercontent.com (media.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n", 100 | "Connecting to media.githubusercontent.com (media.githubusercontent.com)|151.101.0.133|:443... connected.\n", 101 | "HTTP request sent, awaiting response... 200 OK\n", 102 | "Length: 61696513 (59M) [application/zip]\n", 103 | "Saving to: ‘bnwiki-texts.zip’\n", 104 | "\n", 105 | "\rbnwiki-texts.zip 0%[ ] 0 --.-KB/s \rbnwiki-texts.zip 25%[====> ] 14.99M 74.9MB/s \rbnwiki-texts.zip 66%[============> ] 39.30M 97.9MB/s \rbnwiki-texts.zip 100%[===================>] 58.84M 105MB/s in 0.6s \n", 106 | "\n", 107 | "2019-11-02 11:31:00 (105 MB/s) - ‘bnwiki-texts.zip’ saved [61696513/61696513]\n", 108 | "\n" 109 | ], 110 | "name": "stdout" 111 | } 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "metadata": { 117 | "id": "f0UbGF4HNDM7", 118 | "colab_type": "code", 119 | "outputId": "c0613d7f-eff0-47f3-cab0-23f49c42771a", 120 | "colab": { 121 | "base_uri": "https://localhost:8080/", 122 | "height": 119 123 | } 124 | }, 125 | "source": [ 126 | "!ls -al" 127 | ], 128 | "execution_count": 3, 129 | "outputs": [ 130 | { 131 | "output_type": "stream", 132 | "text": [ 133 | "total 60268\n", 134 | "drwxr-xr-x 1 root root 4096 Nov 2 11:30 .\n", 135 | "drwxr-xr-x 1 root root 4096 Nov 2 11:24 ..\n", 136 | "-rw-r--r-- 1 root root 61696513 Nov 2 11:30 bnwiki-texts.zip\n", 137 | "drwxr-xr-x 1 root root 4096 Oct 30 15:14 .config\n", 138 | "drwxr-xr-x 1 root root 4096 Oct 25 16:58 sample_data\n" 139 | ], 140 | "name": "stdout" 141 | } 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "metadata": { 147 | "id": "eqj1LzMqygcn", 148 | "colab_type": "code", 149 | "outputId": "d5610fef-23c4-43f7-c4c1-2b7f1285e4f4", 150 | "colab": { 151 | "base_uri": "https://localhost:8080/", 152 | "height": 51 153 | } 154 | }, 155 | "source": [ 156 | "!unzip bnwiki-texts.zip" 157 | ], 158 | "execution_count": 4, 159 | "outputs": [ 160 | { 161 | "output_type": "stream", 162 | "text": [ 163 | "Archive: bnwiki-texts.zip\n", 164 | " inflating: bnwiki-texts-preprocessed.txt \n" 165 | ], 166 | "name": "stdout" 167 | } 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "metadata": { 173 | "id": "lw0wsKOtOJhC", 174 | "colab_type": "code", 175 | "colab": {} 176 | }, 177 | "source": [ 178 | "import warnings\n", 179 | "warnings.filterwarnings(\"ignore\")" 180 | ], 181 | "execution_count": 0, 182 | "outputs": [] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "metadata": { 187 | "id": "kUES-_k9vAI2", 188 | "colab_type": "code", 189 | "colab": {} 190 | }, 191 | "source": [ 192 | "preprocessed_text_file_path = 'bnwiki-texts-preprocessed.txt'" 193 | ], 194 | "execution_count": 0, 195 | "outputs": [] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "metadata": { 200 | "id": "mmcRCrb8CbjQ", 201 | "colab_type": "text" 202 | }, 203 | "source": [ 204 | "" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "metadata": { 210 | "id": "wpbNqMMjvAI-", 211 | "colab_type": "code", 212 | "colab": {} 213 | }, 214 | "source": [ 215 | "lines_from_file = []\n", 216 | "with open(preprocessed_text_file_path, encoding='utf8') as text_file:\n", 217 | " for line in text_file:\n", 218 | " lines_from_file.append(line)" 219 | ], 220 | "execution_count": 0, 221 | "outputs": [] 222 | }, 223 | { 224 | "cell_type": "markdown", 225 | "metadata": { 226 | "id": "0629wIhUeo4U", 227 | "colab_type": "text" 228 | }, 229 | "source": [ 230 | "কতগুলো লাইন আছে এই ফাইলে?" 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "metadata": { 236 | "id": "0wD_U3zBaqnv", 237 | "colab_type": "code", 238 | "outputId": "d32165b4-9176-44d5-8839-c0b45e3011f5", 239 | "colab": { 240 | "base_uri": "https://localhost:8080/", 241 | "height": 34 242 | } 243 | }, 244 | "source": [ 245 | "len(lines_from_file)" 246 | ], 247 | "execution_count": 8, 248 | "outputs": [ 249 | { 250 | "output_type": "execute_result", 251 | "data": { 252 | "text/plain": [ 253 | "1363435" 254 | ] 255 | }, 256 | "metadata": { 257 | "tags": [] 258 | }, 259 | "execution_count": 8 260 | } 261 | ] 262 | }, 263 | { 264 | "cell_type": "markdown", 265 | "metadata": { 266 | "id": "hH9Y4ylRvAJG", 267 | "colab_type": "text" 268 | }, 269 | "source": [ 270 | "আমাদের জেনসিম ওয়ার্ড২ভেক এবং ফাস্টটেক্সট শব্দের লিস্ট আশা করে একেকটা বাক্য/প্রতিটা লাইনে। যেমন, [[\"আমি\", \"এখন\", \"বইটি\", \"পরছি\"],[\"বইটি\", \"অনেক\", \"ভাল\"]]\n" 271 | ] 272 | }, 273 | { 274 | "cell_type": "code", 275 | "metadata": { 276 | "id": "9GVytXpevAJI", 277 | "colab_type": "code", 278 | "colab": {} 279 | }, 280 | "source": [ 281 | "tokenized_lines = []\n", 282 | "for single_line in lines_from_file:\n", 283 | " tokenized_lines.append(single_line.split())" 284 | ], 285 | "execution_count": 0, 286 | "outputs": [] 287 | }, 288 | { 289 | "cell_type": "code", 290 | "metadata": { 291 | "id": "Pyfc1xS6vAJN", 292 | "colab_type": "code", 293 | "outputId": "5bfe5f72-a5c9-4828-eb36-561e91ab285b", 294 | "colab": { 295 | "base_uri": "https://localhost:8080/", 296 | "height": 34 297 | } 298 | }, 299 | "source": [ 300 | "print(len(tokenized_lines))" 301 | ], 302 | "execution_count": 10, 303 | "outputs": [ 304 | { 305 | "output_type": "stream", 306 | "text": [ 307 | "1363435\n" 308 | ], 309 | "name": "stdout" 310 | } 311 | ] 312 | }, 313 | { 314 | "cell_type": "code", 315 | "metadata": { 316 | "colab_type": "code", 317 | "id": "LOSqlC-5Ies-", 318 | "outputId": "cda2210a-8d50-4fbc-9b39-e854330d98ad", 319 | "colab": { 320 | "base_uri": "https://localhost:8080/", 321 | "height": 54 322 | } 323 | }, 324 | "source": [ 325 | "print(tokenized_lines[0])" 326 | ], 327 | "execution_count": 11, 328 | "outputs": [ 329 | { 330 | "output_type": "stream", 331 | "text": [ 332 | "['বাংলা', 'ভাষা', 'বাংলা', 'ভাষা', 'বাংলা', 'ভাষা', 'বাঙলা', 'বাঙ্গলা', 'তথা', 'বাঙ্গালা', 'নামগুলোতেও', 'পরিচিত', 'একটি', 'ইন্দো', 'আর্য', 'ভাষা', 'যা', 'দক্ষিণ', 'এশিয়ার', 'বাঙালি', 'জাতির', 'প্রধান', 'কথ্য', 'ও', 'লেখ্য', 'ভাষা']\n" 333 | ], 334 | "name": "stdout" 335 | } 336 | ] 337 | }, 338 | { 339 | "cell_type": "code", 340 | "metadata": { 341 | "colab_type": "code", 342 | "id": "qK9XVTfWIetC", 343 | "colab": {} 344 | }, 345 | "source": [ 346 | "model = Word2Vec(tokenized_lines, size=200, window=5, min_count=10)" 347 | ], 348 | "execution_count": 0, 349 | "outputs": [] 350 | }, 351 | { 352 | "cell_type": "code", 353 | "metadata": { 354 | "id": "F6MgKNt6EOzo", 355 | "colab_type": "code", 356 | "colab": {} 357 | }, 358 | "source": [ 359 | "fasttext_model = FastText(tokenized_lines, size=200, window=5, min_count=10)" 360 | ], 361 | "execution_count": 0, 362 | "outputs": [] 363 | }, 364 | { 365 | "cell_type": "code", 366 | "metadata": { 367 | "colab_type": "code", 368 | "outputId": "476a04be-5c7a-4346-fab7-a49636464d52", 369 | "id": "8MUnrU8svJT4", 370 | "colab": { 371 | "base_uri": "https://localhost:8080/", 372 | "height": 867 373 | } 374 | }, 375 | "source": [ 376 | "vector_a = fasttext_model.wv.get_vector('বাংলা')\n", 377 | "print(vector_a)" 378 | ], 379 | "execution_count": 19, 380 | "outputs": [ 381 | { 382 | "output_type": "stream", 383 | "text": [ 384 | "[ 3.10908675e+00 -2.38071442e+00 -9.70043957e-01 -5.66025078e-01\n", 385 | " -1.07619822e+00 4.47757959e+00 1.04165363e+00 -3.94767380e+00\n", 386 | " 8.38609576e-01 1.27086356e-01 1.58916414e+00 1.51394141e+00\n", 387 | " -3.70067549e+00 1.15824379e-01 -1.39869475e+00 3.59312266e-01\n", 388 | " -1.59174860e+00 -1.45606112e+00 -3.01952982e+00 -1.06670964e+00\n", 389 | " -1.29315794e+00 3.36348724e+00 7.83140123e-01 6.18322074e-01\n", 390 | " 1.14647853e+00 -2.00278902e+00 9.76834223e-02 -9.06915814e-02\n", 391 | " -1.35848308e+00 2.61401916e+00 4.11325842e-01 -4.95882053e-03\n", 392 | " -1.85620356e+00 5.86531878e-01 1.08975947e+00 -3.19484925e+00\n", 393 | " 1.23863375e+00 2.76096940e+00 2.97603616e-03 -1.30080569e+00\n", 394 | " 2.34073353e+00 -1.85653794e+00 1.02879755e-01 -3.77274960e-01\n", 395 | " -2.59784341e-01 6.55955255e-01 8.40521753e-01 -1.47725356e+00\n", 396 | " -1.72506261e+00 -7.94822931e-01 -1.88007689e+00 -2.67613363e+00\n", 397 | " -4.13702548e-01 3.10200858e+00 1.60251498e+00 -2.42116499e+00\n", 398 | " -2.17445597e-01 5.44657588e-01 -1.67769241e+00 -3.09820700e+00\n", 399 | " -1.82247496e+00 -3.44184041e-01 -1.98292565e+00 1.74476191e-01\n", 400 | " 7.13144481e-01 1.84219733e-01 -4.89389211e-01 1.22903883e+00\n", 401 | " -4.62614506e-01 8.72249544e-01 -1.02143466e+00 -7.08562076e-01\n", 402 | " 4.35238071e-02 -4.38914359e-01 -1.78393006e+00 6.14556909e-01\n", 403 | " 2.00046968e+00 -8.33699644e-01 -1.65362835e-01 1.87251043e+00\n", 404 | " 2.25374508e+00 5.84853113e-01 -3.86706561e-01 3.64042020e+00\n", 405 | " 5.63881040e-01 2.74816084e+00 3.06213784e+00 -1.27971780e+00\n", 406 | " 6.31614104e-02 -5.19928942e-03 -1.30408287e+00 -1.02523315e+00\n", 407 | " 4.31055456e-01 2.32871556e+00 -1.75728738e+00 3.65903759e+00\n", 408 | " 1.15330887e+00 2.57397246e+00 1.75699246e+00 1.51429367e+00\n", 409 | " -5.08826971e-01 6.03686869e-01 1.06674775e-01 -1.83964896e+00\n", 410 | " 5.94799995e-01 2.07831573e+00 3.67107248e+00 -2.18767881e+00\n", 411 | " 1.45486808e+00 8.22293937e-01 -2.80425692e+00 8.74112919e-02\n", 412 | " 2.61130476e+00 1.90909696e+00 -2.17753386e+00 -1.29431283e+00\n", 413 | " 1.42674744e+00 2.51345158e+00 1.60967469e+00 -1.49379475e-02\n", 414 | " 1.82418466e+00 3.22541207e-01 -6.54721975e-01 1.93768248e-01\n", 415 | " -4.59367305e-01 -5.77465892e-01 5.70271313e-01 1.34173846e+00\n", 416 | " -2.81269073e+00 -5.05183041e-01 5.47760844e-01 -2.12078497e-01\n", 417 | " -1.20465791e+00 1.72587800e+00 -8.81767571e-01 -1.85678065e+00\n", 418 | " -1.73196900e+00 9.98399138e-01 2.41938639e+00 -1.84551394e+00\n", 419 | " -9.06164587e-01 6.02274477e-01 1.45480418e+00 1.88540429e-01\n", 420 | " 4.30855453e-01 1.64853406e+00 2.47411418e+00 1.92891765e+00\n", 421 | " -2.19521308e+00 4.46682781e-01 2.29912496e+00 2.62496662e+00\n", 422 | " 1.42786920e-01 -2.22294140e+00 -1.13450503e+00 7.42880940e-01\n", 423 | " -8.45801473e-01 -9.76455331e-01 -7.41640627e-01 2.71321321e+00\n", 424 | " -7.17415869e-01 3.15599829e-01 -7.29448199e-01 6.16044462e-01\n", 425 | " 1.56039202e+00 3.18467259e-01 3.07009757e-01 1.85720837e+00\n", 426 | " 2.67635882e-01 2.57449389e-01 1.40100133e+00 -4.22247839e+00\n", 427 | " 8.93778503e-02 -1.38147458e-01 9.96620238e-01 -1.24018073e+00\n", 428 | " -3.63333583e+00 -1.43686414e+00 9.18335259e-01 1.02571118e+00\n", 429 | " -1.52965581e+00 -2.65177935e-01 -2.10557431e-01 6.07441604e-01\n", 430 | " -2.89372039e+00 -1.29234934e+00 -5.66955984e-01 1.62534082e+00\n", 431 | " 1.42871010e+00 -1.20629227e+00 1.34718075e-01 1.88303065e+00\n", 432 | " -1.41340458e+00 -1.39459598e+00 7.42102444e-01 -2.43158758e-01\n", 433 | " -3.70771098e+00 1.77078104e+00 2.13890696e+00 7.48850524e-01]\n" 434 | ], 435 | "name": "stdout" 436 | } 437 | ] 438 | }, 439 | { 440 | "cell_type": "code", 441 | "metadata": { 442 | "colab_type": "code", 443 | "id": "7ICb0qLSIetF", 444 | "colab": {} 445 | }, 446 | "source": [ 447 | "# শুরুতে ওয়ার্ড২ভেক ব্যবহার করছি, model এসেছে ওয়ার্ড২ভেক\n", 448 | "print(\"ছেলে শব্দটার সবচেয়ে কাছাকাছি শব্দ কি?\")\n", 449 | "model.wv.most_similar('ছেলে', topn=5)" 450 | ], 451 | "execution_count": 0, 452 | "outputs": [] 453 | }, 454 | { 455 | "cell_type": "markdown", 456 | "metadata": { 457 | "id": "TYxJekyefvgD", 458 | "colab_type": "text" 459 | }, 460 | "source": [ 461 | "আমার পছন্দ ফাস্টটেক্সট, দেখি তাদের রেজাল্ট। " 462 | ] 463 | }, 464 | { 465 | "cell_type": "code", 466 | "metadata": { 467 | "id": "CwFmlgLAFXwr", 468 | "colab_type": "code", 469 | "colab": {} 470 | }, 471 | "source": [ 472 | "print(\"ফাস্টটেক্সট উদাহরণ: ছেলে শব্দটার সবচেয়ে কাছাকাছি শব্দ কি?\")\n", 473 | "fasttext_model.wv.most_similar('ছেলে', topn=5)" 474 | ], 475 | "execution_count": 0, 476 | "outputs": [] 477 | }, 478 | { 479 | "cell_type": "code", 480 | "metadata": { 481 | "colab_type": "code", 482 | "id": "vlf8VWhEIetJ", 483 | "colab": {} 484 | }, 485 | "source": [ 486 | "print(\"What is Father + Girl - Boy =?\")\n", 487 | "model.wv.most_similar(positive=['বাবা', 'মেয়ে'], negative=['ছেলে'], topn=5)" 488 | ], 489 | "execution_count": 0, 490 | "outputs": [] 491 | }, 492 | { 493 | "cell_type": "code", 494 | "metadata": { 495 | "id": "K2eHIpTQG25f", 496 | "colab_type": "code", 497 | "colab": {} 498 | }, 499 | "source": [ 500 | "print(model.wv['বাংলাদেশ'])" 501 | ], 502 | "execution_count": 0, 503 | "outputs": [] 504 | }, 505 | { 506 | "cell_type": "code", 507 | "metadata": { 508 | "colab_type": "code", 509 | "id": "vJz_u8ZQIetL", 510 | "colab": {} 511 | }, 512 | "source": [ 513 | "print('এখানে কোন শব্দটা যাচ্ছে না বাকিদের সাথে?')\n", 514 | "model.wv.doesnt_match(\"ঢাকা রাজশাহী রংপুর নজরুল\".split())" 515 | ], 516 | "execution_count": 0, 517 | "outputs": [] 518 | }, 519 | { 520 | "cell_type": "code", 521 | "metadata": { 522 | "id": "Es7_9BJ2Mudn", 523 | "colab_type": "code", 524 | "colab": {} 525 | }, 526 | "source": [ 527 | "print(model.wv.similarity('শিক্ষা', 'শিক্ষিত'))" 528 | ], 529 | "execution_count": 0, 530 | "outputs": [] 531 | }, 532 | { 533 | "cell_type": "code", 534 | "metadata": { 535 | "id": "WDfvR_yUHZmY", 536 | "colab_type": "code", 537 | "colab": {} 538 | }, 539 | "source": [ 540 | "semantically_similar_words = {words: [item[0] for item in model.wv.most_similar([words], topn=5)]\n", 541 | " for words in ['বাংলা', 'মাতা', 'একুশে', 'ভাষা', 'আনন্দ', 'আকাশ']}\n", 542 | "\n", 543 | "for k,v in semantically_similar_words.items():\n", 544 | " print(k+\":\"+str(v))" 545 | ], 546 | "execution_count": 0, 547 | "outputs": [] 548 | }, 549 | { 550 | "cell_type": "code", 551 | "metadata": { 552 | "id": "tZWWvpNdIPlK", 553 | "colab_type": "code", 554 | "colab": {} 555 | }, 556 | "source": [ 557 | "semantically_similar_words = {words: [item[0] for item in fasttext_model.wv.most_similar([words], topn=5)]\n", 558 | " for words in ['বাংলা', 'মাতা', 'একুশে', 'ভাষা', 'আনন্দ', 'আকাশ']}\n", 559 | "\n", 560 | "for k,v in semantically_similar_words.items():\n", 561 | " print(k+\":\"+str(v))" 562 | ], 563 | "execution_count": 0, 564 | "outputs": [] 565 | }, 566 | { 567 | "cell_type": "code", 568 | "metadata": { 569 | "id": "aiRl4C0tljHW", 570 | "colab_type": "code", 571 | "colab": {} 572 | }, 573 | "source": [ 574 | "from gensim.models.phrases import Phrases\n", 575 | "bigram = Phrases(tokenized_lines, min_count=3, threshold=10)\n", 576 | "print(bigram[tokenized_lines[0]])" 577 | ], 578 | "execution_count": 0, 579 | "outputs": [] 580 | }, 581 | { 582 | "cell_type": "code", 583 | "metadata": { 584 | "id": "0C7-x-x6I1PO", 585 | "colab_type": "code", 586 | "colab": {} 587 | }, 588 | "source": [ 589 | "from sklearn.decomposition import PCA\n", 590 | "\n", 591 | "all_similar_words = sum([[k] + v for k, v in semantically_similar_words.items()], [])\n", 592 | "\n", 593 | "print(all_similar_words)\n", 594 | "print(type(all_similar_words))\n", 595 | "print(len(all_similar_words))" 596 | ], 597 | "execution_count": 0, 598 | "outputs": [] 599 | }, 600 | { 601 | "cell_type": "code", 602 | "metadata": { 603 | "id": "5rygCrGZJJZr", 604 | "colab_type": "code", 605 | "colab": {} 606 | }, 607 | "source": [ 608 | "word_vectors = fasttext_model.wv[all_similar_words]\n", 609 | "\n", 610 | "pca = PCA(n_components=2)\n", 611 | "\n", 612 | "p_comps = pca.fit_transform(word_vectors)\n", 613 | "word_names = all_similar_words\n", 614 | "\n", 615 | "import matplotlib.pyplot as plt\n", 616 | "plt.figure(figsize=(18, 10))\n", 617 | "plt.scatter(p_comps[:, 0], p_comps[:, 1], c='red')\n", 618 | "\n", 619 | "for word_names, x, y in zip(word_names, p_comps[:, 0], p_comps[:, 1]):\n", 620 | " plt.annotate(word_names, xy=(x+0.06, y+0.03), xytext=(0, 0), textcoords='offset points')" 621 | ], 622 | "execution_count": 0, 623 | "outputs": [] 624 | } 625 | ] 626 | } --------------------------------------------------------------------------------