├── .ipynb_checkpoints ├── Metrics-checkpoint.ipynb ├── analyze_bench-checkpoint.ipynb ├── analyze_results-checkpoint.ipynb ├── batch_transform-checkpoint.ipynb ├── benchmark-checkpoint.ipynb ├── classification_tool-checkpoint.ipynb ├── clean and prepare data-checkpoint.ipynb ├── nsfw-training_built_in_aws-checkpoint.ipynb └── nsfw_deploy-checkpoint.ipynb ├── AWS.svg ├── Capstone_Project_Report.pdf ├── LICENSE ├── Metrics.ipynb ├── Project_Proposal.pdf ├── README.md ├── analyze_bench.ipynb ├── analyze_results.ipynb ├── batch_transform.ipynb ├── benchmark.ipynb ├── classification_tool.ipynb ├── clean and prepare data.ipynb ├── demo ├── favicon.ico ├── index.html ├── nsfw.js ├── scr.sh ├── styles.css └── sun.jpg ├── lambda.py ├── nsfw-training_built_in_aws.ipynb ├── nsfw_deploy.ipynb ├── results.csv ├── results_bench.csv └── useful_scripts └── useful_scripts ├── csv_maker.py ├── example.py ├── image_dataset_handler.py ├── image_edit.py └── split_dataset.py /.ipynb_checkpoints/batch_transform-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "This notebook contains code that evaluates the model by supplying images through a batch tranform.\n", 8 | "You will need three things for this notebook to run.
\n", 9 | "1. **trainining Job Name (name of the job with which you have trained the model).**\n", 10 | "2. **url of the location in s3 where images are uploaded.**\n", 11 | "3. **url in the s3 where output is to be stored.**\n", 12 | "\n", 13 | "You can then copy them to your computer and follow analyze.results.ipynb to analyze the results.\n" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 1, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "import os\n", 23 | "\n", 24 | "import time\n", 25 | "from time import gmtime, strftime\n", 26 | "\n", 27 | "import numpy as np\n", 28 | "import pandas as pd\n", 29 | "\n", 30 | "import matplotlib.pyplot as plt\n", 31 | "\n", 32 | "from sklearn.datasets import load_boston\n", 33 | "import sklearn.model_selection\n" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 2, 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "import sagemaker\n", 43 | "from sagemaker import get_execution_role\n", 44 | "from sagemaker.amazon.amazon_estimator import get_image_uri\n", 45 | "\n", 46 | "# This is an object that represents the SageMaker session that we are currently operating in. This\n", 47 | "# object contains some useful information that we will need to access later such as our region.\n", 48 | "session = sagemaker.Session()\n", 49 | "\n", 50 | "# This is an object that represents the IAM role that we are currently assigned. When we construct\n", 51 | "# and launch the training job later we will need to tell it what IAM role it should have. Since our\n", 52 | "# use case is relatively simple we will simply assign the training job the role we currently have.\n", 53 | "role = get_execution_role()" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 3, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "container = get_image_uri(session.boto_region_name, 'image-classification', repo_version=\"latest\")" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 4, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "\n", 72 | "job_name = 'image-classification-2019-09-12-17-02-05-917'\n", 73 | "training_job_info = session.sagemaker_client.describe_training_job(TrainingJobName=job_name)\n", 74 | "\n", 75 | "model_artifacts = training_job_info['ModelArtifacts']['S3ModelArtifacts']" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 5, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "model_name = job_name + \"-model\"\n", 85 | "\n", 86 | "# We also need to tell SageMaker which container should be used for inference and where it should\n", 87 | "# retrieve the model artifacts from. In our case, the xgboost container that we used for training\n", 88 | "# can also be used for inference.\n", 89 | "primary_container = {\n", 90 | " \"Image\": container,\n", 91 | " \"ModelDataUrl\": model_artifacts\n", 92 | "}\n", 93 | "\n", 94 | "# And lastly we construct the SageMaker model\n", 95 | "model_info = session.sagemaker_client.create_model(\n", 96 | " ModelName = model_name,\n", 97 | " ExecutionRoleArn = role,\n", 98 | " PrimaryContainer = primary_container)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 6, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "transform_job_name = 'nsfwbatchtransform' + strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n", 108 | "\n", 109 | "# Now we construct the data structure which will describe the batch transform job.\n", 110 | "transform_request = \\\n", 111 | "{\n", 112 | " \"TransformJobName\": transform_job_name,\n", 113 | " \n", 114 | " # This is the name of the model that we created earlier.\n", 115 | " \"ModelName\": model_name,\n", 116 | " \n", 117 | " # This describes how many compute instances should be used at once. If you happen to be doing a very large\n", 118 | " # batch transform job it may be worth running multiple compute instances at once.\n", 119 | " \"MaxConcurrentTransforms\": 1,\n", 120 | " \n", 121 | " # This says how big each individual request sent to the model should be, at most. One of the things that\n", 122 | " # SageMaker does in the background is to split our data up into chunks so that each chunks stays under\n", 123 | " # this size limit.\n", 124 | " \"MaxPayloadInMB\": 6,\n", 125 | " \n", 126 | " # Sometimes we may want to send only a single sample to our endpoint at a time, however in this case each of\n", 127 | " # the chunks that we send should contain multiple samples of our input data.\n", 128 | " \"BatchStrategy\": \"MultiRecord\",\n", 129 | " \n", 130 | " # This next object describes where the output data should be stored. Some of the more advanced options which\n", 131 | " # we don't cover here also describe how SageMaker should collect output from various batches.\n", 132 | " \"TransformOutput\": {\n", 133 | " \"S3OutputPath\": \"s3://project-completion-udacity/nsfw_dataset/batch-transform/\"\n", 134 | " },\n", 135 | " \n", 136 | " # Here we describe our input data. Of course, we need to tell SageMaker where on S3 our input data is stored, in\n", 137 | " # addition we need to detail the characteristics of our input data. In particular, since SageMaker may need to\n", 138 | " # split our data up into chunks, it needs to know how the individual samples in our data file appear. In our\n", 139 | " # case each line is its own sample and so we set the split type to 'line'. We also need to tell SageMaker what\n", 140 | " # type of data is being sent, in this case csv, so that it can properly serialize the data.\n", 141 | " \"TransformInput\": {\n", 142 | " \"ContentType\": \"application/x-image\",\n", 143 | " \"SplitType\": \"None\",\n", 144 | " \"DataSource\": {\n", 145 | " \"S3DataSource\": {\n", 146 | " \"S3DataType\": \"S3Prefix\",\n", 147 | " \"S3Uri\": 's3://project-completion-udacity/evaluation/testing/',\n", 148 | " }\n", 149 | " }\n", 150 | " },\n", 151 | " \n", 152 | " # And lastly we tell SageMaker what sort of compute instance we would like it to use.\n", 153 | " \"TransformResources\": {\n", 154 | " \"InstanceType\": \"ml.m4.xlarge\",\n", 155 | " \"InstanceCount\": 1\n", 156 | " }\n", 157 | "}" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 7, 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [ 166 | "transform_response = session.sagemaker_client.create_transform_job(**transform_request)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": {}, 173 | "outputs": [ 174 | { 175 | "name": "stdout", 176 | "output_type": "stream", 177 | "text": [ 178 | "..........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................." 179 | ] 180 | } 181 | ], 182 | "source": [ 183 | "transform_desc = session.wait_for_transform_job(transform_job_name)" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": null, 189 | "metadata": {}, 190 | "outputs": [], 191 | "source": [] 192 | } 193 | ], 194 | "metadata": { 195 | "kernelspec": { 196 | "display_name": "Python 3", 197 | "language": "python", 198 | "name": "python3" 199 | }, 200 | "language_info": { 201 | "codemirror_mode": { 202 | "name": "ipython", 203 | "version": 3 204 | }, 205 | "file_extension": ".py", 206 | "mimetype": "text/x-python", 207 | "name": "python", 208 | "nbconvert_exporter": "python", 209 | "pygments_lexer": "ipython3", 210 | "version": "3.5.2" 211 | } 212 | }, 213 | "nbformat": 4, 214 | "nbformat_minor": 2 215 | } 216 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/benchmark-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "All the procedure is same as batch-transform.ipynb . see batch-transform.ipynb." 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | " **for doing a batch transform on the dataset from [benchmark](https://towardsdatascience.com/comparison-of-the-best-nsfw-image-moderation-apis-2018-84be8da65303)**" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import os\n", 24 | "\n", 25 | "import time\n", 26 | "from time import gmtime, strftime\n", 27 | "\n", 28 | "import numpy as np\n", 29 | "import pandas as pd\n", 30 | "\n", 31 | "import matplotlib.pyplot as plt\n", 32 | "\n", 33 | "from sklearn.datasets import load_boston\n", 34 | "import sklearn.model_selection\n" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 2, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "import sagemaker\n", 44 | "from sagemaker import get_execution_role\n", 45 | "from sagemaker.amazon.amazon_estimator import get_image_uri\n", 46 | "\n", 47 | "# This is an object that represents the SageMaker session that we are currently operating in. This\n", 48 | "# object contains some useful information that we will need to access later such as our region.\n", 49 | "session = sagemaker.Session()\n", 50 | "\n", 51 | "# This is an object that represents the IAM role that we are currently assigned. When we construct\n", 52 | "# and launch the training job later we will need to tell it what IAM role it should have. Since our\n", 53 | "# use case is relatively simple we will simply assign the training job the role we currently have.\n", 54 | "role = get_execution_role()" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 3, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "container = get_image_uri(session.boto_region_name, 'image-classification', repo_version=\"latest\")" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 4, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "\n", 73 | "job_name = 'image-classification-2019-09-12-17-02-05-917'\n", 74 | "training_job_info = session.sagemaker_client.describe_training_job(TrainingJobName=job_name)\n", 75 | "\n", 76 | "model_artifacts = training_job_info['ModelArtifacts']['S3ModelArtifacts']" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 5, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "ename": "ClientError", 86 | "evalue": "An error occurred (ValidationException) when calling the CreateModel operation: Cannot create already existing model \"arn:aws:sagemaker:us-east-2:733184320490:model/image-classification-2019-09-12-17-02-05-917-model\".", 87 | "output_type": "error", 88 | "traceback": [ 89 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 90 | "\u001b[0;31mClientError\u001b[0m Traceback (most recent call last)", 91 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0mModelName\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0mExecutionRoleArn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrole\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m PrimaryContainer = primary_container)\n\u001b[0m", 92 | "\u001b[0;32m~/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 355\u001b[0m \"%s() only accepts keyword arguments.\" % py_operation_name)\n\u001b[1;32m 356\u001b[0m \u001b[0;31m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_api_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 358\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 359\u001b[0m \u001b[0m_api_call\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpy_operation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 93 | "\u001b[0;32m~/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 659\u001b[0m \u001b[0merror_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Error\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Code\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 660\u001b[0m \u001b[0merror_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 661\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0merror_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparsed_response\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 662\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 663\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 94 | "\u001b[0;31mClientError\u001b[0m: An error occurred (ValidationException) when calling the CreateModel operation: Cannot create already existing model \"arn:aws:sagemaker:us-east-2:733184320490:model/image-classification-2019-09-12-17-02-05-917-model\"." 95 | ] 96 | } 97 | ], 98 | "source": [ 99 | "model_name = job_name + \"-model\"\n", 100 | "\n", 101 | "# We also need to tell SageMaker which container should be used for inference and where it should\n", 102 | "# retrieve the model artifacts from. In our case, the xgboost container that we used for training\n", 103 | "# can also be used for inference.\n", 104 | "primary_container = {\n", 105 | " \"Image\": container,\n", 106 | " \"ModelDataUrl\": model_artifacts\n", 107 | "}\n", 108 | "\n", 109 | "# And lastly we construct the SageMaker model\n", 110 | "model_info = session.sagemaker_client.create_model(\n", 111 | " ModelName = model_name,\n", 112 | " ExecutionRoleArn = role,\n", 113 | " PrimaryContainer = primary_container)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 6, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "transform_job_name = 'nsfwbatchtransform' + strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n", 123 | "\n", 124 | "# Now we construct the data structure which will describe the batch transform job.\n", 125 | "transform_request = \\\n", 126 | "{\n", 127 | " \"TransformJobName\": transform_job_name,\n", 128 | " \n", 129 | " # This is the name of the model that we created earlier.\n", 130 | " \"ModelName\": model_name,\n", 131 | " \n", 132 | " # This describes how many compute instances should be used at once. If you happen to be doing a very large\n", 133 | " # batch transform job it may be worth running multiple compute instances at once.\n", 134 | " \"MaxConcurrentTransforms\": 1,\n", 135 | " \n", 136 | " # This says how big each individual request sent to the model should be, at most. One of the things that\n", 137 | " # SageMaker does in the background is to split our data up into chunks so that each chunks stays under\n", 138 | " # this size limit.\n", 139 | " \"MaxPayloadInMB\": 6,\n", 140 | " \n", 141 | " # Sometimes we may want to send only a single sample to our endpoint at a time, however in this case each of\n", 142 | " # the chunks that we send should contain multiple samples of our input data.\n", 143 | " \"BatchStrategy\": \"MultiRecord\",\n", 144 | " \n", 145 | " # This next object describes where the output data should be stored. Some of the more advanced options which\n", 146 | " # we don't cover here also describe how SageMaker should collect output from various batches.\n", 147 | " \"TransformOutput\": {\n", 148 | " \"S3OutputPath\": \"s3://project-completion-udacity/nsfw_dataset/batch-transform_bench/\"\n", 149 | " },\n", 150 | " \n", 151 | " # Here we describe our input data. Of course, we need to tell SageMaker where on S3 our input data is stored, in\n", 152 | " # addition we need to detail the characteristics of our input data. In particular, since SageMaker may need to\n", 153 | " # split our data up into chunks, it needs to know how the individual samples in our data file appear. In our\n", 154 | " # case each line is its own sample and so we set the split type to 'line'. We also need to tell SageMaker what\n", 155 | " # type of data is being sent, in this case csv, so that it can properly serialize the data.\n", 156 | " \"TransformInput\": {\n", 157 | " \"ContentType\": \"application/x-image\",\n", 158 | " \"SplitType\": \"None\",\n", 159 | " \"DataSource\": {\n", 160 | " \"S3DataSource\": {\n", 161 | " \"S3DataType\": \"S3Prefix\",\n", 162 | " \"S3Uri\": 's3://project-completion-udacity/bench/',\n", 163 | " }\n", 164 | " }\n", 165 | " },\n", 166 | " \n", 167 | " # And lastly we tell SageMaker what sort of compute instance we would like it to use.\n", 168 | " \"TransformResources\": {\n", 169 | " \"InstanceType\": \"ml.m4.xlarge\",\n", 170 | " \"InstanceCount\": 1\n", 171 | " }\n", 172 | "}" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 7, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "transform_response = session.sagemaker_client.create_transform_job(**transform_request)" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": 8, 187 | "metadata": {}, 188 | "outputs": [ 189 | { 190 | "name": "stdout", 191 | "output_type": "stream", 192 | "text": [ 193 | "..............................................................!\n" 194 | ] 195 | } 196 | ], 197 | "source": [ 198 | "transform_desc = session.wait_for_transform_job(transform_job_name)" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "metadata": {}, 205 | "outputs": [], 206 | "source": [] 207 | } 208 | ], 209 | "metadata": { 210 | "kernelspec": { 211 | "display_name": "Python 3", 212 | "language": "python", 213 | "name": "python3" 214 | }, 215 | "language_info": { 216 | "codemirror_mode": { 217 | "name": "ipython", 218 | "version": 3 219 | }, 220 | "file_extension": ".py", 221 | "mimetype": "text/x-python", 222 | "name": "python", 223 | "nbconvert_exporter": "python", 224 | "pygments_lexer": "ipython3", 225 | "version": "3.5.2" 226 | } 227 | }, 228 | "nbformat": 4, 229 | "nbformat_minor": 2 230 | } 231 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/classification_tool-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

Put all the images in pics folder and run the cell below and it will make your bulk classification task simple." 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "\n", 17 | "\n", 18 | "import sys, termios, tty, os, time\n", 19 | "#import autopy\n", 20 | "import ipywidgets as widgets\n", 21 | "current_directory = os.getcwd()\n", 22 | "try :\n", 23 | " path_nude = os.path.join(current_directory , 'Nude')\n", 24 | " os.makedirs(path_nude)\n", 25 | "except FileExistsError as e :\n", 26 | " pass\n", 27 | "\n", 28 | "try :\n", 29 | " path_animated = os.path.join(current_directory , 'Animated')\n", 30 | " os.makedirs(path_animated)\n", 31 | "except FileExistsError as e :\n", 32 | " pass\n", 33 | "\n", 34 | "try :\n", 35 | " path_porn = os.path.join(current_directory , 'Porn')\n", 36 | " os.makedirs(path_porn)\n", 37 | "except FileExistsError as e :\n", 38 | " pass\n", 39 | "\n", 40 | "import glob , pickle\n", 41 | "datadir = 'pics'\n", 42 | "imgs = glob.glob(os.path.join(datadir, '*.jpg'))\n", 43 | "print(len(imgs))\n", 44 | "\n", 45 | "import pickle as pickle\n", 46 | "def save_obj(obj, name ):\n", 47 | " with open(name + '.pkl', 'wb') as f:\n", 48 | " pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n", 49 | "\n", 50 | "def load_obj(name ):\n", 51 | " with open(name + '.pkl', 'rb') as f:\n", 52 | " return pickle.load(f) \n", 53 | " \n", 54 | "try:\n", 55 | " lookup = load_obj('work_done')\n", 56 | "except (OSError, IOError) as e:\n", 57 | " lookup = {}\n", 58 | " for img in imgs:\n", 59 | " lookup[img] = 0\n", 60 | " save_obj( lookup , 'work_done')\n", 61 | "\n", 62 | "import matplotlib.pyplot as plt\n", 63 | "plt.ion()\n", 64 | "#from PIL import Image\n", 65 | "from IPython.display import Image\n", 66 | "from IPython.display import clear_output\n", 67 | "import shutil\n", 68 | "\n", 69 | "lookup = load_obj('work_done')\n", 70 | "def on_nude_Button(b):\n", 71 | " global i\n", 72 | " global lookup\n", 73 | " global imgs\n", 74 | " #plt.close()\n", 75 | " if i < len(imgs) and lookup[imgs[i]] == 0:\n", 76 | " clear_output()\n", 77 | " lookup[imgs[i]] = 1\n", 78 | " shutil.move(imgs[i],path_nude)\n", 79 | " i += 1\n", 80 | " display(Image(imgs[i]))\n", 81 | " display(hb)\n", 82 | " \n", 83 | "# im = Image.open( imgs[i]) \n", 84 | "# plt.imshow(im)\n", 85 | " \n", 86 | " print('nude_clicked')\n", 87 | " \n", 88 | "def on_Porn_Button(b):\n", 89 | " global i\n", 90 | " global lookup\n", 91 | " global imgs\n", 92 | " if i < len(imgs) and lookup[imgs[i]] == 0 :\n", 93 | " clear_output()\n", 94 | " lookup[imgs[i]] = 1\n", 95 | " shutil.move(imgs[i],path_porn)\n", 96 | " i += 1\n", 97 | " display(Image(imgs[i]))\n", 98 | " display(hb)\n", 99 | " \n", 100 | " print('porn_clicked')\n", 101 | " \n", 102 | "def on_animated_Button(b):\n", 103 | " global i\n", 104 | " global lookup\n", 105 | " global imgs\n", 106 | " if i < len(imgs) and lookup[imgs[i]] == 0:\n", 107 | " clear_output()\n", 108 | " lookup[imgs[i]] = 1\n", 109 | " shutil.move(imgs[i],path_animated)\n", 110 | " i += 1\n", 111 | " display(Image(imgs[i]))\n", 112 | " display(hb)\n", 113 | " \n", 114 | " print('animated_clicked')\n", 115 | " \n", 116 | "def on_Quit_Button(b):\n", 117 | " print('Quit_clicked')\n", 118 | " save_obj(lookup , 'work_done')\n", 119 | " clear_output()\n", 120 | "\n", 121 | "Nude_button = widgets.Button(description= \"Nude\")\n", 122 | "Porn_button = widgets.Button(description= \"Porn\")\n", 123 | "Animated_button = widgets.Button(description= \"Animated\")\n", 124 | "Quit_Button = widgets.Button(description= \"Quit\")\n", 125 | "\n", 126 | "Nude_button.on_click(on_nude_Button)\n", 127 | "Porn_button.on_click(on_Porn_Button)\n", 128 | "Animated_button.on_click(on_animated_Button)\n", 129 | "Quit_Button.on_click(on_Quit_Button)\n", 130 | "\n", 131 | "\n", 132 | "hb= widgets.HBox([ Nude_button , Porn_button , Animated_button , Quit_Button ])\n", 133 | "\n", 134 | "\n", 135 | "i = 0 \n", 136 | "if i < len(imgs):\n", 137 | " display( Image(imgs[i]) )\n", 138 | " display(hb)\n", 139 | "\n", 140 | " \n", 141 | "\n", 142 | "\n", 143 | "\n", 144 | "\n", 145 | "\n", 146 | " \n", 147 | " " 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [] 156 | } 157 | ], 158 | "metadata": { 159 | "kernelspec": { 160 | "display_name": "Python 3", 161 | "language": "python", 162 | "name": "python3" 163 | }, 164 | "language_info": { 165 | "codemirror_mode": { 166 | "name": "ipython", 167 | "version": 3 168 | }, 169 | "file_extension": ".py", 170 | "mimetype": "text/x-python", 171 | "name": "python", 172 | "nbconvert_exporter": "python", 173 | "pygments_lexer": "ipython3", 174 | "version": "3.5.2" 175 | } 176 | }, 177 | "nbformat": 4, 178 | "nbformat_minor": 2 179 | } 180 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/clean and prepare data-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## As discussed in the report we need to discard bad examples. " 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import os\n", 17 | "import PIL\n", 18 | "from PIL import Image\n", 19 | "import glob\n", 20 | "import shutil\n", 21 | "\n", 22 | "def check_and_discard_data(data_dir , dump_dir):\n", 23 | " for class_name in os.listdir(data_dir):\n", 24 | " list_of_files_of_single_class = glob.glob(os.path.join(data_dir , class_name , '*') ) \n", 25 | " for file_name in list_of_files_of_single_class:\n", 26 | " if file_name.lower().endswith(( '.jpg', '.jpeg')):\n", 27 | " try:\n", 28 | " img=Image.open(file_name)\n", 29 | " img.verify()\n", 30 | " height , width = img.size\n", 31 | " if height < 80 or width < 80:\n", 32 | " print('dim less')\n", 33 | " shutil.move(file_name, dump_dir)\n", 34 | " except(IOError,SyntaxError)as e:\n", 35 | " print('io error')\n", 36 | " shutil.move(file_name, dump_dir)\n", 37 | " else:\n", 38 | " print('print other format')\n", 39 | " shutil.move( file_name , dump_dir)" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 1, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "check_and_discard_data('nsfw_datset' , 'dump')" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 4, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "data": { 58 | "text/plain": [ 59 | "16864" 60 | ] 61 | }, 62 | "execution_count": 4, 63 | "metadata": {}, 64 | "output_type": "execute_result" 65 | } 66 | ], 67 | "source": [ 68 | "len(os.listdir('dump'))" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "metadata": {}, 74 | "source": [ 75 | "## Spliting the datset into train , test , validation" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 2, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mos\u001b[39;49;00m\r\n", 88 | "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mnumpy\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mnp\u001b[39;49;00m\r\n", 89 | "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mshutil\u001b[39;49;00m\r\n", 90 | "\r\n", 91 | "\u001b[37m# # Creating Train / Val / Test folders (One time use)\u001b[39;49;00m\r\n", 92 | "\r\n", 93 | "\u001b[34mdef\u001b[39;49;00m \u001b[32msplit_dataset_one_time\u001b[39;49;00m(root_dir , valid_size, test_size , train_path , valid_path , test_path):\r\n", 94 | " \r\n", 95 | "\r\n", 96 | " class_names = os.listdir(root_dir)\r\n", 97 | "\r\n", 98 | " \u001b[37m#print(class_names)\u001b[39;49;00m\r\n", 99 | " \u001b[34mfor\u001b[39;49;00m single_class \u001b[35min\u001b[39;49;00m class_names:\r\n", 100 | " \u001b[34mtry\u001b[39;49;00m :\r\n", 101 | "\r\n", 102 | "\r\n", 103 | " os.makedirs(os.path.join( train_path , single_class))\r\n", 104 | " os.makedirs(os.path.join( valid_path , single_class))\r\n", 105 | " os.makedirs(os.path.join( test_path , single_class))\r\n", 106 | "\r\n", 107 | "\r\n", 108 | " train_single_path = os.path.join( train_path , single_class)\r\n", 109 | " test_single_path = os.path.join( test_path , single_class)\r\n", 110 | " valid_single_path = os.path.join( valid_path , single_class)\r\n", 111 | " \u001b[34mexcept\u001b[39;49;00m FileExistsError:\r\n", 112 | " \u001b[34mpass\u001b[39;49;00m\r\n", 113 | " \r\n", 114 | " single_full_list = os.listdir(os.path.join(root_dir , single_class))\r\n", 115 | " \u001b[37m#print(single_full_list[0])\u001b[39;49;00m\r\n", 116 | "\r\n", 117 | " num_examples_per_class = \u001b[36mlen\u001b[39;49;00m(single_full_list)\r\n", 118 | " indices = \u001b[36mlist\u001b[39;49;00m(\u001b[36mrange\u001b[39;49;00m(num_examples_per_class))\r\n", 119 | " split_valid = \u001b[36mint\u001b[39;49;00m(np.floor(valid_size * num_examples_per_class))\r\n", 120 | " split_test = \u001b[36mint\u001b[39;49;00m(np.floor(test_size * num_examples_per_class)) + split_valid\r\n", 121 | " np.random.shuffle(indices)\r\n", 122 | " train_idx, valid_idx , test_idx = indices[split_test:], indices[:split_valid] , indices[split_valid:split_test]\r\n", 123 | "\r\n", 124 | " \u001b[37m#print(train_idx[:5])\u001b[39;49;00m\r\n", 125 | " single_full_list_np = np.array(single_full_list)\r\n", 126 | " train_list , valid_list , test_list = single_full_list_np[train_idx].tolist() , single_full_list_np[valid_idx].tolist() , single_full_list_np[test_idx].tolist()\r\n", 127 | "\r\n", 128 | " \u001b[37m#print(len(train_list) , len(valid_list) , len(test_list) , len(single_full_list))\u001b[39;49;00m\r\n", 129 | "\r\n", 130 | "\r\n", 131 | "\r\n", 132 | " \u001b[34mfor\u001b[39;49;00m name \u001b[35min\u001b[39;49;00m train_list:\r\n", 133 | " \u001b[37m#print(os.path.join(root_dir , single_class , name) , train_single_path)\u001b[39;49;00m\r\n", 134 | " shutil.move( os.path.join(root_dir , single_class , name) , train_single_path)\r\n", 135 | "\r\n", 136 | " \u001b[34mfor\u001b[39;49;00m name \u001b[35min\u001b[39;49;00m valid_list:\r\n", 137 | " \u001b[37m#print(os.path.join(root_dir , single_class , name) , valid_single_path)\u001b[39;49;00m\r\n", 138 | " shutil.move(os.path.join(root_dir , single_class , name) ,valid_single_path)\r\n", 139 | "\r\n", 140 | " \u001b[34mfor\u001b[39;49;00m name \u001b[35min\u001b[39;49;00m test_list:\r\n", 141 | " \u001b[37m#print(os.path.join(root_dir , single_class , name) , test_single_path)\u001b[39;49;00m\r\n", 142 | " shutil.move(os.path.join(root_dir , single_class , name), test_single_path)\r\n" 143 | ] 144 | } 145 | ], 146 | "source": [ 147 | "!pygmentize useful_scripts/useful_scripts/split_dataset.py" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 2, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "from useful_scripts.useful_scripts.split_dataset import split_dataset_one_time" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 3, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "split_dataset_one_time('nsfw_datset' , 0.075 ,0.075 , 'nsfw_dataset_train' , 'nsfw_dataset_valid' , 'nsfw_dataset_test')" 166 | ] 167 | }, 168 | { 169 | "cell_type": "markdown", 170 | "metadata": {}, 171 | "source": [ 172 | "## Setting the im2rec tool in the machine. " 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 1, 178 | "metadata": {}, 179 | "outputs": [ 180 | { 181 | "name": "stdout", 182 | "output_type": "stream", 183 | "text": [ 184 | "env: IM2REC=/home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages/mxnet/tools/im2rec.py\n" 185 | ] 186 | } 187 | ], 188 | "source": [ 189 | "import sys,os\n", 190 | "\n", 191 | "suffix='/mxnet/tools/im2rec.py'\n", 192 | "im2rec = list(filter( (lambda x: os.path.isfile(x + suffix )), sys.path))[0] + suffix\n", 193 | "%env IM2REC=$im2rec" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": {}, 199 | "source": [ 200 | "## We need to create .lst file before creating .rec file Notice That test_ratio is kept at 0.0 because we have already split the dataset. Also The --resize is set 224 because image classification algorithm accepts image size (224,224)." 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": 2, 206 | "metadata": {}, 207 | "outputs": [ 208 | { 209 | "name": "stdout", 210 | "output_type": "stream", 211 | "text": [ 212 | "Creating LST files\n", 213 | "Label classes:\n", 214 | "animated 0\n", 215 | "nude 1\n", 216 | "porn 2\n", 217 | "safe_for_work 3\n", 218 | "semi_nude 4\n", 219 | "Creating RecordIO files\n", 220 | "Creating .rec file from /home/ec2-user/SageMaker/nsfw_dataset_valid.lst in /home/ec2-user/SageMaker\n", 221 | "time: 0.3074500560760498 count: 0\n", 222 | "time: 28.615877151489258 count: 1000\n", 223 | "time: 29.252779245376587 count: 2000\n", 224 | "time: 27.263898849487305 count: 3000\n", 225 | "time: 24.09006953239441 count: 4000\n", 226 | "time: 24.43202519416809 count: 5000\n", 227 | "time: 23.392404556274414 count: 6000\n", 228 | "time: 22.804959058761597 count: 7000\n", 229 | "time: 23.166723251342773 count: 8000\n", 230 | "time: 21.34347176551819 count: 9000\n", 231 | "time: 20.13561248779297 count: 10000\n", 232 | "time: 20.357985973358154 count: 11000\n", 233 | "-rw-rw-r-- 1 ec2-user ec2-user 387M Sep 11 18:49 nsfw_dataset_test.rec\n", 234 | "-rw-rw-r-- 1 ec2-user ec2-user 318M Sep 12 07:16 nsfw_dataset_valid.rec\n" 235 | ] 236 | } 237 | ], 238 | "source": [ 239 | "%%bash\n", 240 | "echo \"Creating LST files\"\n", 241 | "python $IM2REC --list --recursive --pass-through --test-ratio=0.0 --train-ratio=1.0 nsfw_dataset_valid nsfw_dataset_valid > nsfw_dataset_valid_classes\n", 242 | "\n", 243 | "echo \"Label classes:\"\n", 244 | "cat nsfw_dataset_valid_classes\n", 245 | "\n", 246 | "echo \"Creating RecordIO files\"\n", 247 | "python $IM2REC --num-thread=4 nsfw_dataset_valid.lst nsfw_dataset_valid/ --resize=224\n", 248 | "#python $IM2REC --num-thread=4 ${DATASET_NAME}_test.lst $DATASET_NAME\n", 249 | "ls -lh *.rec" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "metadata": {}, 255 | "source": [ 256 | "## upload the .rec file to s3." 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 3, 262 | "metadata": {}, 263 | "outputs": [ 264 | { 265 | "name": "stdout", 266 | "output_type": "stream", 267 | "text": [ 268 | "upload: ./nsfw_dataset_valid.rec to s3://project-completion-udacity/nsfw_dataset/validation/nsfw_dataset_valid.rec\n" 269 | ] 270 | } 271 | ], 272 | "source": [ 273 | "!aws s3 cp nsfw_dataset_valid.rec s3://project-completion-udacity/nsfw_dataset/validation/" 274 | ] 275 | }, 276 | { 277 | "cell_type": "markdown", 278 | "metadata": {}, 279 | "source": [ 280 | "## We did for validation. Now doing for testing." 281 | ] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": 8, 286 | "metadata": {}, 287 | "outputs": [ 288 | { 289 | "name": "stdout", 290 | "output_type": "stream", 291 | "text": [ 292 | "Creating LST files\n", 293 | "Label classes:\n", 294 | "animated 0\n", 295 | "nude 1\n", 296 | "porn 2\n", 297 | "safe_for_work 3\n", 298 | "semi_nude 4\n", 299 | "Creating RecordIO files\n", 300 | "Creating .rec file from /home/ec2-user/SageMaker/nsfw_dataset_test.lst in /home/ec2-user/SageMaker\n", 301 | "time: 0.05958127975463867 count: 0\n", 302 | "time: 9.245877742767334 count: 1000\n", 303 | "time: 7.712120532989502 count: 2000\n", 304 | "time: 7.699880361557007 count: 3000\n", 305 | "time: 9.303281545639038 count: 4000\n", 306 | "time: 9.592679262161255 count: 5000\n", 307 | "time: 8.724056959152222 count: 6000\n", 308 | "time: 8.638715028762817 count: 7000\n", 309 | "time: 10.148676872253418 count: 8000\n", 310 | "time: 7.879869699478149 count: 9000\n", 311 | "time: 7.991632699966431 count: 10000\n", 312 | "time: 8.284252405166626 count: 11000\n", 313 | "-rw-rw-r-- 1 ec2-user ec2-user 387M Sep 11 18:49 nsfw_dataset_test.rec\n", 314 | "-rw-rw-r-- 1 ec2-user ec2-user 388M Sep 11 18:46 nsfw_dataset_valid.rec\n" 315 | ] 316 | } 317 | ], 318 | "source": [ 319 | "%%bash\n", 320 | "echo \"Creating LST files\"\n", 321 | "python $IM2REC --list --recursive --pass-through --test-ratio=0.0 --train-ratio=1.0 nsfw_dataset_test nsfw_dataset_test > nsfw_dataset_test_classes\n", 322 | "\n", 323 | "echo \"Label classes:\"\n", 324 | "cat nsfw_dataset_test_classes\n", 325 | "\n", 326 | "echo \"Creating RecordIO files\"\n", 327 | "python $IM2REC --num-thread=4 nsfw_dataset_test.lst nsfw_dataset_test/ --resize=256\n", 328 | "#python $IM2REC --num-thread=4 ${DATASET_NAME}_test.lst $DATASET_NAME\n", 329 | "ls -lh *.rec" 330 | ] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "execution_count": 9, 335 | "metadata": {}, 336 | "outputs": [ 337 | { 338 | "name": "stdout", 339 | "output_type": "stream", 340 | "text": [ 341 | "upload: ./nsfw_dataset_test.rec to s3://project-completion-udacity/nsfw_dataset/testing/nsfw_dataset_test.rec\n" 342 | ] 343 | } 344 | ], 345 | "source": [ 346 | "!aws s3 cp nsfw_dataset_test.rec s3://project-completion-udacity/nsfw_dataset/testing/" 347 | ] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "metadata": {}, 352 | "source": [ 353 | "## Similarly doing this training ." 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": 4, 359 | "metadata": {}, 360 | "outputs": [ 361 | { 362 | "name": "stdout", 363 | "output_type": "stream", 364 | "text": [ 365 | "Creating LST files\n", 366 | "Label classes:\n", 367 | "animated 0\n", 368 | "nude 1\n", 369 | "porn 2\n", 370 | "safe_for_work 3\n", 371 | "semi_nude 4\n", 372 | "Creating RecordIO files\n", 373 | "Creating .rec file from /home/ec2-user/SageMaker/nsfw_dataset_train.lst in /home/ec2-user/SageMaker\n", 374 | "imread read blank (None) image for file: /home/ec2-user/SageMaker/nsfw_dataset_train/porn/le8LkPs.jpg\n", 375 | "time: 0.046601057052612305 count: 0\n", 376 | "time: 18.696394205093384 count: 1000\n", 377 | "time: 17.1350200176239 count: 2000\n", 378 | "time: 17.429579734802246 count: 3000\n", 379 | "time: 15.988012313842773 count: 4000\n", 380 | "time: 15.514463901519775 count: 5000\n", 381 | "time: 16.88562774658203 count: 6000\n", 382 | "time: 14.923904657363892 count: 7000\n", 383 | "time: 15.456107378005981 count: 8000\n", 384 | "time: 14.62105655670166 count: 9000\n", 385 | "time: 15.861175060272217 count: 10000\n", 386 | "time: 12.652273893356323 count: 11000\n", 387 | "time: 12.256561517715454 count: 12000\n", 388 | "time: 15.415345668792725 count: 13000\n", 389 | "time: 12.36422848701477 count: 14000\n", 390 | "time: 13.456058979034424 count: 15000\n", 391 | "time: 12.073819398880005 count: 16000\n", 392 | "time: 12.335830688476562 count: 17000\n", 393 | "time: 11.697640180587769 count: 18000\n", 394 | "time: 12.603720903396606 count: 19000\n", 395 | "time: 12.044790506362915 count: 20000\n", 396 | "time: 10.525394201278687 count: 21000\n", 397 | "time: 10.050683975219727 count: 22000\n", 398 | "time: 11.899126768112183 count: 23000\n", 399 | "time: 11.846455335617065 count: 24000\n", 400 | "time: 10.586645364761353 count: 25000\n", 401 | "time: 12.133596897125244 count: 26000\n", 402 | "time: 9.788665056228638 count: 27000\n", 403 | "time: 8.77630352973938 count: 28000\n", 404 | "time: 11.108652353286743 count: 29000\n", 405 | "time: 9.29899525642395 count: 30000\n", 406 | "time: 8.131813526153564 count: 31000\n", 407 | "time: 8.894354820251465 count: 32000\n", 408 | "time: 12.37432050704956 count: 33000\n", 409 | "time: 7.808954477310181 count: 34000\n", 410 | "time: 7.941628694534302 count: 35000\n", 411 | "time: 10.267624855041504 count: 36000\n", 412 | "time: 8.453442811965942 count: 37000\n", 413 | "time: 10.05301833152771 count: 38000\n", 414 | "time: 8.793598651885986 count: 39000\n", 415 | "time: 10.400083780288696 count: 40000\n", 416 | "time: 7.682428359985352 count: 41000\n", 417 | "time: 8.213168621063232 count: 42000\n", 418 | "time: 11.412482500076294 count: 43000\n", 419 | "time: 8.707566261291504 count: 44000\n", 420 | "time: 8.02009630203247 count: 45000\n", 421 | "time: 8.045879125595093 count: 46000\n", 422 | "time: 9.501078844070435 count: 47000\n", 423 | "time: 7.770918846130371 count: 48000\n", 424 | "time: 8.169719696044922 count: 49000\n", 425 | "time: 8.926843404769897 count: 50000\n", 426 | "time: 9.74822998046875 count: 51000\n", 427 | "time: 6.555929183959961 count: 52000\n", 428 | "time: 8.067629337310791 count: 53000\n", 429 | "time: 8.084633827209473 count: 54000\n", 430 | "time: 10.004342794418335 count: 55000\n", 431 | "time: 7.856613397598267 count: 56000\n", 432 | "time: 8.123029708862305 count: 57000\n", 433 | "time: 8.658063650131226 count: 58000\n", 434 | "time: 9.630893230438232 count: 59000\n", 435 | "time: 7.653128623962402 count: 60000\n", 436 | "time: 8.164822578430176 count: 61000\n", 437 | "time: 8.797566413879395 count: 62000\n", 438 | "time: 8.291775941848755 count: 63000\n", 439 | "time: 8.04925537109375 count: 64000\n", 440 | "time: 8.506189346313477 count: 65000\n", 441 | "time: 10.95771312713623 count: 66000\n", 442 | "time: 7.351006031036377 count: 67000\n", 443 | "time: 7.488478899002075 count: 68000\n", 444 | "time: 9.657930850982666 count: 69000\n", 445 | "time: 10.078916072845459 count: 70000\n", 446 | "time: 7.4690022468566895 count: 71000\n", 447 | "time: 8.133487224578857 count: 72000\n", 448 | "time: 8.533123970031738 count: 73000\n", 449 | "time: 9.47664999961853 count: 74000\n", 450 | "time: 7.935559034347534 count: 75000\n", 451 | "time: 7.936443567276001 count: 76000\n", 452 | "time: 7.421195030212402 count: 77000\n", 453 | "time: 10.084851026535034 count: 78000\n", 454 | "time: 7.899816274642944 count: 79000\n", 455 | "time: 7.024080038070679 count: 80000\n", 456 | "time: 7.249448776245117 count: 81000\n", 457 | "time: 9.525150060653687 count: 82000\n", 458 | "time: 7.0800535678863525 count: 83000\n", 459 | "time: 7.291053056716919 count: 84000\n", 460 | "time: 7.127345085144043 count: 85000\n", 461 | "time: 11.072758436203003 count: 86000\n", 462 | "time: 8.476456880569458 count: 87000\n", 463 | "time: 6.555356502532959 count: 88000\n", 464 | "time: 7.437634706497192 count: 89000\n", 465 | "time: 9.70185112953186 count: 90000\n", 466 | "time: 7.41411566734314 count: 91000\n", 467 | "time: 6.791863918304443 count: 92000\n", 468 | "time: 7.763757705688477 count: 93000\n", 469 | "time: 10.760865449905396 count: 94000\n", 470 | "time: 8.289317607879639 count: 95000\n", 471 | "time: 6.7506866455078125 count: 96000\n", 472 | "time: 7.14838719367981 count: 97000\n", 473 | "time: 10.034961223602295 count: 98000\n", 474 | "time: 8.622971773147583 count: 99000\n", 475 | "time: 6.736122369766235 count: 100000\n", 476 | "time: 6.962104082107544 count: 101000\n", 477 | "time: 7.514451742172241 count: 102000\n", 478 | "time: 10.299906492233276 count: 103000\n", 479 | "time: 8.237797975540161 count: 104000\n", 480 | "time: 7.880961179733276 count: 105000\n", 481 | "time: 9.844894409179688 count: 106000\n", 482 | "time: 8.546481132507324 count: 107000\n", 483 | "time: 8.704168319702148 count: 108000\n", 484 | "time: 7.344848394393921 count: 109000\n", 485 | "time: 9.523374795913696 count: 110000\n", 486 | "time: 8.907413244247437 count: 111000\n", 487 | "time: 23.584630012512207 count: 112000\n", 488 | "time: 7.0646071434021 count: 113000\n", 489 | "time: 6.461396217346191 count: 114000\n", 490 | "time: 8.127644538879395 count: 115000\n", 491 | "time: 9.173459768295288 count: 116000\n", 492 | "time: 7.150416374206543 count: 117000\n", 493 | "time: 6.67113995552063 count: 118000\n", 494 | "time: 9.35042119026184 count: 119000\n", 495 | "time: 35.60696053504944 count: 120000\n", 496 | "time: 44.333606481552124 count: 121000\n", 497 | "time: 44.246965408325195 count: 122000\n", 498 | "time: 37.153340578079224 count: 123000\n", 499 | "time: 26.17804741859436 count: 124000\n", 500 | "time: 16.095215320587158 count: 125000\n", 501 | "time: 10.093717336654663 count: 126000\n", 502 | "-rw-rw-r-- 1 ec2-user ec2-user 387M Sep 11 18:49 nsfw_dataset_test.rec\n", 503 | "-rw-rw-r-- 1 ec2-user ec2-user 3.6G Sep 12 07:39 nsfw_dataset_train.rec\n", 504 | "-rw-rw-r-- 1 ec2-user ec2-user 318M Sep 12 07:16 nsfw_dataset_valid.rec\n" 505 | ] 506 | } 507 | ], 508 | "source": [ 509 | "%%bash\n", 510 | "echo \"Creating LST files\"\n", 511 | "python $IM2REC --list --recursive --pass-through --test-ratio=0.0 --train-ratio=1.0 nsfw_dataset_train nsfw_dataset_train > nsfw_dataset_train_classes\n", 512 | "\n", 513 | "echo \"Label classes:\"\n", 514 | "cat nsfw_dataset_train_classes\n", 515 | "\n", 516 | "echo \"Creating RecordIO files\"\n", 517 | "python $IM2REC --num-thread=4 nsfw_dataset_train.lst nsfw_dataset_train/ --resize=224\n", 518 | "#python $IM2REC --num-thread=4 ${DATASET_NAME}_test.lst $DATASET_NAME\n", 519 | "ls -lh *.rec" 520 | ] 521 | }, 522 | { 523 | "cell_type": "code", 524 | "execution_count": 5, 525 | "metadata": {}, 526 | "outputs": [ 527 | { 528 | "name": "stdout", 529 | "output_type": "stream", 530 | "text": [ 531 | "upload: ./nsfw_dataset_train.rec to s3://project-completion-udacity/nsfw_dataset/training/nsfw_dataset_train.rec\n" 532 | ] 533 | } 534 | ], 535 | "source": [ 536 | "!aws s3 cp nsfw_dataset_train.rec s3://project-completion-udacity/nsfw_dataset/training/" 537 | ] 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": null, 542 | "metadata": {}, 543 | "outputs": [], 544 | "source": [] 545 | } 546 | ], 547 | "metadata": { 548 | "kernelspec": { 549 | "display_name": "Python 3", 550 | "language": "python", 551 | "name": "python3" 552 | }, 553 | "language_info": { 554 | "codemirror_mode": { 555 | "name": "ipython", 556 | "version": 3 557 | }, 558 | "file_extension": ".py", 559 | "mimetype": "text/x-python", 560 | "name": "python", 561 | "nbconvert_exporter": "python", 562 | "pygments_lexer": "ipython3", 563 | "version": "3.5.2" 564 | } 565 | }, 566 | "nbformat": 4, 567 | "nbformat_minor": 2 568 | } 569 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/nsfw-training_built_in_aws-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## This Notebook contains code for tuning the hyper parameters and then training the model." 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import sagemaker\n", 17 | "from sagemaker import get_execution_role\n", 18 | "from sagemaker.amazon.amazon_estimator import get_image_uri\n", 19 | "\n", 20 | "role = get_execution_role()\n", 21 | "sess = sagemaker.Session()\n", 22 | "\n", 23 | "training_image = get_image_uri(sess.boto_region_name, 'image-classification', repo_version=\"latest\")" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "## we need training and validation data in the form of record io format.\n", 31 | "Notice s3train_path and s3validation_path contains location of training data and validation data in record io format." 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "\n", 41 | "s3train_path = 's3://project-completion-udacity/nsfw_dataset/training'\n", 42 | "s3validation_path = 's3://project-completion-udacity/nsfw_dataset/validation'\n", 43 | "\n", 44 | "train_data = sagemaker.session.s3_input(\n", 45 | " s3train_path, \n", 46 | " distribution='FullyReplicated', \n", 47 | " content_type='application/x-recordio', \n", 48 | " s3_data_type='S3Prefix'\n", 49 | ")\n", 50 | "\n", 51 | "validation_data = sagemaker.session.s3_input(\n", 52 | " s3validation_path, \n", 53 | " distribution='FullyReplicated', \n", 54 | " content_type='application/x-recordio', \n", 55 | " s3_data_type='S3Prefix'\n", 56 | ")\n", 57 | "\n", 58 | "data_channels = {'train': train_data, 'validation': validation_data}\n" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": {}, 64 | "source": [ 65 | "## Specifying the instance for training." 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 3, 71 | "metadata": {}, 72 | "outputs": [], 73 | "source": [ 74 | "bucket = 'project-completion-udacity'\n", 75 | "dataset_name = 'nsfw_dataset'\n", 76 | "\n", 77 | "\n", 78 | "s3_output_location = 's3://{}/{}/output'.format(bucket, dataset_name)\n", 79 | "\n", 80 | "image_classifier = sagemaker.estimator.Estimator(\n", 81 | " training_image,\n", 82 | " role, \n", 83 | " train_instance_count=1, \n", 84 | " train_instance_type='ml.p2.xlarge',\n", 85 | " output_path=s3_output_location,\n", 86 | " sagemaker_session=sess\n", 87 | ")" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 4, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | "126254 5\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "num_classes=5\n", 105 | "\n", 106 | "num_training_samples=! cat nsfw_dataset_train.lst | wc -l\n", 107 | "num_training_samples = int(num_training_samples[0])\n", 108 | "print(num_training_samples , num_classes)" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "## Declaring base hyperparameters that we dont wish to tune. " 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 5, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "\n", 125 | "# # Learn more about the Sagemaker built-in Image Classifier hyperparameters here: https://docs.aws.amazon.com/sagemaker/latest/dg/IC-Hyperparameter.html\n", 126 | "\n", 127 | "# # These hyperparameters we won't want to change, as they define things like\n", 128 | "# # the size of the images we'll be sending for input, the number of training classes we have, etc.\n", 129 | "base_hyperparameters=dict(\n", 130 | " use_pretrained_model=1,\n", 131 | " image_shape='3,224,224',\n", 132 | " num_classes=num_classes,\n", 133 | " num_training_samples=num_training_samples,\n", 134 | " augmentation_type = 'crop_color_transform',\n", 135 | " epochs = 1\n", 136 | " \n", 137 | " \n", 138 | ")\n", 139 | "\n", 140 | "# # These are hyperparameters we may want to tune, as they can affect the model training success:\n", 141 | "hyperparameters={\n", 142 | " **base_hyperparameters, \n", 143 | " **dict(\n", 144 | " learning_rate=0.001,\n", 145 | " mini_batch_size=5,\n", 146 | " )\n", 147 | "}\n", 148 | "\n", 149 | "\n", 150 | "image_classifier.set_hyperparameters(**hyperparameters)\n", 151 | "\n", 152 | "# hyperparameters\n" 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "metadata": {}, 158 | "source": [ 159 | "## Declaring the hyperparameters we wish to tune. And then creating hyper parameter tuning job." 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": 13, 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "from sagemaker.tuner import HyperparameterTuner, IntegerParameter, CategoricalParameter, ContinuousParameter\n", 169 | "hyperparameter_ranges = {'optimizer': CategoricalParameter(['nag', 'adam']),\n", 170 | " 'learning_rate': ContinuousParameter(0.0001, 0.01),\n", 171 | " 'mini_batch_size': IntegerParameter(15, 32),\n", 172 | " }\n", 173 | "\n", 174 | "objective_metric_name = 'validation:accuracy'\n", 175 | "\n", 176 | "tuner = HyperparameterTuner(image_classifier,\n", 177 | " objective_metric_name,\n", 178 | " hyperparameter_ranges,\n", 179 | " max_jobs=5,\n", 180 | " max_parallel_jobs=1)\n", 181 | "\n", 182 | "tuner.fit(inputs=data_channels, logs=True, include_cls_metadata=False)\n" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": 8, 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [ 191 | "# best_image_classifier = sagemaker.estimator.Estimator.attach(tuner.best_training_job())\n" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "# %%time\n", 201 | "\n", 202 | "# import time\n", 203 | "# now = str(int(time.time()))\n", 204 | "# training_job_name = 'IC-' + dataset_name.replace('_', '-') + '-' + now\n", 205 | "\n", 206 | "# image_classifier.fit(inputs=data_channels, job_name=training_job_name, logs=True)\n", 207 | "\n", 208 | "# job = image_classifier.latest_training_job\n", 209 | "# model_path = f\"{base_dir}/{job.name}\"\n", 210 | "\n", 211 | "# print(f\"\\n\\n Finished training! The model is available for download at: {image_classifier.output_path}/{job.name}/output/model.tar.gz\")\n" 212 | ] 213 | } 214 | ], 215 | "metadata": { 216 | "kernelspec": { 217 | "display_name": "Python 3", 218 | "language": "python", 219 | "name": "python3" 220 | }, 221 | "language_info": { 222 | "codemirror_mode": { 223 | "name": "ipython", 224 | "version": 3 225 | }, 226 | "file_extension": ".py", 227 | "mimetype": "text/x-python", 228 | "name": "python", 229 | "nbconvert_exporter": "python", 230 | "pygments_lexer": "ipython3", 231 | "version": "3.5.2" 232 | } 233 | }, 234 | "nbformat": 4, 235 | "nbformat_minor": 2 236 | } 237 | -------------------------------------------------------------------------------- /.ipynb_checkpoints/nsfw_deploy-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Contains code for deploying the trained model" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import sagemaker\n", 17 | "from sagemaker import get_execution_role\n", 18 | "from sagemaker.amazon.amazon_estimator import get_image_uri\n", 19 | "\n", 20 | "role = get_execution_role()\n", 21 | "sess = sagemaker.Session()\n", 22 | "\n", 23 | "training_image = get_image_uri(sess.boto_region_name, 'image-classification', repo_version=\"latest\")\n" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 2, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "# bucket = 'project-completion-udacity'\n", 33 | "# dataset_name = 'nsfw_dataset'\n", 34 | "\n", 35 | "\n", 36 | "# s3_output_location = 's3://project-completion-udacity/nsfw_dataset/output'\n", 37 | "\n", 38 | "# image_classifier = sagemaker.estimator.Estimator(\n", 39 | "# training_image,\n", 40 | "# role, \n", 41 | "# train_instance_count = 1, \n", 42 | "# train_instance_type = 'ml.p2.xlarge',\n", 43 | "# output_path=s3_output_location,\n", 44 | "# sagemaker_session=sess\n", 45 | "# )" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "# the only change you need to make is to change job_name with the name of your own training job." 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 3, 58 | "metadata": {}, 59 | "outputs": [ 60 | { 61 | "name": "stdout", 62 | "output_type": "stream", 63 | "text": [ 64 | "DEMO-full-image-classification-model-2019-09-11-06-17-19\n", 65 | "s3://project-completion-udacity/nsfw_dataset/output/IC-nsfw-dataset-1568090782/output/model.tar.gz\n", 66 | "arn:aws:sagemaker:us-east-2:733184320490:model/demo-full-image-classification-model-2019-09-11-06-17-19\n" 67 | ] 68 | } 69 | ], 70 | "source": [ 71 | "\n", 72 | "\n", 73 | "job_name = 'IC-nsfw-dataset-1568090782'\n", 74 | "import boto3\n", 75 | "from time import gmtime, strftime\n", 76 | "import time\n", 77 | "sage = boto3.Session().client(service_name='sagemaker') \n", 78 | "timestamp = time.strftime('-%Y-%m-%d-%H-%M-%S', time.gmtime())\n", 79 | "model_name=\"DEMO-full-image-classification-model\" + timestamp\n", 80 | "print(model_name)\n", 81 | "info = sage.describe_training_job(TrainingJobName=job_name)\n", 82 | "model_data = info['ModelArtifacts']['S3ModelArtifacts']\n", 83 | "print(model_data)\n", 84 | "\n", 85 | "hosting_image = get_image_uri(boto3.Session().region_name, 'image-classification')\n", 86 | "\n", 87 | "primary_container = {\n", 88 | " 'Image': hosting_image,\n", 89 | " 'ModelDataUrl': model_data,\n", 90 | "}\n", 91 | "\n", 92 | "create_model_response = sage.create_model(\n", 93 | " ModelName = model_name,\n", 94 | " ExecutionRoleArn = role,\n", 95 | " PrimaryContainer = primary_container)\n", 96 | "\n", 97 | "print(create_model_response['ModelArn'])" 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": {}, 103 | "source": [ 104 | "

For endpoint configuration" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "# %%time\n", 114 | "# # Deploying a model to an endpoint takes a few minutes to complete\n", 115 | "\n", 116 | "# deployed_endpoint = image_classifier.deploy(\n", 117 | "# initial_instance_count = 1,\n", 118 | "# instance_type = 'ml.t2.medium'\n", 119 | "# )\n", 120 | "\n", 121 | "from time import gmtime, strftime\n", 122 | "\n", 123 | "timestamp = time.strftime('-%Y-%m-%d-%H-%M-%S', time.gmtime())\n", 124 | "endpoint_config_name = 'my-endpoint' + timestamp\n", 125 | "endpoint_config_response = sage.create_endpoint_config(\n", 126 | " EndpointConfigName = endpoint_config_name,\n", 127 | " ProductionVariants=[{\n", 128 | " 'InstanceType':'ml.t2.medium',\n", 129 | " 'InitialInstanceCount':1,\n", 130 | " 'ModelName':model_name,\n", 131 | " 'VariantName':'AllTraffic'}])\n", 132 | "\n", 133 | "print('Endpoint configuration name: {}'.format(endpoint_config_name))\n", 134 | "print('Endpoint configuration arn: {}'.format(endpoint_config_response['EndpointConfigArn']))" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "## code for lauanching the endpoint." 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 5, 147 | "metadata": {}, 148 | "outputs": [ 149 | { 150 | "name": "stdout", 151 | "output_type": "stream", 152 | "text": [ 153 | "Endpoint name: myprojectcapstone\n", 154 | "EndpointArn = arn:aws:sagemaker:us-east-2:733184320490:endpoint/myprojectcapstone\n" 155 | ] 156 | } 157 | ], 158 | "source": [ 159 | "endpoint_name = 'myprojectcapstone'\n", 160 | "print('Endpoint name: {}'.format('myprojectcapstone'))\n", 161 | "\n", 162 | "import sagemaker as sm\n", 163 | "\n", 164 | "endpoint_params = {\n", 165 | " 'EndpointName': endpoint_name,\n", 166 | " 'EndpointConfigName': endpoint_config_name,\n", 167 | "}\n", 168 | "endpoint_response = sess.sagemaker_client.create_endpoint(**endpoint_params)\n", 169 | "print('EndpointArn = {}'.format(endpoint_response['EndpointArn']))" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 6, 175 | "metadata": {}, 176 | "outputs": [ 177 | { 178 | "name": "stdout", 179 | "output_type": "stream", 180 | "text": [ 181 | "------------------------------------------------------------------------------------------------------------!" 182 | ] 183 | } 184 | ], 185 | "source": [ 186 | "endpoint_dec = sess.wait_for_endpoint(endpoint_name)" 187 | ] 188 | }, 189 | { 190 | "cell_type": "markdown", 191 | "metadata": {}, 192 | "source": [ 193 | "## Testing the deployed endpoint" 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": 7, 199 | "metadata": {}, 200 | "outputs": [], 201 | "source": [ 202 | "import json\n", 203 | "import numpy as np\n", 204 | "import os\n", 205 | "\n", 206 | "def classify_deployed(file_name, classes , endpoint_name):\n", 207 | " payload = None\n", 208 | " with open(file_name, 'rb') as f:\n", 209 | " payload = f.read()\n", 210 | " payload = bytearray(payload)\n", 211 | "\n", 212 | " response = sess.sagemaker_runtime_client.invoke_endpoint(\n", 213 | " EndpointName = endpoint_name,\n", 214 | " ContentType = 'application/x-image',\n", 215 | " Body = payload)\n", 216 | " \n", 217 | " #print(response)\n", 218 | " result = response['Body'].read()\n", 219 | " result = json.loads(result)\n", 220 | " print(result)\n", 221 | " best_prob_index = np.argmax( np.array( result ) )\n", 222 | " print(best_prob_index)\n", 223 | " return (classes[best_prob_index], result[best_prob_index])" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": 8, 229 | "metadata": {}, 230 | "outputs": [ 231 | { 232 | "name": "stdout", 233 | "output_type": "stream", 234 | "text": [ 235 | "[0.0002461728290654719, 0.0017670468660071492, 0.00032288278453052044, 0.9976606369018555, 3.218596020815312e-06]\n", 236 | "3\n", 237 | "safe_for_work 0.9976606369018555\n" 238 | ] 239 | } 240 | ], 241 | "source": [ 242 | "file_name = 'flowes/test/daisy/10555826524_423eb8bf71_n.jpg'\n", 243 | "classes = ['animated' , 'nude' ,'porn' , 'safe_for_work' , 'semi_nude']\n", 244 | "\n", 245 | "class_predicted , class_score = classify_deployed(file_name, classes , endpoint_name)\n", 246 | "print(class_predicted , class_score)\n" 247 | ] 248 | }, 249 | { 250 | "cell_type": "markdown", 251 | "metadata": {}, 252 | "source": [ 253 | "## Deleting the endpoint." 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": 9, 259 | "metadata": {}, 260 | "outputs": [ 261 | { 262 | "data": { 263 | "text/plain": [ 264 | "{'ResponseMetadata': {'RequestId': '75b459e9-b20b-466b-aab3-17afe9b8fa29',\n", 265 | " 'HTTPStatusCode': 200,\n", 266 | " 'HTTPHeaders': {'x-amzn-requestid': '75b459e9-b20b-466b-aab3-17afe9b8fa29',\n", 267 | " 'content-type': 'application/x-amz-json-1.1',\n", 268 | " 'content-length': '0',\n", 269 | " 'date': 'Wed, 11 Sep 2019 06:30:04 GMT'},\n", 270 | " 'RetryAttempts': 0}}" 271 | ] 272 | }, 273 | "execution_count": 9, 274 | "metadata": {}, 275 | "output_type": "execute_result" 276 | } 277 | ], 278 | "source": [ 279 | "sess.sagemaker_client.delete_endpoint(EndpointName = endpoint_name)" 280 | ] 281 | }, 282 | { 283 | "cell_type": "code", 284 | "execution_count": null, 285 | "metadata": {}, 286 | "outputs": [], 287 | "source": [] 288 | } 289 | ], 290 | "metadata": { 291 | "kernelspec": { 292 | "display_name": "Python 3", 293 | "language": "python", 294 | "name": "python3" 295 | }, 296 | "language_info": { 297 | "codemirror_mode": { 298 | "name": "ipython", 299 | "version": 3 300 | }, 301 | "file_extension": ".py", 302 | "mimetype": "text/x-python", 303 | "name": "python", 304 | "nbconvert_exporter": "python", 305 | "pygments_lexer": "ipython3", 306 | "version": "3.5.2" 307 | } 308 | }, 309 | "nbformat": 4, 310 | "nbformat_minor": 2 311 | } 312 | -------------------------------------------------------------------------------- /Capstone_Project_Report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepanshu-yadav/NSFW-Classifier/8ee516cf4dc9045f8cd94fd13a4b6fa2cc92a554/Capstone_Project_Report.pdf -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 deepanshu-yadav 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Project_Proposal.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepanshu-yadav/NSFW-Classifier/8ee516cf4dc9045f8cd94fd13a4b6fa2cc92a554/Project_Proposal.pdf -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Email me at awsrohanyadav@gmail.com if you want the trained model. 2 | 3 | ![alt text](AWS.svg?raw=true) 4 | 5 | 6 | 7 | # NSFW-Classifier 8 | An AWS Sagemaker Model developed for Nudity / NSFW Images Classification
9 | 10 | 11 | This is project done for the fullfillment of Udacity Machine Learning Engineer NanoDegree. I have build a model that classifies 12 | input image into five categories. 13 | 1. **Nude** 14 | 2. **Semi Nude** 15 | 3. **Animated** 16 | 4. **Porn** 17 | 5. **Safe For Work** 18 | 19 | # Demo 20 | The model is deployed on aws check it yourself by downloading this repository , then going to demo folder and running index.html. 21 | 22 | 23 | # Contents 24 | 1. [classification_tool.ipynb](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/classification_tool.ipynb) 25 | 2. [clean and prepare data.ipynb](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/clean%20and%20prepare%20data.ipynb) 26 | 3. [nsfw-training_built_in_aws.ipynb](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/nsfw-training_built_in_aws.ipynb) 27 | 4. [nsfw_deploy.ipynb](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/nsfw_deploy.ipynb) 28 | 5. [batch_transform.ipynb](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/batch_transform.ipynb) 29 | 6. [benchmark.ipynb](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/benchmark.ipynb) 30 | 7. [analyze_bench.ipynb](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/analyze_bench.ipynb) 31 | 8. [Metrics.ipynb](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/Metrics.ipynb) 32 | 9. [analyze_results.ipynb ](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/analyze_results.ipynb) 33 | 10. [Capstone_Project_Report.pdf](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/Capstone_Project_Report.pdf) 34 | 11. [Project_Proposal.pdf](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/Project_Proposal.pdf) 35 | 12. [results.csv](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/results.csv) 36 | 13. [results_bench.csv](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/results_bench.csv) 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | # Data Collection And Organization 47 | The following guys had collected the data 48 | 1. [B Praneeth 's Data](https://archive.org/details/NudeNet_classifier_dataset_v1) . He did an awesome job in collection 49 | of data . The data is for three classes
50 | * Nude 51 | * Sexy 52 | * Safe 53 | 1. But the problem is I need more categories for my problem . So I made a simple [tool](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/classification_tool.ipynb) that is helpful for sub classifying the above Nude Images. You just keep all the training samples in one folder and run and run it in a jupyter notebook. 54 | I classified a few thousands of these , but then i realized that it would take a while to gather huge data. For class **Safe For Work** i sampled randomly from his huge dataset. 55 | 2. Further More I also made a [tool](https://github.com/deepanshu-yadav/NSFW-Classifier/blob/master/useful_scripts/useful_scripts/example.py) that takes a screenshot of the screen and saves it into a folder. It becomes handy when you want to deliberately put difficult examples in your dataset. 56 | 57 | The above tools proved helpful but did not solved the problem of gathering large number of examples for training. Therefore scraping was necessary. 58 | 59 | 2. [Bazarov 's Dataset](https://github.com/EBazarov/nsfw_data_source_urls) . For collecting set of nude images I included the the sub category in the list he provided namely:
60 | * Female genitalia 61 | * Male genitalia 62 | * Breasts
63 | By now I had enough examples of class **nude.**
64 | 65 | 66 | 3.[ Alex's Dataset](https://github.com/alex000kim/nsfw_data_scraper/tree/master/raw_data) . For classes **animated** and **porn** i scraped the data from here. 67 | 68 | 69 | 4. [Instagram Scrapper](https://github.com/rarcega/instagram-scraper) For class **Semi Nude** I used his tool to scrape few Instagram pages that regularly post arousing images of men and women. 70 | -------------------------------------------------------------------------------- /analyze_results.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

For conversion of .out files from batch transform to a csv file" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 3, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "name": "stdout", 17 | "output_type": "stream", 18 | "text": [ 19 | "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mos\u001b[39;49;00m \r\n", 20 | "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mcsv\u001b[39;49;00m\r\n", 21 | "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mre\u001b[39;49;00m \r\n", 22 | "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mnumpy\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mnp\u001b[39;49;00m\r\n", 23 | "\r\n", 24 | "\u001b[34mdef\u001b[39;49;00m \u001b[32mgive_prediction_label\u001b[39;49;00m(result_string):\r\n", 25 | " \u001b[37m#x = re.match(\"[(.*)]\", result_string)\u001b[39;49;00m\r\n", 26 | " x = result_string[result_string.find(\u001b[33m'\u001b[39;49;00m\u001b[33m[\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)+\u001b[34m1\u001b[39;49;00m : result_string.find(\u001b[33m'\u001b[39;49;00m\u001b[33m]\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)].split(\u001b[33m'\u001b[39;49;00m\u001b[33m,\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\r\n", 27 | " x_float = [ \u001b[36mfloat\u001b[39;49;00m(xi) \u001b[34mfor\u001b[39;49;00m xi \u001b[35min\u001b[39;49;00m x ]\r\n", 28 | " ind = x_float.index(\u001b[36mmax\u001b[39;49;00m(x_float)) \r\n", 29 | " \r\n", 30 | " \u001b[34mreturn\u001b[39;49;00m ind \r\n", 31 | "\r\n", 32 | "\r\n", 33 | "\r\n", 34 | "\u001b[34mdef\u001b[39;49;00m \u001b[32mmake_results_csv\u001b[39;49;00m( results_directory , csv_path , csv_file_name , eval_image_path ):\r\n", 35 | " \u001b[34mwith\u001b[39;49;00m \u001b[36mopen\u001b[39;49;00m(os.path.join(csv_path , csv_file_name), mode=\u001b[33m'\u001b[39;49;00m\u001b[33mw\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m) \u001b[34mas\u001b[39;49;00m \u001b[36mfile\u001b[39;49;00m:\r\n", 36 | " writer = csv.writer(\u001b[36mfile\u001b[39;49;00m, delimiter=\u001b[33m'\u001b[39;49;00m\u001b[33m,\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, quotechar=\u001b[33m'\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, quoting=csv.QUOTE_MINIMAL) \r\n", 37 | " writer.writerow([\u001b[33m'\u001b[39;49;00m\u001b[33mImage_path\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, \u001b[33m'\u001b[39;49;00m\u001b[33mground_truth_label\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m, \u001b[33m'\u001b[39;49;00m\u001b[33mPredicted\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m])\r\n", 38 | " i = \u001b[34m0\u001b[39;49;00m\r\n", 39 | " sorted_list = \u001b[36msorted\u001b[39;49;00m(os.listdir(results_directory))\r\n", 40 | " \u001b[34mfor\u001b[39;49;00m single_class \u001b[35min\u001b[39;49;00m sorted_list:\r\n", 41 | " list_of_files = os.listdir(os.path.join( results_directory , single_class ))\r\n", 42 | " \u001b[34mfor\u001b[39;49;00m img_result \u001b[35min\u001b[39;49;00m list_of_files:\r\n", 43 | " result_file = \u001b[36mopen\u001b[39;49;00m( os.path.join( results_directory , single_class , img_result ) , \u001b[33m'\u001b[39;49;00m\u001b[33mr\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\r\n", 44 | " result_string = result_file.read()\r\n", 45 | " prediction_val = give_prediction_label(result_string)\r\n", 46 | " \u001b[37m#print(result_string.split(','))\u001b[39;49;00m\r\n", 47 | " label = \u001b[36mstr\u001b[39;49;00m(i)\r\n", 48 | " img_path = os.path.join( eval_image_path , single_class ,img_result[:-\u001b[34m4\u001b[39;49;00m] )\r\n", 49 | " writer.writerow([img_path, label , prediction_val ]) \r\n", 50 | " result_file.close()\r\n", 51 | " \u001b[37m#print(i)\u001b[39;49;00m\r\n", 52 | " i += \u001b[34m1\u001b[39;49;00m\r\n", 53 | "\u001b[37m#make_results_csv('analyze/analyze' , 'dump' , 'results.csv' , 'nsfw_dataset_test' )\u001b[39;49;00m\r\n" 54 | ] 55 | } 56 | ], 57 | "source": [ 58 | "#make_results_csv( 'batch_transform_results' , '' , 'results.csv' , 'nsfw_dataset_test' )\n", 59 | "!pygmentize useful_scripts/useful_scripts/csv_maker.py" 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "

This notebook is used for first convert the results of batch transform to a csv file and then analyzing it. " 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": 5, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "import pandas as pd\n", 76 | "df = pd.read_csv('results.csv')\n" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 6, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "data": { 86 | "text/html": [ 87 | "
\n", 88 | "\n", 101 | "\n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | "
Image_pathground_truth_labelPredicted
0nsfw_dataset_test/animated/(m=e-yaaGqaa)(mh=SW...00
1nsfw_dataset_test/animated/(m=e-yaaGqaa)(mh=-z...00
2nsfw_dataset_test/animated/(m=e-yaaGqaa)(mh=4Q...00
3nsfw_dataset_test/animated/ouwo5EE.jpg00
4nsfw_dataset_test/animated/(m=e-yaaGqaa)(mh=Sx...00
\n", 143 | "
" 144 | ], 145 | "text/plain": [ 146 | " Image_path ground_truth_label \\\n", 147 | "0 nsfw_dataset_test/animated/(m=e-yaaGqaa)(mh=SW... 0 \n", 148 | "1 nsfw_dataset_test/animated/(m=e-yaaGqaa)(mh=-z... 0 \n", 149 | "2 nsfw_dataset_test/animated/(m=e-yaaGqaa)(mh=4Q... 0 \n", 150 | "3 nsfw_dataset_test/animated/ouwo5EE.jpg 0 \n", 151 | "4 nsfw_dataset_test/animated/(m=e-yaaGqaa)(mh=Sx... 0 \n", 152 | "\n", 153 | " Predicted \n", 154 | "0 0 \n", 155 | "1 0 \n", 156 | "2 0 \n", 157 | "3 0 \n", 158 | "4 0 " 159 | ] 160 | }, 161 | "execution_count": 6, 162 | "metadata": {}, 163 | "output_type": "execute_result" 164 | } 165 | ], 166 | "source": [ 167 | "df.head()" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": 7, 173 | "metadata": {}, 174 | "outputs": [], 175 | "source": [ 176 | "comparison = df.iloc[:, [1,2]].values" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 8, 182 | "metadata": {}, 183 | "outputs": [ 184 | { 185 | "name": "stdout", 186 | "output_type": "stream", 187 | "text": [ 188 | "[[1901 2 28 7 45]\n", 189 | " [ 13 494 755 11 186]\n", 190 | " [ 137 38 3492 7 473]\n", 191 | " [ 155 23 53 806 55]\n", 192 | " [ 10 5 27 0 2413]]\n" 193 | ] 194 | } 195 | ], 196 | "source": [ 197 | "import numpy as np\n", 198 | "samples = comparison.shape[0]\n", 199 | "conf_matrix = np.zeros( (5, 5) , dtype=np.int32 )\n", 200 | "\n", 201 | "for i in range(samples):\n", 202 | " conf_matrix[ int(comparison[i][0]) ][ int(comparison[i][1]) ] += 1\n", 203 | "\n", 204 | "print(conf_matrix)" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": 9, 210 | "metadata": {}, 211 | "outputs": [ 212 | { 213 | "data": { 214 | "text/plain": [ 215 | "" 216 | ] 217 | }, 218 | "execution_count": 9, 219 | "metadata": {}, 220 | "output_type": "execute_result" 221 | }, 222 | { 223 | "data": { 224 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAGgCAYAAAAn2ypWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd4VEUXx/HvSULovVdRsKMivfdeLYigKIIKIlhAsAMCSlEQEURBRcBCUWkiUqXXgDSRIljoPfSezPvHLrz0BAnZ5O7v8zz7sDt37ty5S7I5e2bmXnPOISIiIiKJX0igOyAiIiIicUOBnYiIiIhHKLATERER8QgFdiIiIiIeocBORERExCMU2ImIiIh4hAI7EREREY9QYCciIiLiEQrsRERERDwiLNAdENhwZw3d/uMGu2vT6kB3wfNCzALdhaAQrbsF3XChIcp5xIeTJ7bE64fG6b1/xdkvT5JMtyTYDzz99IqIiIh4hDJ2IiIi4n3RUYHuQbxQxk5ERETEI5SxExEREe9z0YHuQbxQYCciIiLeFx0cgZ2GYkVEREQ8Qhk7ERER8TynoVgRERERj9BQrIiIiIgkJsrYiYiIiPdpKFZERETEI3SBYhERERFJTJSxExEREe/TUKyIiIiIR2hVrIiIiIgkJsrYiYiIiOfpAsUiIiIiXqGhWBERERFJTJSxExEREe/TUKyIiIiIR+gCxSIiIiKSmChjJyIiIt6noVgRERERj9CqWBERERFJTJSxExEREe/TUKyIiIiIR2goVkREREQSE2XsRERExPOcC47r2CmwExEREe8Lkjl2GooVERER8Qhl7ERERMT7gmTxhAI7ERER8T4NxYqIiIhIYqLALkhkfbctt8wbyU0TPrvs9pA0qcjRvyM3jfuUPKP6EX7rTdd9TEuShOwfvkHeyUPIPfIjwnJkBSDZPbeRZ8wn5BnzCTeNHUiqKqWu+1hekitXDqZP/Z5VK2eycsWvvNDm6UB3yRNy5crO1CmjWbniV1Ysn0Eb//t63713MXfOBCKWTGHhgp8pUqRggHvqDbfdlo+lEVPPPfbvXceLLzwT6G55RkhICIsX/cLYMV8B8PnnH7J+3XyWLJ7MksWTuffeuwLcwwQoOiruHgmYhmKDxKFx0zjw3U9k69n+stsztGjEibV/sf2FbiS5ORdZO7Zma/M3YtV2WI6sZOvxClubvnpBeZoG1Yk6eIR/ajQnda3yZG7fnB3tenDyz3/Z/MgLEBVNaOYM3DR2IEdmLoKo4EiTx+TMmTN0eLULy1f8TqpUKVmyeDLTZ8xh7do/A921RO3MmShefa0rK/zv6+JFvzBj+hy693iLd9/ry5QpM6lRoxI9ur9F1WqPBLq7id6GDZsoUrQa4AtCNv+zjHHjfwlwr7zjhTZPs279RtKkTnWu7PU33mPs2EkB7FUCp6HYwDGz58zsyThq683/sM9TZjYgDo5dz8xev9524sLxpb8TdeDwFbeH58/D8cUrADj991bCcmYlNGM6AFLXrUSeUf3IM+YTsrzzIoTE7scmVaWSHBo/HYDDU+aSooQvE+JOnDwXxFl4EnDuP5+XF+3cuZvlK34H4MiRo6xb9yc5c2QLcK8Sv507d7Piovc1R85sOOfO/XFMmyY1O3bsCmQ3PalypTL89de/bN68LdBd8YScObNRs2YlvvpqRKC7IglQggzsnHOfOeeGx1Fz1xzYxQUzC3POTXDO9QzE8a/VyXV/kapqacA3VJokR1bCsmYi/JbcpK5Zjs2Pt2PzQ60hOorUdSvGqs2wrBk5s2OP70VUNFGHjxKSLo3vGPfezk0/DSLv+M/Y3aW/snVXcNNNuSh4XwEWL1ke6K54yk035eK++wqwZMly2rd/hx493mbTxiX07NmRtzv2CHT3PKdhw/qMHDUu0N3wjN4fvMMbb3Yn+qJVnl27vMrSiKl88H5nwsPDA9S7BCw6Ou4eCVi8BXZmNs7MlpnZGjNr4S87YmbvmdlKM1tkZln95e+YWXv/81lm1tfMlprZWjMramZjzOxPM3s3hvZ7AsnNbIWZfesva2JmS/xlg8ws1F/ezMw2mNkSoHQM55LXzH41s1VmNsPM8vjLh5rZZ2a2GHj/apk/M2vhP6elow5suc539/pFfj6akNSpyDPmE9I1qc/JtZsgOpoUJQqS7O5byTP6Y/KM+YQUJe4nPFd2AHL070ieMZ+Qc1BXXx3/vLk0D1aN8XgnVq3n37ot2dzwRTI8+6gvcycXSJkyBaNHfU679p05fPhIoLvjGSlTpmDUyMG0b/8Ohw8foUWLJ+nQoQv58hejQ4d3GDSod6C76ClJkiShbp1q/PDjxEB3xRNq1azMnj37WL589QXlHTv25J57K1CqdB3SZ0hL+/atAtTDBMxFx90jAYvPOXbNnXP7zSw5EGFmPwIpgUXOubfM7H3gWeDdy+x7yjlXxMxeAsYDhYH9wCYz6+uc23e59p1zr5tZG+dcQQAzuxN4FCjtnDttZgOBx81sGtDF3+5BYCZwtRRJf2CYc26YmTUHPgYe8G/LBZRyzkWZ2VNXasA5NxgYDLDhzhoBH4uMPnqMXW99eO71zdOHcXrLTpIXLsChcdPZ2/erS/bZ/kI34Mpz7M7s2kdY9syc2bUXQkMITZ2S6AOHLqhz6q8tRB87TviteTm5RnPIzgoLC+P7UZ8zYsRYxo3TvKS4EhYWxqhRgxkxcuy5+V5PNGlAu3adAPjhx4l89tkHgeyi59SoUZHly1eze/feQHfFE0qWKkLt2lWpXqMiyZImJU2a1Hz1VT+aNXsJgFOnTjF8+GjavtwywD2VQInPodgXzWwlsAjIDdwKnALOfo1bBuS9wr4T/P+uBtY453Y4504Cf/nbulL7F6uML3iLMLMV/te3AMWBWc65Pc65U8CoGM6lJPCd//nXQJnztn3vEuEN6UJSp4Qkvjg/7SM1OL50NdFHj3Fs0QpSVS9DaIa0vnppUxGWI0us2jwycxFp6lcBIHX1shxbtBKAsJxZIdT3oxeWIwvht+Tm9DbNazrf54P7sHbdRj7qNzjQXfGUwYN6s27dRvr1+/xc2Y4duyhXriQAFSuWZuPGvwPVPU9q9OgDGoaNQx079iJf/mLcfnspnniyNbNmzadZs5fIlu3/n8v16lZnzZr1AexlAhUkQ7HxkrEzswpAFaCkc+6Ymc0CkgGnnTs3cz7qKv056f83+rznZ1+HXaX9S7qCL9N2wXJPM3vgMnX/q6Nx2Facydb7dVIUu5fQdGm4eebX7BvwDRYWCsDBUZMIz5eHbD1eAQcnN/7Lrrf7AnBq02b29htGzi+6YyEhuDNn2N3tE85s3x3jMQ/9MJlsvV4l7+QhRB88zI5XfHOXkhcuQIZnG+JOnwHn2NV1wCWZvGBWulRRnmjSgFWr/2BpxFTAN8zyy+RfA9yzxK1UqaI0adKA1avXErFkCgAdO/XiuVav8mGfLoSFhXHixElaPf9agHvqHSlSJKdK5XJ6T+PB0KEfkzlTRsyMlavW0KZN7K5qEFQSeEAWV+JrKDYtEOkPuu4ASsRj+6fNLIlz7jQwAxjvH77dbWYZgNTAYqCfmWUEDgGPACuvcrwFQCN82brHgblxfD5xbmf7q6/hOLFiLf/UvPw1po78Mocjv8y54r5ntu+6ZBgWwJ06zY62711SfnjCDA5PmBFDj4PX/AURhIXnDHQ3PGfBggjCk+a67LYSJWvFc2+Cw7Fjx8mavUCgu+FZc+YsYs6cRQDUqNEowL1J+BLhYNp/El9DsZPxZdbWAj3xDZfGV/uDgVVm9q1z7g/gbWCqma0CpgHZnXM7gHeAhcB8YG0Mx3sBaOZv4wngpbg8GREREZH/wpyuIRZwCWHxhNfdtWl1zJXkuoSYBboLQSFan9k3XGgsr9Up1+fkiS3x+qFxfNaQOPvlSV6heYL9wNOdJ0RERMT7EvhlSuKKArurMLO38M23O9/3zrlLJ46JiIiIBJgCu6vwB3AK4kRERBI7rYoVERER8YggGYrVDFERERERj1BgJyIiIt4XT3eeMLNk/nvSr/Tfv76Lv/xmM1tsZhvNbJSZhfvLk/pfb/Rvz3teW2/4y9ebWfXYnKYCOxEREfE+Fx13j6s7CVRyzt0HFARqmFkJoBfQ1zmXH4gEnvbXfxrfTRbyA3399TCzu/DdDOFuoAYw0MxCYzq4AjsRERGROOJ8jvhfJvE/HFAJ+MFfPgw4ezvT+v7X+LdXNjPzl490zp10zv0NbASKxXR8BXYiIiLiffE0FAtgZqFmtgLYje8uV5uAA865M/4qW4Gz947MCWwB8G8/CGQ8v/wy+1yRVsWKiIiI98Xh5U7MrAXQ4ryiwc65wWdfON+NaQuaWTpgLHBHnB08BgrsRERERK6BP4gbHIt6B8xsJlASSGdmYf6sXC5gm7/aNiA3sNXMwoC0wL7zys86f58r0lCsiIiIeF88LZ4ws8z+TB1mlhyoCqwFZgIN/NWaAuP9zyf4X+Pf/qtzzvnLG/lXzd4M3Aosiek0lbETERER74u/O09kB4b5V7CGAKOdcxPN7A9gpJm9CywHvvTX/xL42sw2AvvxrYTFObfGzEYDfwBngNb+Id6rUmAnIiIiEkecc6uA+y9T/heXWdXqnDvBpfelP7vtmm9tqsBOREREvC9IbimmwE5ERES8L/6GYgNKiydEREREPEIZOxEREfE+DcWKiIiIeISGYkVEREQkMVHGTkRERLwvSDJ2CuxERETE+5wLdA/ihYZiRURERDxCGTsRERHxPg3FioiIiHhEkAR2GooVERER8Qhl7ERERMT7dIFiEREREY/QUKyIiIiIJCbK2ImIiIj3Bcl17BTYiYiIiPcFyVCsArsE4K5NqwPdBc8rlCl/oLvgeXmSpA10F4LCuB3LAt0Fz0uTNEWguyDynymwExEREe9Txk5ERETEI4LkcidaFSsiIiLiEcrYiYiIiOe5aK2KFREREfGGIJljp6FYEREREY9Qxk5ERES8L0gWTyiwExEREe8Lkjl2GooVERER8Qhl7ERERMT7gmTxhAI7ERER8b4gCew0FCsiIiLiEcrYiYiIiPe54Fg8ocBOREREvE9DsSIiIiKSmChjJyIiIt4XJNexU2AnIiIi3hckd57QUKyIiIiIRyhjJyIiIt6noVgRERERb3BBsipWgZ2IiIh4X5Bk7DTHTkRERMQjlLETERER7wuSVbEK7ERERMT7NBQrIiIiIomJMnYiIiLifVoVKyIiIuIRGooVERERkcREGTsRERHxPq2KFREREfEIDcWKiIiISGKijJ2IiIh4nu4VKyIiIuIVGooV8fl8cB+2b13JiuUzzpV1eacDvy2bxtKIqfzy83dkz541gD1MOEJCQvh66hd8OKzHJduy5czKJ6M+5NvpQ/j0h4/Ikj3zdR8vTbrU9B/Zhx/mfUv/kX1InTYVANUfrMK304fw3Yyv+GLCJ9x6V77rPlZCkOOWnHww6aNzj+G/j6R283qX1Lu7RAE+mPQRfacNoMuo7td93LDwMNoO6ED/2YPoMe4DMufKAsC9ZQrSa+KH9JnyMb0mfkiBUvde97ESu8t9Xjz8cB1WrviVUye2ULiQ3iOAfgO688fGBcxZ+NNlt6dNl4ah3wxg1vwJTPn1e+6489brPmZ4eBI+/6ovS5ZPZfKM0eTOkxOA8hVLMX32j8xeMIHps3+kTLkS130sCRwFdhKj4cNHU7vO4xeU9e7zKYUKV6VI0Wr8PGk6b7/VNkC9S1gaPdOAf/7897LbXur0PJN+mMLjVZrzZd9hPP9Gi1i3W6hkQTr1ff2S8qZtHidi3jIalPH927SN7/9p+5YdPPfwizxWuRlf9h3OG++3/28nlMBs/2sbHWq9TIdaL/NanXacPH6SxVMWXlAnRZqUPPPuc/R65l3aVm1Dn+d7xbr9zLmy0GXke5eUV360KkcPHuGF8i2Z+OUEmrzeFIDDkYfo2fxdXqn+IgPafcQLffV7cLnPizVr1vFIw2eZO3dRgHqV8Iz8bgyNHn7mittffuU5fl+9lgql69G65Wu81+utWLedO09Oxk0cfkn5408+woEDhyh2fzU+GziUTl18nwv790Xy+KOtKF+qHm2ee52Bg96/9hNKDKJd3D0SsKAP7MxslpkVuYHtdzWzKjeq/fgwd95i9kceuKDs8OEj556nTJkC5xL2D3p8yJI9M6Url2D8dxMvu/3m224iYv5vACydv5xy1Uuf29akVSOGThrEt9OH8Gz7ZrE+Zrnqpfl59GQAfh49mfI1ygCweukaDh/0/R/9/tuaOMkOJjT3lL6XXZt3snfbngvKy9Yvx+LJC9m7fS8Ah/Yd/P+2ByvQY3xvPpj0ES26P09ISOw+AotWLc6sH38FYOGk+dxT+j4A/l7zF5G79wOwZcNmwpOFExYe3DNcLvd5sW7dRjZs2BSgHiVMCxcsJTLy4BW33357PubN8QXCG//8i9x5cpI5c0YAGjSsx5Rfv2fm3HH0/qhLrH+Oa9aqxKjvxgLw07gplC1fEoDVq9aya+duANat/ZNkyZMSHp7kP59bguWi4+6RgAV9YHcjmVmoc66Tc256oPtyI3Tr+hp/b4qgceMHeafLB4HuTsC17dKG/u9+RvQVvs39+ccmKtYsB0CFmmVJlToladOnoXj5IuS+ORdP1WpJk6pPc+c9t3F/8dgNV2XIlJ59/sBi3+79ZMiU/pI69RrXZuHMxf/xrBKu0vXKMW/CnEvKs9+ck1RpU9Fl5Hv0mvgh5R+qCEDO/LkoXacMbz/8Gh1qvUx0dDRlHygfq2NlyJbxXKAYHRXNscNHSZ0+9QV1StQqxd+/b+LMqTPXeWYisOb3ddSuWw2A+wvdQ+7cOcieMxu33nYLDzxUk9rVGlOx7ANERUXToGHdWLWZLXtWtm3bAUBUVBSHDh0mQ4YLPzPq1q/OqpV/cOrU6bg9IYk3if6rpZnlBX4B5gGlgG1AfX9Ze+fcUjPLBCx1zuU1s+TAV8B9wDog+XltVQO6AEmBTUAz59z/U1MXHrcy0BvfexgBtHLOnTSzf4BRQFXgfTOrAUx0zv0Qx6cecB079aJjp1689mobWj/fjC5d+wS6SwFTpkpJIvceYN3qDRQqWfCydfp1HUiH916mzqM1Wb5oJbu27yYqKpri5YtSvHwRvpn2BQDJUyQn9y25WL54FUMmfkp40iQkT5GcNOnSnKsz4N1BLJodcckxLk6cFi51P/Ua16bFA23i9oQDLCxJGEWqFOPbXpcON4WGhXJLgfx0eextwpOF033sB2xYvp57St/HLffko+cE389peLJwDu31ZUw6DHqDLLmzEhYeRqYcmflg0kcATPrqJ2Z+P+OSY1ws1625afJ6U7o16RyHZynBrF/fwXTv+RYz547jjz82sHrVWqKjoihXviT3FSzAtJm+PynJkidj7559AAz9ZgA33ZSLJOFJyJUrOzPnjgNg8GfDGfHtmBiPefsd+enYpT0NH2x+404skBL4EGpcSfSBnd+tQGPn3LNmNhp4+Cp1WwHHnHN3mtm9wG8A/uDvbaCKc+6omb0GtAO6XtyAmSUDhgKVnXMbzGy4v92P/FX2OecK+evWuFwnzKwF0ALAQtMSEpLyWs85wfhuxBh+mvB1UAd29xYtQNlqpShVuThJk4aTMnVKuvR/i84v/H++1t5d+3jtmY6AL3irWKscRw4dwTCG9f+Wsd9cOom6eZ1WgG+OXZ2GNejatucF2/fvjSRjlgzs272fjFkyELkv8ty2/Hfewlu9O/Byk1c5GHnoRpx2wNxfoTB//76Jg3sPXLJt3469HI48xMnjJzl5/CR/LFlD3jtvxgxm/TCT796/NBj8oKVvsUvmXFlo0/slOje6cD7T/p37yJQjE/t37iMkNIQUqVNyOPIw4MvmvTr4Tfq3+4hdm3fegLOVYHTk8FFebP3mudfLVs3gn3+2UKJkEUaNGMu7XT68ZJ+nmvi+wOXOk5P+A3vwQJ0nL9i+c8cucubMzo7tuwgNDSVNmtTs3+/7zMieIyvDvh1Am5av8c/fW27gmQWOC5LAzitDsX8751b4ny8D8l6lbjngGwDn3Cpglb+8BHAXMN/MVgBNgZuu0Mbt/mNu8L8e5m/3rFExddg5N9g5V8Q5VyQxBnX589987nm9utVZvz64588M7PE5dYs8wgPFG/FWq64snffbBUEdQNoMaTEzAJ564XF+GvULAItmL6Fuo1okT+FLHmfOlon0GdPF6rhzps6ndkPfd4faDWswZ8p8ALLmzEKvL7rR+cX32PzX1jg5x4SkTL2ylx2GBYiYtpg7i95FSGgI4cnCubXgbWzduIXV81dRslYp0mRMC0CqtKnIlDN2cw+XTl9ChYcrAVCyVml+X+D72EiRJiVvftWJb3sNZ/3StXFwZiI+adKmJkkS3zy3Jk0fYeGCpRw5fJQ5sxdSt351MmXKAEC69GnJlTtHrNqcPOlXHn3sQQDqPlD93By+NGlT893owXR7pw9LFv92A84muJhZbjObaWZ/mNkaM3vpou2vmJnzJ5Qwn4/NbKOZrTKzQufVbWpmf/ofTWNzfK9k7E6e9zwK3/DqGf4fuCaLRRsGTHPONY6D/hyNgzYSjG++/oTy5UqSKVMG/vlrKV269qZmzUrcdls+oqOj2bx5G8+3vnTFpkCLDs1Zu3Idc6cuoHDJgr6VsM6xfPFK3n/Tl+BdPHspefPfxJc/DQTg+NHjdHrhXSL3XZqNutjwAd/R/bN3qNeoNju37eTNlu8A8EzbpqRNn5bXevhWaUadiaJpzZY35iTjWdLkSbm3bEEGvTnwXFm1x33B7dRvJ7Nt41aWz/6NPlM+xkU7ZoycxpYNmwEY0fsbOn7tm2x+5swZvug46JLFF5czY9Q0Xuzbjv6zB3HkwGH6tvHNKa3ZtDbZ8manwYuP0uDFRwHo9kTnCxZsBJvLfV7sjzxAv77vkjlzBiaMH87KlWuoddHK2WAz6Ms+lC5TjAwZ07Pyj9m836M/YUl8f5KHDRnJbbflY8BnPXEO1q37k5fb+LLIG9Zvose7H/H92CFYSAhnzpzmtVe6snXL9hiP+e3XPzBw8AcsWT6VyMiDtGju+3x45tkm3HxLHtq/2pr2r7YG4JEHm7N37/4bdPYBEn8ZuzPAK86538wsNbDMzKY55/4ws9xANWDzefVr4ht5vBUoDnwKFDezDEBnoAjg/O1McM5FchWW2Fcz+ufYTXTOFfC/bg+kAnIBy5xzn5rZy8DL/jl27YC7nHPPmFkBYAW+bN2/+LJ9lZxzG80sJZDzvKzc+cdMBmw4r+5QYLlzrp9/jl0R59xef92hxDDHLiw8Z+L+T0gECmXKH+gueF6eJGkD3YWgMG7HskB3wfPSJ08V6C4EhT0H11t8Hu9wm1px9rc29YBJse67mY0HBjjnppnZD0A3YDz+WMHMBgGznHMj/PXXAxXOPpxzLf3lF9S7Eq8MxV5Ob6CVmS0HMp1X/imQyszW4ps/twzAObcHeAoYYWargIXAHZdr2Dl3AmgGfG9mq4Fo4LMbdB4iIiKSgJhZCzNbet7jshcm9Sef7gcWm1l9YJtzbuVF1XIC509s3Oovu1L5VSX6oVjn3D9AgfNe9z5v8/nXjHjbv/040OgKbf0KFI3lcWfg+8+6uDzvRa+fik17IiIicgPF4VCsc24wMPhqdcwsFfAj8DK+4dk38Q3D3lBeztiJiIiI+MTjnSfMLAm+oO5b59wYIB9wM7DSP2UrF/CbmWXDd5m23OftnstfdqXyq1JgFwMzG2tmKy56VA90v0RERCThMd/lD74E1jrnPgRwzq12zmVxzuX1j+xtBQo553YCE4An/atjSwAHnXM7gClANTNLb2bp8WX7psR0/EQ/FHujOeceDHQfRERE5PrE42LR0sATwGr/5dMA3nTOTbpC/UlALWAjcAzfHH6cc/vNrBu+myAAdHXOxbhUWYGdiIiIeF88Xe7EOTcP3yXUrlYn73nPHdD6CvWGAEOu5fgaihURERHxCGXsRERExPuC5JZiCuxERETE83SvWBERERFJVJSxExEREe8LkoydAjsRERHxvuhAdyB+aChWRERExCOUsRMRERHPC5bFEwrsRERExPuCJLDTUKyIiIiIRyhjJyIiIt4XJIsnFNiJiIiI5wXLHDsNxYqIiIh4hDJ2IiIi4n0aihURERHxBg3FioiIiEiiooydiIiIeJ+GYkVERES8wQVJYKehWBERERGPUMZOREREvC9IMnYK7ERERMTzNBQrIiIiIomKMnYiIiLifUGSsVNgJyIiIp6noVgRERERSVSUsRMRERHPC5aMnQI7ERER8TwFdhJvUocnD3QXPG/Fvk2B7oLnzd82J9BdCArJc5QNdBc876YUWQLdBbkRnAW6B/FCc+xEREREPEIZOxEREfE8DcWKiIiIeISL1lCsiIiIiCQiytiJiIiI52koVkRERMQjnFbFioiIiEhiooydiIiIeJ6GYkVEREQ8QqtiRURERCRRUcZOREREPM+5QPcgfiiwExEREc/TUKyIiIiIJCrK2ImIiIjnBUvGToGdiIiIeF6wzLHTUKyIiIiIRyhjJyIiIp6noVgRERERj9C9YkVEREQkUVHGTkRERDxP94oVERER8YhoDcWKiIiISGKijJ2IiIh4XrAsnlBgJyIiIp4XLJc70VCsiIiIiEcoYyciIiKeFyy3FFNgJyIiIp6noVgRERERSVQU2ImIiIjnRTuLs0dMzGyIme02s9/PKytoZovMbIWZLTWzYv5yM7OPzWyjma0ys0Ln7dPUzP70P5rG5jwV2ImIiIjnOWdx9oiFoUCNi8reB7o45woCnfyvAWoCt/ofLYBPAcwsA9AZKA4UAzqbWfqYDqzATkRERCQOOefmAPsvLgbS+J+nBbb7n9cHhjufRUA6M8sOVAemOef2O+cigWlcGixeQosnRERExPMSwKrYl4EpZtYbX2KtlL88J7DlvHpb/WVXKr8qZexERETE8+Jyjp2ZtfDPkzv7aBGLLrQC2jrncgNtgS9vxHkqYxck+g/sQfWaldi7Zx+litW6Yr37C93D1F+/5+mnXmbCuMkaW0S5AAAgAElEQVTXdcx06dMyZFg/8uTJxebNW2n25IscPHCIRxrW46V2LTAzjhw+yisvd+L339dd17ESu6RJk/LrjB9JmjScsLBQxoyZRNdufahYsTQ9e7xNSEgIR44c5Zln27Fp0z+B7m7AnDx5iqatO3Dq9GmizkRRtWIZ2jzzxAV1xv08jT4DvyBLpkwANH64Lg3qxTh6cVUHDx3mlY492L5zFzmyZaVPtzdImyY1v85dSP/PhxNiIYSGhvL6Sy0odF+B6zqWV9x2Wz6++/bTc69vuTkP73Tpzcf9vwhgrxKWkJAQvp78Obt37qXtk69dsK1dlxcoXOp+AJIlT0aGTOmoeMeVP7tjI0261PT4rAvZc2djx5advN6yE4cPHqHGQ1Vp2vpxzODokWP0fL0Pf/6x6bqO5XXOucHA4GvcrSnwkv/598DZX4ZtQO7z6uXyl20DKlxUPiumgyhjFyRGfDuGBg80v2qdkJAQ3un2KjNnzLumtkuXLc4nn/W6pLxtu5bMmbWQIgWrMGfWQtq2awnAv/9uoXaNxyhdvDYf9BpA3/7vXtPxvOjkyZNUq96QIkWrUaRodapVq0CxYoUY0L8HTZ96gaLFqjNy1DjeeP3FQHc1oMLDkzDk456MGTaQH4Z9wvzFy1j5+9pL6tWoVJ4fh33Cj8M+uaagbslvq3jr3T6XlH/x9WhKFCnIpFFfUqJIQb78ZjQAJQoXZMywgfw47BO6vdmWzj37/feT85gNGzb5f56rUax4DY4dO8648b8EulsJSuNnH+HvP/+97LYPO/fn8arNebxqc0YP+ZGZk+bEut3CJQvS+aM3Lyl/qk0TlsxbxkOlH2PJvGU81aYJANs376DFQ21oVOkpvvxoGG998Op/O6EELp4XT1zOdqC8/3kl4E//8wnAk/7VsSWAg865HcAUoJqZpfcvmqjmL7sqBXY3kJnlMLMfAt0PgAXzI4iMPHDVOi2ee5Kfxk9hz559F5S/8NIzzJg9hnmLJvL6Wy9dYe9L1axdhRHfjgF8gWWtOlUBWLJ4OQcPHAIgImIFOXJmu5ZT8ayjR48BkCRJGEmShOGcwzlH6tSpAUibJjU7duwKZBcDzsxIkSI5AGfOnOHMmTOYxf5Ddsi3P/Do0y/y4JOtGPDF17Heb+bchdSvWQWA+jWr8OuchQCkSJH83PGPnzgB19CXYFK5Uhn++utfNm/eFuiuJBhZsmemdOWSjPtuYox1qz1QmSnjpp97/USrxgz7ZTAjZgylRfurf2E/X/nqZZg42jcSM3H0ZCrUKAvAqqW/c/jgEQBWL1tDluyZr+VUEg3n4u4REzMbASwEbjezrWb2NPAs0MfMVgLd8a2ABZgE/AVsBD4Hnvf11+0HugER/kdXf9lVaSgWMLNQ51xUHLcZ5pzbDjSIy3ZvlOzZs1KnXjXq1nycAZ/2PFdesVIZbsmfl8rlH8LMGDF6EKVKF2XB/IgY28ySJRO7du0BYNeuPWTJkumSOk88+QjTp8b+m6iXhYSEsHjRL+TLl5fPPhtGRMRyWj7XgQnjh3P8+AkOHz5MmbL1At3NgIuKiqJh8xfZvG07jR+qw71333FJnWmz57F05Wry5s7Jqy+2JHvWzMxfvIzNW7cx8ot+OOdo81oXlq5YTZGC98R4zH2RB8icKQMAmTKmZ995X5Kmz55Pv8+Gsi/yAAN7d427E/WQhg3rM3LUuEB3I0F5peuLfPzuQFKmTHHVetlyZSVnnhxEzPsNgOLli5L7llw0rembzvLhsJ7cX+I+li9aGeMxM2ROz77dvi/u+3bvI0PmS6+cUb9xHRb8uvg/nJGczznX+AqbCl+mrgNaX6GdIcCQazm2ZwI7M8sLTAaWAYWANcCTQEmgN75zjQBaOedOmtk/wCigKvC+mT0HLAYqAumAp51zc69wrGT4rjNTBDgDtHPOzTSzp4CHgFRAqP9ighOdcwl+0k3399/mnY7v4y76KlKxchkqVSrDnAUTAEiZMiW35MvLgvkRTJv5A0mThpMyZUrSp097rs47HT/g1xmXvnUXt12mXAmaNH2EmlUb3aCzSlyio6MpWqw6adOm4fvRX3D3Xbfz0ovPUq/+k0RELKddu+f44P3OPNeqQ6C7GlChoaH8OOwTDh0+wktvdOPPv/7h1lvyntteoUxxalUtT3h4OKPHTeKtd/swpH9PFkT8xoIlv9HgqTYAHDt+nH+3bKdIwXto/OzLnDp1mmPHj3Pw0GEebur7jG33fHNKF7/wc9jMLsgSVilfmirlS7N0xWoGfD6cL/r1uPFvQiKSJEkS6tapxltv6305q0yVUuzfG8m6VRsoXLLgVetWr1+ZGRNnER0dDUCJ8kUpUb4o307z/a1PkTI5eW7OxfJFKxn68yCShCchRcrkpEmX5lyd/u99xqJZSy5p++LMU+FS91P/sdo8U/+yMUaiF5sLC3uBZwI7v9vxBWTzzWwI0A5oCVR2zm0ws+H4VqV85K+/zzlXCMAf2IU554qZWS18FwWscoXjtMYXZN9jZncAU83sNv+2QsC9zrn9/mDzsvwraFoAJA/PTNIkaa5UNV7cf38Bvhzqe1syZExP1eoVzg1z9e3zGUOHjLxkn6oVfcnI0mWL89jjD9H6uQsn/+7evZesWTOza9cesmbNfMEQ7913387HA7rzyEPNidx/9SHiYHPw4CFmz15A9RoVuefeO4mIWA7A999PYOJP3wS4dwlHmtSpKFboXuYtWnpBYJcu7f9/lx6uW50PB/oXnjl45olHafjApRPQR3zu+9lf8tsqxk+axntvv3LB9ozp07Fn734yZ8rAnr37yZAu7SVtFCl4D1u37yTywEHSX2Z7sKpRoyLLl69m9+69ge5KgnFfsXsoV600pSuXIDxpOKlSp6TrgI50atPtkrrV6lem15t9z702M4b2/4YxX0+4pO5TtX3zmAuXLEidR2vR5eXuF2zfvyeSjFkysm/3PjJmyUjk3shz2/LfmY+OfV7jxcc7cDDyUFydaoJyHXPjEhWvzbHb4pyb73/+DVAZ+Ns5t8FfNgwod179URftP8b/7zIg71WOU8bfPs65dcC/wNnAblpsxsCdc4Odc0Wcc0UCHdQBFCxQkfvursB9d1dgwrjJtG/bmUkTp/Pr9Lk8/kSDc8MF2bNnJVPmDLFqc/KkGTR+/CEAGj/+EL/87JsjkitXdoZ/N5Dnnn2FTRv/uSHnk9hkypSBtP6AJFmyZFSuXJZ16/4kbZo03HrrzQBUrlyOdes2BrKbAbc/8gCHDvvmAp04eZKFEcu5+abcF9TZs/f/v34z5y3iFv/2UsUKMfbnqRw7dhyAXXv2XjCkejUVypRg/C++n9/xv0ynYtmSAGzeuv1cJvqP9Rs5der0BYGlQKNHH9Aw7EU+6T6I2oUfpl6xhrz13DtEzPvtskHdTfnzkDpdalYtPXdXKhbOWkK9RrVJ7p9rmjlbJtJnTBer486eOp86DX2Lieo0rMHsKb6FcllzZuGDL9+l0wvvsvmvLVdrQhIBr2XsLp7SeADIeJX6Ry96fdL/bxT//b25uM0E4Yuv+lK6bHEyZkzP7+vn0fO9fiRJ4jvFr74cccX9Zv46j9vuyMfUX78H4MiRY7R85hX27okxdqXvh4P4avjHNHnyEbZs2UazJ30rOju8/gIZMqSjd98uAJw5E0Wlcg9e7ykmatmzZeXLL/sSGhpKSIjxww8TmTRpBq1avcqokZ8THR1NZORBWrR8JebGPGzPvkjeerc3UdHRuGhH9UplqVC6OAM+H87dd9xGxbIl+Ob78cyat4jQsFDSpk7Nu/7sW+nihfnr3y083rIdACmSJ6NHpw5kTB/zH8VnnmjIKx27M2biFHJky0Kfbr4Vh9NmzWPCLzMICwsjWdJwend9/ZoWc3hdihTJqVK5HK2efy3mykLLDk+zduU65kz15Seq16/M1HEzLqizeHYEN996E19N9F1K5tjR43Rs043IfTF/SRk24Bt6DOpK/ca12bF1F2+07ATAs22bkTZ9Wl7r4fvdiIqK4skaz8blqSUIwTIUaxfPe0qs/MOefwOlnHMLzewL/+uWQCXn3EYzGwosd87188+xK+Kc2+vffxbQ3jm31MwyAUudc3mvcKx2wN3Ouaf9Q7DT8GXsGvvbbHNen2KcY5c+VX5v/CckYEdPnwh0Fzzv6DYtgokPyXOUDXQXPK9gxlsC3YWgsHTH3HiNtBbleCjO/taW2D4mwUaJXhuKXQ+0NrO1QHqgL9AM+N7MVgPRwGdxcJyBQIi/zVHAU865kzHsIyIiInJDeW0o9oxzrslFZTOA+y+ueHE2zjlX4bzne7nKHDvn3Al8AePF5UOBoee9/gdI8CtiRUREvC5YhmK9FtiJiIiIXCJYVsV6JrC7EdkxM6sOXHyvrL+dc8E9019EREQSJM8EdjeCc24Ksbgvm4iIiCRs0YHuQDxRYCciIiKe5wiOoVivrYoVERERCVrK2ImIiIjnRQfJFWMV2ImIiIjnRWsoVkREREQSE2XsRERExPOCZfGEAjsRERHxPF3uRERERMQjgiVjpzl2IiIiIh6hjJ2IiIh4noZiRURERDwiWAI7DcWKiIiIeIQydiIiIuJ5wbJ4QoGdiIiIeF50cMR1GooVERER8Qpl7ERERMTzguVesQrsRERExPNcoDsQTzQUKyIiIuIRytiJiIiI5wXLdewU2ImIiIjnRVtwzLHTUKyIiIiIRyhjJyIiIp4XLIsnFNiJiIiI5wXLHDsNxYqIiIh4hDJ2IiIi4nnBcksxBXYiIiLiecFy5wkNxYqIiIh4hDJ2IiIi4nlaFSvxxoLkoomB5Fyw/EoHTqpc5QPdhaDQMHuxQHfB88bsWhboLsgNECxz7DQUKyIiIuIRytiJiIiI5wXLdewU2ImIiIjnBcuEHA3FioiIiHiEMnYiIiLiecGyeEKBnYiIiHhesMyx01CsiIiIiEcoYyciIiKeFywZOwV2IiIi4nkuSObYaShWRERExCOUsRMRERHP01CsiIiIiEcES2CnoVgRERERj1DGTkRERDwvWG4ppsBOREREPC9Y7jyhoVgRERERj1DGTkRERDxPiydEREREPCI6Dh8xMbMhZrbbzH4/r+wDM1tnZqvMbKyZpTtv2xtmttHM1ptZ9fPKa/jLNprZ67E5TwV2IiIiInFrKFDjorJpQAHn3L3ABuANADO7C2gE3O3fZ6CZhZpZKPAJUBO4C2jsr3tVCuxERETE81wcPmI8lnNzgP0XlU11zp3xv1wE5PI/rw+MdM6ddM79DWwEivkfG51zfznnTgEj/XWvSnPsRERExPMS2KrY5sAo//Oc+AK9s7b6ywC2XFRePKaGlbETERERuQZm1sLMlp73aHEN+74FnAG+vRF9U8ZOREREPC8uV8U65wYDg691PzN7CqgDVHbOnR3V3QbkPq9aLn8ZVym/ImXsRERExPPic47d5ZhZDeBVoJ5z7th5myYAjcwsqZndDNwKLAEigFvN7GYzC8e3wGJCTMdRxk5EREQkDpnZCKACkMnMtgKd8a2CTQpMMzOARc6555xza8xsNPAHviHa1s65KH87bYApQCgwxDm3JqZjK7ATERERz4uOx7vFOucaX6b4y6vUfw947zLlk4BJ13JsBXYiIiLiebrzhIiIiIgkKsrYiYiIiOfF30BsYCmwExEREc/TUKyIiIiIJCrK2ImIiIjnJbBbit0wCuxERETE8+LzcieBpMAuSPQf2INqNSqyd88+Shevfdk6pcsUo3uvt0mSJIx9+yKpW/Px6zpmeHg4nw5+n/sKFiBy/wGaP/USWzZvo0LF0nTq0p7w8CScOnWazm/3Yu6cRTE36GG5cuXgqyH9yJI1E845vvziW/oP+JJ33ulAvbrViI527N69l6efacuOHbsC3d1Ea/36BRw5fJSoqCjOnImiVOnadO7cnrp1qhEdHc2ePft45tl2Qf8e13y6LhUbVcE52LLuXwZ16M/pk6fPbb+j2F080bk5ee7IS/8X+rBk0sLrPmbKtKl48ZNXyJwrC3u27ubj53tz9NBRSj9QjrrPPQhmnDh6nCFvDWLz2n+u+3iJ3fr18zl83s9y6dJ1ePvttjRr1pi9e/cB0KnT+0yZMjPAPU04giOsA/v/rcokUDKkvvWG/yeULF2Uo0eO8ungDy4b2KVJm5op00fT4MHmbNu6g0yZMrB37/5YtZ07T04++awX9Wo1uaC8+TOPcXeBO3jl5U489HBtatetytNPvcw9997Fnt172blzN3feeSvfjxtCgdvLxsl5Xsnhk8dirhRA2bJlIXu2LCxf8TupUqVk8eLJNGjQnK1bd3D48BEA2rRuzp133kbrNq8HuLeXFxKS8Kfsrl+/gFKlarNvX+S5stSpU517j1s/34w777yVNi+8GaguxujhrEVuaPvps2ag84/d6VD5RU6fPMWLn7RnxcxlzPnh/wFCplyZSZ4qBXVa1GfZ9IhrCuzuLHE35RpUYlD7/heUN37jSY4cOMJPn46hbquHSJk2JSN7fs2thW9n+59bOXroKPdVKMTDLz9Kpwdei7PzvZwxu5bd0Pbjwvr18ylVqs4FP8tvv92WI0eO8tFH13wL04A4cWJzvA6OvpX3sTj7W/veP98l2IHdOP0kNrM7zGyFmS03s3yx3Kesma3x75c8LvtzrcxslpnF6aemmX1hZnfFZZv/xcL5EURGHrzi9gaP1OWnCVPZtnUHwAVB3SOP1mPazB+YPX8CH/brFus/4LVqV2Hkd2MAGD9uMuUqlARg9ao/2LlzNwBr1/5J8mTJCA8P/0/n5RU7d+5m+YrfAThy5Cjr1v1JjhzZzgUcAClSpkBfxOLepe9xADuTQISGhhKeLJyQ0BDCkyclcteFX/L2bt3DlnX/Eh196ZtVp+UDdJvwPj0n9+Xhto1ifczCVYsx90df8Dj3x5kUqVYcgD+XrefooaMAbPxtPRmyZ/yvpyVBLjoOHwlZXH/FfgD4wTl3v3NuUyz3eRzo4Zwr6Jw7HlNlM7shw8dmFnoj2nTOPeOc+yOu245r+fPfTLp0aZgw6Rt+nTOWRxs/AMBtt+fjwYdrU7NqI8qXrkdUVBSPPFovVm1mz5GVbVt3AhAVFcWhg0fIkDH9BXXq1a/BypVrOHXqVNyeUCJ20025KHhfAZYsWQ5A166v8demCBo3fpB3unwQ4N4lcs7x88RvWbjgZ55++rFzxV26vMrGjYtp3OhBunTtHcAOBl7krv38PHg8/RcOZmDEEI4fPsrquStjte89Ze8jW97sdKz3Km/UbMfN9+TjjmKx+16bNlM6Duz2ZZ8O7I4kbaZ0l9Sp0KgKK2f9FvuT8TDnHBMnfsOCi36WW7VqSkTEFAYN+oB06dIGsIcJTzQuzh4J2VUDOzNLaWY/m9lKM/vdzB71l3cyswh/2WDzqQW8DLQys5n+ek3MbIk/Gzfo4uDJzJ4BGgLdzOxbfzsf+Ntdfd7xKpjZXDObgO8muee38YiZfeh//pKZ/eV/fouZzfc/r+zPIq42syFmltRf/o+Z9TKz34BHzmszxMyGmtm7V3lvGvvb+93Mep1XfsTM+pjZSqDkjcgC3gihYaEUvL8AjRo8S4MHm9P+1dbky5+XcuVLcl/Bu5kxewyz50+gXIWS5M2bG4Dh333C7PkTGP3jFxS8vwCz509g9vwJPNbk4Vgd84478tO5awfavdTpRp5aopIyZQpGj/qcV9p3PpdJ6tSpF7fkK8qIEWN5/vlmAe5h4lax0sOUKFmLevWf5LmWTSlTxpcV6tz5ffLnL86IkWNp1eqpwHYywFKmSUnhasV4qcxztC72NEmTJ6P0g+Vjte895QpyT9mCdJ/0Ie/93Icc+XKS7ebsAHQd14vukz7k2V6tKVy1KN0nfUj3SR9yb7mCV2jtwj+ed5UsQIVHqzCix9fXc3qeUanSw5QsWZv69Z+kZcsnKVOmGIMHf82dd5alWLEa7Ny5m1693g50NyUAYsp+1QC2O+dqA5jZ2fB/gHOuq7/sa6COc+4nM/sMOOKc621mdwKPAqWdc6fNbCC+7Nzws407574wszLAROfcD2b2MFAQuA/IBESY2Rx/9UJAAefc3xf1cS7wqv95WWCfmeX0P59jZsmAoUBl59wGMxsOtAI+8u+zzzlXyH8uz/nfk2+B3/035b2EmeUAegGFgUhgqpk94JwbB6QEFjvnXvHXvewba2YtgBYAKZJmJmmSwH6z2r59J5H7D3Ds2HGOHTvOwgURFChwB2bGyO/G0u2dPpfs8+RjrYErz7HbsX0XOXNlY/v2nYSGhpImbSr2++eD5MiRjeEjBvJ8yw788/fmG3+CiUBYWBijR33OiBFjGTful0u2jxgxhgkTvqZr10v/LyR2tm/3ZZD37NnH+AmTKVqkIPPmLT63feTIsYwfN5xu3T4MVBcDrkCZ+9i9ZReH9x8CIGLyIm4rfDvzx86OcV8zY/zAH/n1u6mXbDs7L+5Kc+wO7j1AuizpObA7knRZ0nNw7/+njuS+4yae7dWaXk27ceTA4es5Pc/Yvt23wGfPnn1MmDCFIkUKMm/eknPbhwwZwZgxXwWqewlSws6zxZ2YhmJXA1X9Wa2yzrmzv2kVzWyxma0GKgF3X2bfyvgCnwgzW+F/fUsMxysDjHDORTnndgGzgaL+bUsuE9ThnNsJpDKz1EBu4DugHL7Abi5wO/C3c26Df5dh/u1njbqoyUFcJajzKwrMcs7tcc6dwRcInm0zCvgxhvPEOTfYOVfEOVck0EEdwC8/z6B4ycKEhoaSPHkyChe5jw3rNzFn1kLq1a9BpkwZAEiXPi25cueIXZuTZtDosYcAqP9ADebO9q18TZM2NSN/GEzXzr1ZvEjDKmd9PrgP69Zt5KN+/5/4nD//zeee16tbnfXrYzvDQS6WIkVyUqVKee55lcrlWLNmPfnz5T1Xp26daqxfvzFAPUwY9m7fw63330Z4Mt+817tL38u2jVtjte+q2cup0LAySVMkA3wLMdJkjN3n22/TIyj7cEUAyj5ckWXTfEFKxhyZaDvoNQa2/Yidf2+/1tPxpIt/litXLsuaNevJli3LuTr16lVnzZr1gepighQsc+yumrHzZ7gKAbWAd81sBvA+MBAo4pzbYmbvAMkus7sBw5xzb8RRX49eZdsCoBmwHl8w1xwoCbwC5L3GdhfgC1z7OOdO/Id+nnDORf2H/W6oz4f0pXTZYmTMmJ7f182lZ/d+hIUlAWDokBFsWL+JX6fPZd6iiURHR/P1sO9Zu/ZPALp368uP44cSEmKcPn2GV1/pwtYtMX/AfjP8ez77vDdLV0wnMvIAzzRrC8CzLZ7g5ltuosNrbejwWhsAHq7/VKxX4XpR6VJFadKkAatX/8HSCF+24+2OPWnWrBG33ZYPFx3Nv5u30bp1wlwRmxhkzZqZ0aM+ByAsLJSRo8YzddosRo4YxG235SM6OprNm7cm6BWx8WHTij9ZPGkh3X/uQ1RUNP+s+Ytfv5tKg3aN+WvVRn6bHsEt9+an7eDXSJk2FYWqFKVB20a8WvUlVs9dSc78uekyticAJ4+d4JOXPuLQvisv3DprwsAxvDiwPRUfrczebXvo97xvruNDLzUkdfrUNOvWEoDoqCjertvhxr0BiUDWrJkZNcr3BTAsLIxRo8Yxbdpshgz5iHvvvQvnHP/+u5U2beLqz68kJle93Il/yHG/c+6EmdUBngGewhdA5QVCgUX4Fky84w/yzg7F3gWMxzcUu9vMMgCpnXP/XnSMofx/KPYhoCW+QDIDsBQoDtwBtHfO1blCP58CuvofXwG/A8edc4X8Q7EbgErOuY3+4y13zvUzs3/wBah7/e3MAtrjy75VAB7yZ+QuPl52/3mfHYqdAvR3zo03syPOuVTn1Z3l7/vSK73P8XG5k2CX0C934gWJ4XInXnCjL3ciieNyJ14Q35c7aZe3UZz9rf3wn5EJ9nInMc2xuwf4wMyigdNAK+fcATP7HF/wtBOIuNyOzrk/zOxtfPPPQvz7twb+vVx9v7H4Mm0r8Q2Hv+qc22lmd8TQz7n4hmHnOOeizGwLsM7fjxNm1gz43r+iNgL47GqNOec+9M8n/NrMHnfORV+0fYeZvQ7MxJeZ/Nk5Nz6GPoqIiEiABEsGRRcoTgCUsbvxlLG78ZSxix/K2N14ytjFj/jO2LWNw4xd30ScsRMRERFJ9BL6ooe4osAuBma2GEh6UfETzrnVgeiPiIiIXDsXJIOxCuxi4JwrHug+iIiIiMSGAjsRERHxPA3FioiIiHhEQr/Ha1zRMjYRERERj1DGTkRERDwvOPJ1CuxEREQkCGgoVkREREQSFWXsRERExPO0KlZERETEI4LlAsUaihUREZH/tXfncVYUd7/HP9+ZYQcRREAE12hMonEJBolrXqhxjea6JW5Rk8s1MYneq0k0+iQa9XGLS4zGiHHXoGiiuIvRh0SRVUEREURFQSGg4gLINvO7f3QBh2HOzABntp7ve17nNd11uquqq5dTp6q6j+WEW+zMzMws99wVa2ZmZpYT7oo1MzMzsxbFLXZmZmaWe+6KNTMzM8uJqnBXrJmZmZm1IG6xMzMzs9xrHe11rtiZmZlZK+DfijUzMzOzFsUtdmZmZpZ7reU5dq7YmZmZWe61lseduCvWzMzMLCfcYmdmZma511punnDFzszMzHKvtYyxc1esmZmZWU64xc7MzMxyr7XcPOGKnZmZmeVe+LdizczMzKwlcYudmZmZ5Z7virVG89nSxU2dBbMNVlXVWkawNK1hc8Y1dRZyb8HgnZs6C9YAWssVyl2xZmZmZjnhFjszMzPLvdbyHDtX7MzMzCz3WssYO3fFmpmZmeWEW+zMzMws91rLc+xcsTMzM7Pc812xZmZmZtaiuGJnZmZmuRcl/KuLpI0lPSjpDUlTJQ2U1F3SM5LeTPFiQgEAABenSURBVP+7pWUl6XpJMyS9Kmm3DdlOV+zMzMws96qIkr3q4Y/AUxGxA7AzMBU4F3g2IrYDnk3zAAcD26XXYOCmDdlOV+zMzMzMSkRSV2Af4FaAiFgWEZ8ARwB3psXuBI5M00cAd0VmDLCxpM3WN31X7MzMzCz3IqJkrzpsDcwHbpc0UdJfJXUCekXEnLTMXKBXmt4cmFWw/uwUtl5csTMzM7PcK2VXrKTBkiYUvAYXJFUB7AbcFBG7AotY3e0KQGS1wwZ5/oofd2JmZma5V8qfFIuIIcCQIm/PBmZHxNg0/yBZxe4/kjaLiDmpq3Veev99oF/B+n1T2Hpxi52ZmZlZiUTEXGCWpC+noEHA68AjwA9T2A+B4Wn6EeDkdHfsHsCnBV2268wtdmZmZpZ7VY37yxM/B+6V1BZ4GziVrDFtmKQfAe8Cx6ZlnwAOAWYAi9Oy680VOzMzM8u9xqzWRcQkoH8Nbw2qYdkAzihV2u6KNTMzM8sJt9iZmZlZ7tXzwcItnit2ZmZmlnutpWLnrlgzMzOznHCLnZmZmeVePX4xIhdcsTMzM7Pcc1esmZmZmbUobrEzMzOz3CvlT4o1Z67YmZmZWe61ljF27oo1MzMzywm32JmZmVnutZabJ1yxMzMzs9xzV6yZmZmZtShusTMzM7Pcc1esmZmZWU60lseduCvWzMzMLCfcYmdmZma5V9VKbp5wxc7qdMuQqzn0kP2ZN/9Ddtl1EADdum3M0HtvYsst+/Huu7P4/vGn88knnzZxTvNjxvQxfL5wIZWVVaxYsYI9Bh7S1Flq8fr27cPtt/2Rnr16EBHc+td7+dMNt3LvvTfx5e23BaBr14349NPP6L/7gU2c23z4zoH7cc01v6e8rIzbbh/KlVfd2NRZaha0cQ/an3w26tINCJaPeorlI4fXuGzZFtvR8exrWHL75ayYNGrDEu7YmQ6nnUdZ955UfTyPL269DL5YSMVOe9D2sJMgqqCqiqUP3kzl269vWFrNUGvpilVruf23Oatou3mz3gl77zWAhQsXcfvtf1xVsbv8svP5+ONPuPKqG/nVL8+gW7eunPeb/27inObHjOljGDDwYD76aEFTZ6Xe1NQZqEPv3j3ZrHdPJk56jc6dOzF27FMcffRpTJ365qplrrzit3z62Wdceul1TZjT2jXri0WBsrIypk55noMO+QGzZ89hzOgnOPGkn65R3s3VgsE7N2j82qgb2qg7VbPfgnYd6PTr6/liyO+pmjur2oJldPjZpbBiGctHj6h3xa58u51oM2B/ltxz7Rrh7Y44jVj8OcueeYC2BxwDHTuzbPjt0LY9LFsCQFmfrWh/2nksvuT/lGRba9Plhica9bLxtV4DSnb6TPnP2GZ7yWv2Y+wkvdjA8c+U1KMB4/+rpK82VPyN4fkXxvLxgk/WCDv88O9w190PAHDX3Q/w3e8e1BRZM6u3uXPnMXHSawAsXLiIN954kz59eq+xzNFHH87999fccmLr5pu778pbb83knXfeY/ny5QwbNpzvHv6dps5WsxCfLcgqdQBLv6By7nto47U/htrsezgrXhlFfL7m9bfNoKPo+Mvr6HjejbQ95IR6p1vx9T1YPvafACwf+0/afH1g9kaq1AHQrj0t5+vDuqmKKNmrOWv2FbuI+FZT52F9SSqPiB9HRO7atHv17MHcufOA7AOzV88Gqxu3ShHBk08MZeyYJ/nxj+p/4bb62XLLvuyy846MGzdxVdheew1g3rz5zJjxThPmLD/6bN6bWbM/WDU/+/05a1WkDdS9J+V9t6Vy5htrhnfdhIqdv8Xy5x9fI7x8h10p69mHxVedxeLLf0b5FttRvu2O9Uury8bEZ1kvQHy2AHXZeNV7FV8fSMcLbqbj6Rex5N7m22K9IaKEf81ZScfYSeoEDAP6AuXAxcAM4BqgM/AhcEpEzJE0EpgI7A10Ak4GzgN2Au6PiAtSnAsjonOR9PYDLkzx7gi8BJwYESFpJtA/Ij6U1B/4Q0TsJ2kTYCiwOTCagh4kSScCvwDaAmOBn0ZEZZG0fwD8Jq3/eET8emV+gZuB/YEzJF0CnBMRE+pfki2Pu/RLa99vf48PPpjLpptuwlNP3se0aTN4/oWxTZ2tXOjUqSPD7r+Fs8/5HZ9/vnBV+PePO5L73Fpnjaltezr8+HyW/n0ILPlijbfaHTWYpcNvg2rX1oqv7EbFDrtRfu6fAFC7DpT17EPlW6/R8ZxroaICteuAOnahY1pm6fDbqZz6cg0ZWB33ildHs+LV0ZRvuyPtDj2JL244v7Tbao2m1DdPHAR8EBGHAkjqCjwJHBER8yUdB1wKnJaWXxYR/SWdCQwHvgF8DLwl6dqI+Kgeae4KfA34ABgF7Am8UMvyvwNeiIjfSzoU+FHK61eA44A9I2K5pD8DJwB3VY9AUh/gipTfBcAISUdGxMNkldSxEXF2WrbGTEgaDAwGUHlXyso61WNTm4//zPuQ3r17MnfuPHr37sm8+fXZVVZfH3wwF4D58z9i+PAn2X33XVyxK4GKigqG3X8LQ4c+xMMPP7kqvLy8nCOPPJgBexzchLnLlw/en0u/vn1WzffdfLNVx7UBZeV0+N/ns3zCSFa8svaIo/IttqPDqecCoM4bUf613aGqChDLRgxj+agn11pn8R/+b7ZukTF28fknaKNuWWvdRt2Iz9e+4a3yrdco69EbddqIWPRZCTa0+WjuXailUuqu2MnAAZKukLQ30I+sJe0ZSZOAC8ha81Z6pGC9KRExJyKWAm+ndetjXETMjogqYBKwVR3L7wPcAxARj5NVzAAGkVXUxqe8DgK2KRLH7sDIiJgfESuAe1O8AJXA3+vKdEQMiYj+EdG/pVXqAB57dAQnn3QMACefdAyPPvp0E+coPzp27EDnzp1WTR+w/75MmTKtiXOVD7cMuZo33pjBdX8cskb4oEF7M23aDN5/f04T5Sx/xk+YxJe+tDVbbdWPNm3acOyxR/DoYyOaOlvNRvsTzqJq7iyWP/dQje8vuvA0Fv3uVBb97lRWTHyBpfffmLWqTX2JNgMPzG54IOuyVeeu9UpzxeQxtBmwPwBtBuzPilfHZHH02GzVMmV9t4WKNrmr1IG7YtdLREyXtBtwCHAJ8BxZhW1gkVWWpv9VBdMr5+ubt8L1KgvWW8Hqimv7esQj4M6IOK+e6RazpFj3bUt1z903su8+A+nRozsz357ARb//A1dcdSP3/e0vnHrKD3jvvdl8//jTmzqbudGr16Y8+MCtAFRUlHPffQ/z9IiRTZupHNjzW7tz4olHM3ny60wYn1UwLvivy3nqqec47tgjfNNEiVVWVnLmWRfwxON/o7ysjDvuvJ/XX5/e1NlqFsq3+SptBgyi8v13VneXPnInZd17ArD8hSeKrlv5xkSW996CjudckwUs/YIv7rwKFtb9uKmlzzxAh9POo9PAA7PHndx2GQBtdtmTigGDoHIFLF/Gktsu38AttKZU0sedpC7KjyNiiaTDgJ8C2wMnRcRoSW2A7SNiShpjd05ETEhj5c6JiMNSPIXv1TXGrnC9G4AJEXGHpH8CV0fEk5KuBXZNY+yuB+ZFxCWSDgaeADYFepJ1B+8ZEfMkdQe6RMS7NaS7GTCG1V2xTwN/iojh1fNbuC3Fyq25P+7ErD6a7b3/OeOLRcNr6MedWKaxH3eybY/dSnb6vPXhy832klfqMXY7AVdJqgKWAz8hazm7Po23qwCuA6aUON2aXATcKuliYGS18KGSpgAvAu8BRMTrki4gGy9XlvJ/BrBWxS7d/HEu8D+svnnCX/fNzMyaqebehVoqfkBxM+AWO8uDZvv1NWd8sWh4brFrHI3dYrdNj11Ldvq8/eHEZnvJ80+KmZmZWe5l91jmX4uo2EnaCbi7WvDSiBjQCGmPBdpVCz4pIiY3dNpmZmZWGlWtpL27RVTsUiVqlyZKu8Erj2ZmZmal0CIqdmZmZmYborXcU+CKnZmZmeVea+mKLfUvT5iZmZlZE3GLnZmZmeWeu2LNzMzMcqKqlVTs3BVrZmZmlhNusTMzM7Pcay0/KeaKnZmZmeVeaxlj565YMzMzs5xwi52ZmZnlXmt5jp0rdmZmZpZ77oo1MzMzsxbFLXZmZmaWe63lOXau2JmZmVnuuSvWzMzMzFoUt9iZmZlZ7vmuWDMzM7OccFesmZmZmbUobrEzMzOz3PNdsWZmZmY5ER5jZ2ZmZpYPraXFzmPszMzMzHLCLXZmZmaWe63lrlhX7MzMzCz3WssYO3fFmpmZmeWEK3ZmZmaWexFRslddJB0kaZqkGZLObYTNW8VdsWZmZpZ7jTXGTlI5cCNwADAbGC/pkYh4vTHSd4udmZmZWel8E5gREW9HxDLgPuCIxkrcFTszMzPLvSjhqw6bA7MK5mensEbhrthmYMWy99XUeVhXkgZHxJCmzkeeuYwbh8u54bmMG57LuG6l/KyVNBgYXBA0pLmUv1vsbH0NrnsR20Au48bhcm54LuOG5zJuRBExJCL6F7wKK3XvA/0K5vumsEbhip2ZmZlZ6YwHtpO0taS2wPeBRxorcXfFmpmZmZVIRKyQ9DPgaaAcuC0ipjRW+q7Y2fpqFmMJcs5l3Dhczg3PZdzwXMbNSEQ8ATzRFGmrtfx2mpmZmVneeYydmZmZWU64YmdmZmaWE67Y5ZCk0yWdXKK4frMe65wi6YZSpF/P9C6UdE5jpVcKkkZK6u985Jek/SQ91oTp7yBpkqSJkrat5zp7S5qS1uvQ0HmsIy9NdmxKmimpRyOk82IDx98o29FS8tFauGKXQxHxl4i4q0TRrXPFrhQk+cae9eSyy6Tfayx1nJLUUq6bRwIPRsSuEfFWPdc5AbgsInaJiC/qWrihjrUG2nfN7ryIiG81dR7WV3MsT8u0lAtUqyfpYUkvpW/Tg1PYQkmXSnpF0hhJvVL4qhas9K33WkkTJE2VtLukf0h6U9IldcR/OdAhfXu/N4WdKGlcCrt55QVY0qmSpksaB+xZx7bcIekvKU/TJR2Wwk+R9Iik54BnU9gvJY2X9KqkiwriOD+t+wLw5VKVc31J2iqV5y2pzEZI6lDYyiCph6SZabqDpPvSOg8BHQriOlDSaEkvS3pAUuda0p0p6UpJk9N++FIKX1mmY4ErJXWSdFtaZqKkI+rKR3OUyvkNSfemPD8oqaOkQWm7JqftbJeWnynpCkkvA8ek/XFFKofpkvauJa1TJA1P67wp6XcFeZgm6S7gNaBfsX0m6aCU35eB/1Xisugk6fF0vr8m6bgU/tt0jrwmaYgyhwBnAT+R9D9puRrP3YL4fwwcC1ycyluSrkrxTi5Ibz9Jz0t6BHi9WhzHSLomTZ8p6e00vY2kUWm6XvuuIM6ydHxfQhHKroXXpnPxWUmbpvCRkq6TNAE4U9Kmkv6eymu8pD3TcpsoO4enSPorsNYvFNRU/pK+Ielfyq6dT0varCDd+lx3F9ayTfuleB4sOAdUUFY90nR/SSPr2o669n9jl6c1oIjwqwW8gO7pfweyD5dNyH6y7vAUfiVwQZq+EDgnTY8ErkjTZwIfAJsB7ch+v26TYvGn+YUFefgK8CjQJs3/GTg5xfcesCnQFhgF3FDLttwBPEX2xWK7lI/2wClpemVeDiS7hV9p2ceAfYBvAJOBjsBGwIyV29uI+2MrYAWwS5ofBpyYyrt/CusBzEzT/4/sWUYAX0/r9k/L/BvolN77NfDbWtKdCZyfpk8GHiso08eA8jT/38CJaXpjYDrQqVg+mvr4rqOcA9gzzd8GXED2O4zbp7C7gLMKyudXBeuPBK5O04cA/6wlrVOAOWTn1srzoH/KQxWwR8F+XWufpWN4VjqmlY6Jx0pYFkcBtxTMdy08d9P03ay+JlzI6utAjedukXPz6IL0niF7DlcvsnN8M2A/YBGwdQ3r9wbGp+kHyR7UujnwQ+CygjKq777bAxhKOuZrKZsATkjTvyVdf1Icfy5Y7m/AXml6C2Bqmr6edN4Bh6b4etRV/sCLwKZp/jhWn1sjqd91d2Et27Qf8CnZrxaUAaML8j5zZf7IjtGRtW1Hffd/Y5anXw33clNqy/ELSd9L0/3IPjyWkX2YA7wEHFBk3ZVPvJ4MTImIOQDp23Q/4KMi8X9ULZ5BZJWq8emLYwdgHjCA7MIyP8V7P7B9HdszLCKqgDdTPnZI4c9ExMdp+sD0mpjmO6d8dQEeiojFKb1Ge6J3Ne9ExKQ0/RJZBaCYfcgudkTEq5JeTeF7AF8FRqUybUt2Aa/N0IL/1xaEPxARlWn6QOC7Wj32sD3ZhbdYPpqzWRExKk3fA/wXWdlPT2F3AmcA16X5+6ut/4/0v659BNnx9xGApH8AewEPA+9GxJi0TLF9tkPK15tp/Xso7c88TQaulnQFWYXx+RT+bUm/Ivui0x2YQvYhXqjYuVubvYCh6Zj6j6R/AbsDnwHjIuKd6itExFxJnSV1IbuO/I3smNubbD98mXXbdzeTXSsurSOvVQXr3sPqfV49zv2Br6YyANhIWWvrPqQW1oh4XNKCGtJYo/yBBcCOwDMpvnKyLwYr1ee6W5dxETE7rTeJ7Ph9oZbli23Huu7/xihPayCu2LUAkvYjO4EGRsTi1OzeHlge6SsRUEnx/bk0/a8qmF45X1FL/GtlBbgzIs6rlr8j13WbyL7B1TS/qFp6l0XEzdXSO2s90msIhWVZSXaxXMHqIQ41lWF1IqtM/GAd0o0i09XL7qiImLZGYmqRPSLVj5VPyFrVillUbX7lfqrtHCmWVrHjcq19JmmXOuLeIBExXdJuZC2Pl0h6lqyl/s9kra6zJF3IOpy7G6B6GRd6ETgVmAY8D5wGDATOpu6KdfV4XySruF4dEUvWIX/FzosyspbXNeKqz3lRvfyB58gqbAOLrFLrdbfOBNeMA9Y8ftfnOrMh+7/k5WkNx2PsWoauwIJU6dqBrMWgseJfLqlNmn4WOFpSTwBJ3SVtCYwF9k3jKtpQMD6mFscoGzuzLbAN2YdAdU8Dp2n1+KXNU9r/Bo5UNl6sC3D4um9yg5lJ9s0Y4OiC8H8DxwNI2pGsGxRgDLCnVo+V6ySprtbO4wr+F2vdexr4ecGYnF3ryEdztoWklR+exwMTgK1WlhlwEvCvEqV1QDquO5DdfDCqhmWK7bM3Ur5W3oG6LpX1OknqAyyOiHuAq4DdWP2h/mE6T44usnqxc7c2zwPHSSpPY6z2AcbVI6vPA+eQHWsTgW8DSyPiU7LzfF323a1kT+8fptoH65exetuPp3ir1gjg5ytnCirjhefFwUC36ivWUP4DgE1XHpuS2kj6Wi15LKWZrL7OHFUQXmw71nX/N3h5WsNxi13L8BRwuqSpZBfGMXUsX8r4hwCvSno5Ik6QdAEwQtmdgcuBMyJiTGopGE3WmjKJur1H9iGxEXB6RCyp/i0vIkZI+gowOr23kGzc2Mupu/cVsu6E8eu74Q3gD2QfQoOBxwvCbwJuT2U8laxbkIiYL+kUYKjSIHKyMWTTKa5b6kJdSvHKw8Vk3Vuvpn31DnBYsXw0c9OAMyTdRjZY/xdkx+gD6cN+PPCXEqU1Dvg72bimeyJigqStChcots9Si85g4HFJi8kqOF1KlC+AnYCrJFWRnXs/iYhPJN1CNh5wLkXOhYh4vaZzF3i3lvQeImtpe4WsxeZXqat1h1rWgWy7+wH/johKSbPIKr2k8/xU1mHfRcQ1kroCd0s6IQ3hqG4R8M20jfNY/eWnul8AN6bzp4KsAnI6cBHZ/pxC1kr4Xg3rrlX+ZC1n16f8VZCdc43xm6AXAbdKuphs3Fth+FrbsR77vzHK0xqIf1LMGp2kO8jGCD3Y1HlpaZTdZds/Ij5s6rw0hlSpeiwidmyEtE4hK9ufNXRaVlqSFkZE0bvJbd24PFs2d8WamZmZ5YRb7KzBSDqftcfbPVCPO9xaPWXPmNu6WvCvI+LppshPnkj6DnBFteB3IuJ7NS1vzYey5zS2qxZ8UkRMbor8lIKkncgeU1NoaUQMaIS0c1ee5oqdmZmZWW64K9bMzMwsJ1yxMzMzM8sJV+zMzMzMcsIVOzMzM7OccMXOzMzMLCf+P7fMsa6wE/gTAAAAAElFTkSuQmCC\n", 225 | "text/plain": [ 226 | "
" 227 | ] 228 | }, 229 | "metadata": { 230 | "needs_background": "light" 231 | }, 232 | "output_type": "display_data" 233 | } 234 | ], 235 | "source": [ 236 | "import seaborn as sn\n", 237 | "import pandas as pd\n", 238 | "import matplotlib.pyplot as plt\n", 239 | "%matplotlib inline\n", 240 | "df_cm = pd.DataFrame(conf_matrix, index = ['animated_ori' , 'nude_ori' , 'porn_ori' , 'safe for work_ori' , 'semi_nude_ori'],\n", 241 | " columns = ['animated_pred' , 'nude_pred' , 'porn_pred' , 'safe for work_pred' , 'semi_nude_pred'])\n", 242 | "plt.figure(figsize = (10,7))\n", 243 | "sn.heatmap(df_cm, annot=True)" 244 | ] 245 | }, 246 | { 247 | "cell_type": "markdown", 248 | "metadata": {}, 249 | "source": [ 250 | "

Cannot show wrong examples . You know it why" 251 | ] 252 | }, 253 | { 254 | "cell_type": "code", 255 | "execution_count": 10, 256 | "metadata": {}, 257 | "outputs": [], 258 | "source": [ 259 | "wrong_examples = df[df['ground_truth_label'] != df['Predicted']].sample(n=9)" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": 11, 265 | "metadata": { 266 | "scrolled": false 267 | }, 268 | "outputs": [], 269 | "source": [ 270 | "# from IPython.display import Image\n", 271 | "# from IPython.display import display\n", 272 | "\n", 273 | "\n", 274 | "\n", 275 | "# for i in range(9):\n", 276 | "# x= Image(wrong_examples.iloc[i,0])\n", 277 | "# display(x)\n", 278 | "\n", 279 | "# ;\n", 280 | "\n" 281 | ] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": null, 286 | "metadata": {}, 287 | "outputs": [], 288 | "source": [] 289 | } 290 | ], 291 | "metadata": { 292 | "kernelspec": { 293 | "display_name": "Python 3", 294 | "language": "python", 295 | "name": "python3" 296 | }, 297 | "language_info": { 298 | "codemirror_mode": { 299 | "name": "ipython", 300 | "version": 3 301 | }, 302 | "file_extension": ".py", 303 | "mimetype": "text/x-python", 304 | "name": "python", 305 | "nbconvert_exporter": "python", 306 | "pygments_lexer": "ipython3", 307 | "version": "3.5.2" 308 | } 309 | }, 310 | "nbformat": 4, 311 | "nbformat_minor": 2 312 | } 313 | -------------------------------------------------------------------------------- /batch_transform.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "This notebook contains code that evaluates the model by supplying images through a batch tranform.\n", 8 | "You will need three things for this notebook to run.
\n", 9 | "1. **trainining Job Name (name of the job with which you have trained the model).**\n", 10 | "2. **url of the location in s3 where images are uploaded.**\n", 11 | "3. **url in the s3 where output is to be stored.**\n", 12 | "\n", 13 | "You can then copy them to your computer and follow analyze.results.ipynb to analyze the results.\n" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 1, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "import os\n", 23 | "\n", 24 | "import time\n", 25 | "from time import gmtime, strftime\n", 26 | "\n", 27 | "import numpy as np\n", 28 | "import pandas as pd\n", 29 | "\n", 30 | "import matplotlib.pyplot as plt\n", 31 | "\n", 32 | "from sklearn.datasets import load_boston\n", 33 | "import sklearn.model_selection\n" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 2, 39 | "metadata": {}, 40 | "outputs": [], 41 | "source": [ 42 | "import sagemaker\n", 43 | "from sagemaker import get_execution_role\n", 44 | "from sagemaker.amazon.amazon_estimator import get_image_uri\n", 45 | "\n", 46 | "# This is an object that represents the SageMaker session that we are currently operating in. This\n", 47 | "# object contains some useful information that we will need to access later such as our region.\n", 48 | "session = sagemaker.Session()\n", 49 | "\n", 50 | "# This is an object that represents the IAM role that we are currently assigned. When we construct\n", 51 | "# and launch the training job later we will need to tell it what IAM role it should have. Since our\n", 52 | "# use case is relatively simple we will simply assign the training job the role we currently have.\n", 53 | "role = get_execution_role()" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 3, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "container = get_image_uri(session.boto_region_name, 'image-classification', repo_version=\"latest\")" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 4, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "\n", 72 | "job_name = 'image-classification-2019-09-12-17-02-05-917'\n", 73 | "training_job_info = session.sagemaker_client.describe_training_job(TrainingJobName=job_name)\n", 74 | "\n", 75 | "model_artifacts = training_job_info['ModelArtifacts']['S3ModelArtifacts']" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 5, 81 | "metadata": {}, 82 | "outputs": [], 83 | "source": [ 84 | "model_name = job_name + \"-model\"\n", 85 | "\n", 86 | "# We also need to tell SageMaker which container should be used for inference and where it should\n", 87 | "# retrieve the model artifacts from. In our case, the xgboost container that we used for training\n", 88 | "# can also be used for inference.\n", 89 | "primary_container = {\n", 90 | " \"Image\": container,\n", 91 | " \"ModelDataUrl\": model_artifacts\n", 92 | "}\n", 93 | "\n", 94 | "# And lastly we construct the SageMaker model\n", 95 | "model_info = session.sagemaker_client.create_model(\n", 96 | " ModelName = model_name,\n", 97 | " ExecutionRoleArn = role,\n", 98 | " PrimaryContainer = primary_container)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 6, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "transform_job_name = 'nsfwbatchtransform' + strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n", 108 | "\n", 109 | "# Now we construct the data structure which will describe the batch transform job.\n", 110 | "transform_request = \\\n", 111 | "{\n", 112 | " \"TransformJobName\": transform_job_name,\n", 113 | " \n", 114 | " # This is the name of the model that we created earlier.\n", 115 | " \"ModelName\": model_name,\n", 116 | " \n", 117 | " # This describes how many compute instances should be used at once. If you happen to be doing a very large\n", 118 | " # batch transform job it may be worth running multiple compute instances at once.\n", 119 | " \"MaxConcurrentTransforms\": 1,\n", 120 | " \n", 121 | " # This says how big each individual request sent to the model should be, at most. One of the things that\n", 122 | " # SageMaker does in the background is to split our data up into chunks so that each chunks stays under\n", 123 | " # this size limit.\n", 124 | " \"MaxPayloadInMB\": 6,\n", 125 | " \n", 126 | " # Sometimes we may want to send only a single sample to our endpoint at a time, however in this case each of\n", 127 | " # the chunks that we send should contain multiple samples of our input data.\n", 128 | " \"BatchStrategy\": \"MultiRecord\",\n", 129 | " \n", 130 | " # This next object describes where the output data should be stored. Some of the more advanced options which\n", 131 | " # we don't cover here also describe how SageMaker should collect output from various batches.\n", 132 | " \"TransformOutput\": {\n", 133 | " \"S3OutputPath\": \"s3://project-completion-udacity/nsfw_dataset/batch-transform/\"\n", 134 | " },\n", 135 | " \n", 136 | " # Here we describe our input data. Of course, we need to tell SageMaker where on S3 our input data is stored, in\n", 137 | " # addition we need to detail the characteristics of our input data. In particular, since SageMaker may need to\n", 138 | " # split our data up into chunks, it needs to know how the individual samples in our data file appear. In our\n", 139 | " # case each line is its own sample and so we set the split type to 'line'. We also need to tell SageMaker what\n", 140 | " # type of data is being sent, in this case csv, so that it can properly serialize the data.\n", 141 | " \"TransformInput\": {\n", 142 | " \"ContentType\": \"application/x-image\",\n", 143 | " \"SplitType\": \"None\",\n", 144 | " \"DataSource\": {\n", 145 | " \"S3DataSource\": {\n", 146 | " \"S3DataType\": \"S3Prefix\",\n", 147 | " \"S3Uri\": 's3://project-completion-udacity/evaluation/testing/',\n", 148 | " }\n", 149 | " }\n", 150 | " },\n", 151 | " \n", 152 | " # And lastly we tell SageMaker what sort of compute instance we would like it to use.\n", 153 | " \"TransformResources\": {\n", 154 | " \"InstanceType\": \"ml.m4.xlarge\",\n", 155 | " \"InstanceCount\": 1\n", 156 | " }\n", 157 | "}" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": 7, 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [ 166 | "transform_response = session.sagemaker_client.create_transform_job(**transform_request)" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": null, 172 | "metadata": {}, 173 | "outputs": [ 174 | { 175 | "name": "stdout", 176 | "output_type": "stream", 177 | "text": [ 178 | "..........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................." 179 | ] 180 | } 181 | ], 182 | "source": [ 183 | "transform_desc = session.wait_for_transform_job(transform_job_name)" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": null, 189 | "metadata": {}, 190 | "outputs": [], 191 | "source": [] 192 | } 193 | ], 194 | "metadata": { 195 | "kernelspec": { 196 | "display_name": "Python 3", 197 | "language": "python", 198 | "name": "python3" 199 | }, 200 | "language_info": { 201 | "codemirror_mode": { 202 | "name": "ipython", 203 | "version": 3 204 | }, 205 | "file_extension": ".py", 206 | "mimetype": "text/x-python", 207 | "name": "python", 208 | "nbconvert_exporter": "python", 209 | "pygments_lexer": "ipython3", 210 | "version": "3.5.2" 211 | } 212 | }, 213 | "nbformat": 4, 214 | "nbformat_minor": 2 215 | } 216 | -------------------------------------------------------------------------------- /benchmark.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "All the procedure is same as batch-transform.ipynb . see batch-transform.ipynb." 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | " **for doing a batch transform on the dataset from [benchmark](https://towardsdatascience.com/comparison-of-the-best-nsfw-image-moderation-apis-2018-84be8da65303)**" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 1, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import os\n", 24 | "\n", 25 | "import time\n", 26 | "from time import gmtime, strftime\n", 27 | "\n", 28 | "import numpy as np\n", 29 | "import pandas as pd\n", 30 | "\n", 31 | "import matplotlib.pyplot as plt\n", 32 | "\n", 33 | "from sklearn.datasets import load_boston\n", 34 | "import sklearn.model_selection\n" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 2, 40 | "metadata": {}, 41 | "outputs": [], 42 | "source": [ 43 | "import sagemaker\n", 44 | "from sagemaker import get_execution_role\n", 45 | "from sagemaker.amazon.amazon_estimator import get_image_uri\n", 46 | "\n", 47 | "# This is an object that represents the SageMaker session that we are currently operating in. This\n", 48 | "# object contains some useful information that we will need to access later such as our region.\n", 49 | "session = sagemaker.Session()\n", 50 | "\n", 51 | "# This is an object that represents the IAM role that we are currently assigned. When we construct\n", 52 | "# and launch the training job later we will need to tell it what IAM role it should have. Since our\n", 53 | "# use case is relatively simple we will simply assign the training job the role we currently have.\n", 54 | "role = get_execution_role()" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 3, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "container = get_image_uri(session.boto_region_name, 'image-classification', repo_version=\"latest\")" 64 | ] 65 | }, 66 | { 67 | "cell_type": "code", 68 | "execution_count": 4, 69 | "metadata": {}, 70 | "outputs": [], 71 | "source": [ 72 | "\n", 73 | "job_name = 'image-classification-2019-09-12-17-02-05-917'\n", 74 | "training_job_info = session.sagemaker_client.describe_training_job(TrainingJobName=job_name)\n", 75 | "\n", 76 | "model_artifacts = training_job_info['ModelArtifacts']['S3ModelArtifacts']" 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": 5, 82 | "metadata": {}, 83 | "outputs": [ 84 | { 85 | "ename": "ClientError", 86 | "evalue": "An error occurred (ValidationException) when calling the CreateModel operation: Cannot create already existing model \"arn:aws:sagemaker:us-east-2:733184320490:model/image-classification-2019-09-12-17-02-05-917-model\".", 87 | "output_type": "error", 88 | "traceback": [ 89 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 90 | "\u001b[0;31mClientError\u001b[0m Traceback (most recent call last)", 91 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0mModelName\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0mExecutionRoleArn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrole\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m PrimaryContainer = primary_container)\n\u001b[0m", 92 | "\u001b[0;32m~/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 355\u001b[0m \"%s() only accepts keyword arguments.\" % py_operation_name)\n\u001b[1;32m 356\u001b[0m \u001b[0;31m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_api_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 358\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 359\u001b[0m \u001b[0m_api_call\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpy_operation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 93 | "\u001b[0;32m~/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 659\u001b[0m \u001b[0merror_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Error\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Code\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 660\u001b[0m \u001b[0merror_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 661\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0merror_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparsed_response\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 662\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 663\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 94 | "\u001b[0;31mClientError\u001b[0m: An error occurred (ValidationException) when calling the CreateModel operation: Cannot create already existing model \"arn:aws:sagemaker:us-east-2:733184320490:model/image-classification-2019-09-12-17-02-05-917-model\"." 95 | ] 96 | } 97 | ], 98 | "source": [ 99 | "model_name = job_name + \"-model\"\n", 100 | "\n", 101 | "# We also need to tell SageMaker which container should be used for inference and where it should\n", 102 | "# retrieve the model artifacts from. In our case, the xgboost container that we used for training\n", 103 | "# can also be used for inference.\n", 104 | "primary_container = {\n", 105 | " \"Image\": container,\n", 106 | " \"ModelDataUrl\": model_artifacts\n", 107 | "}\n", 108 | "\n", 109 | "# And lastly we construct the SageMaker model\n", 110 | "model_info = session.sagemaker_client.create_model(\n", 111 | " ModelName = model_name,\n", 112 | " ExecutionRoleArn = role,\n", 113 | " PrimaryContainer = primary_container)" 114 | ] 115 | }, 116 | { 117 | "cell_type": "code", 118 | "execution_count": 6, 119 | "metadata": {}, 120 | "outputs": [], 121 | "source": [ 122 | "transform_job_name = 'nsfwbatchtransform' + strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime())\n", 123 | "\n", 124 | "# Now we construct the data structure which will describe the batch transform job.\n", 125 | "transform_request = \\\n", 126 | "{\n", 127 | " \"TransformJobName\": transform_job_name,\n", 128 | " \n", 129 | " # This is the name of the model that we created earlier.\n", 130 | " \"ModelName\": model_name,\n", 131 | " \n", 132 | " # This describes how many compute instances should be used at once. If you happen to be doing a very large\n", 133 | " # batch transform job it may be worth running multiple compute instances at once.\n", 134 | " \"MaxConcurrentTransforms\": 1,\n", 135 | " \n", 136 | " # This says how big each individual request sent to the model should be, at most. One of the things that\n", 137 | " # SageMaker does in the background is to split our data up into chunks so that each chunks stays under\n", 138 | " # this size limit.\n", 139 | " \"MaxPayloadInMB\": 6,\n", 140 | " \n", 141 | " # Sometimes we may want to send only a single sample to our endpoint at a time, however in this case each of\n", 142 | " # the chunks that we send should contain multiple samples of our input data.\n", 143 | " \"BatchStrategy\": \"MultiRecord\",\n", 144 | " \n", 145 | " # This next object describes where the output data should be stored. Some of the more advanced options which\n", 146 | " # we don't cover here also describe how SageMaker should collect output from various batches.\n", 147 | " \"TransformOutput\": {\n", 148 | " \"S3OutputPath\": \"s3://project-completion-udacity/nsfw_dataset/batch-transform_bench/\"\n", 149 | " },\n", 150 | " \n", 151 | " # Here we describe our input data. Of course, we need to tell SageMaker where on S3 our input data is stored, in\n", 152 | " # addition we need to detail the characteristics of our input data. In particular, since SageMaker may need to\n", 153 | " # split our data up into chunks, it needs to know how the individual samples in our data file appear. In our\n", 154 | " # case each line is its own sample and so we set the split type to 'line'. We also need to tell SageMaker what\n", 155 | " # type of data is being sent, in this case csv, so that it can properly serialize the data.\n", 156 | " \"TransformInput\": {\n", 157 | " \"ContentType\": \"application/x-image\",\n", 158 | " \"SplitType\": \"None\",\n", 159 | " \"DataSource\": {\n", 160 | " \"S3DataSource\": {\n", 161 | " \"S3DataType\": \"S3Prefix\",\n", 162 | " \"S3Uri\": 's3://project-completion-udacity/bench/',\n", 163 | " }\n", 164 | " }\n", 165 | " },\n", 166 | " \n", 167 | " # And lastly we tell SageMaker what sort of compute instance we would like it to use.\n", 168 | " \"TransformResources\": {\n", 169 | " \"InstanceType\": \"ml.m4.xlarge\",\n", 170 | " \"InstanceCount\": 1\n", 171 | " }\n", 172 | "}" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 7, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "transform_response = session.sagemaker_client.create_transform_job(**transform_request)" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": 8, 187 | "metadata": {}, 188 | "outputs": [ 189 | { 190 | "name": "stdout", 191 | "output_type": "stream", 192 | "text": [ 193 | "..............................................................!\n" 194 | ] 195 | } 196 | ], 197 | "source": [ 198 | "transform_desc = session.wait_for_transform_job(transform_job_name)" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": null, 204 | "metadata": {}, 205 | "outputs": [], 206 | "source": [] 207 | } 208 | ], 209 | "metadata": { 210 | "kernelspec": { 211 | "display_name": "Python 3", 212 | "language": "python", 213 | "name": "python3" 214 | }, 215 | "language_info": { 216 | "codemirror_mode": { 217 | "name": "ipython", 218 | "version": 3 219 | }, 220 | "file_extension": ".py", 221 | "mimetype": "text/x-python", 222 | "name": "python", 223 | "nbconvert_exporter": "python", 224 | "pygments_lexer": "ipython3", 225 | "version": "3.5.2" 226 | } 227 | }, 228 | "nbformat": 4, 229 | "nbformat_minor": 2 230 | } 231 | -------------------------------------------------------------------------------- /classification_tool.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "

Put all the images in pics folder and run the cell below and it will make your bulk classification task simple." 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "\n", 17 | "\n", 18 | "import sys, termios, tty, os, time\n", 19 | "#import autopy\n", 20 | "import ipywidgets as widgets\n", 21 | "current_directory = os.getcwd()\n", 22 | "try :\n", 23 | " path_nude = os.path.join(current_directory , 'Nude')\n", 24 | " os.makedirs(path_nude)\n", 25 | "except FileExistsError as e :\n", 26 | " pass\n", 27 | "\n", 28 | "try :\n", 29 | " path_animated = os.path.join(current_directory , 'Animated')\n", 30 | " os.makedirs(path_animated)\n", 31 | "except FileExistsError as e :\n", 32 | " pass\n", 33 | "\n", 34 | "try :\n", 35 | " path_porn = os.path.join(current_directory , 'Porn')\n", 36 | " os.makedirs(path_porn)\n", 37 | "except FileExistsError as e :\n", 38 | " pass\n", 39 | "\n", 40 | "import glob , pickle\n", 41 | "datadir = 'pics'\n", 42 | "imgs = glob.glob(os.path.join(datadir, '*.jpg'))\n", 43 | "print(len(imgs))\n", 44 | "\n", 45 | "import pickle as pickle\n", 46 | "def save_obj(obj, name ):\n", 47 | " with open(name + '.pkl', 'wb') as f:\n", 48 | " pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n", 49 | "\n", 50 | "def load_obj(name ):\n", 51 | " with open(name + '.pkl', 'rb') as f:\n", 52 | " return pickle.load(f) \n", 53 | " \n", 54 | "try:\n", 55 | " lookup = load_obj('work_done')\n", 56 | "except (OSError, IOError) as e:\n", 57 | " lookup = {}\n", 58 | " for img in imgs:\n", 59 | " lookup[img] = 0\n", 60 | " save_obj( lookup , 'work_done')\n", 61 | "\n", 62 | "import matplotlib.pyplot as plt\n", 63 | "plt.ion()\n", 64 | "#from PIL import Image\n", 65 | "from IPython.display import Image\n", 66 | "from IPython.display import clear_output\n", 67 | "import shutil\n", 68 | "\n", 69 | "lookup = load_obj('work_done')\n", 70 | "def on_nude_Button(b):\n", 71 | " global i\n", 72 | " global lookup\n", 73 | " global imgs\n", 74 | " #plt.close()\n", 75 | " if i < len(imgs) and lookup[imgs[i]] == 0:\n", 76 | " clear_output()\n", 77 | " lookup[imgs[i]] = 1\n", 78 | " shutil.move(imgs[i],path_nude)\n", 79 | " i += 1\n", 80 | " display(Image(imgs[i]))\n", 81 | " display(hb)\n", 82 | " \n", 83 | "# im = Image.open( imgs[i]) \n", 84 | "# plt.imshow(im)\n", 85 | " \n", 86 | " print('nude_clicked')\n", 87 | " \n", 88 | "def on_Porn_Button(b):\n", 89 | " global i\n", 90 | " global lookup\n", 91 | " global imgs\n", 92 | " if i < len(imgs) and lookup[imgs[i]] == 0 :\n", 93 | " clear_output()\n", 94 | " lookup[imgs[i]] = 1\n", 95 | " shutil.move(imgs[i],path_porn)\n", 96 | " i += 1\n", 97 | " display(Image(imgs[i]))\n", 98 | " display(hb)\n", 99 | " \n", 100 | " print('porn_clicked')\n", 101 | " \n", 102 | "def on_animated_Button(b):\n", 103 | " global i\n", 104 | " global lookup\n", 105 | " global imgs\n", 106 | " if i < len(imgs) and lookup[imgs[i]] == 0:\n", 107 | " clear_output()\n", 108 | " lookup[imgs[i]] = 1\n", 109 | " shutil.move(imgs[i],path_animated)\n", 110 | " i += 1\n", 111 | " display(Image(imgs[i]))\n", 112 | " display(hb)\n", 113 | " \n", 114 | " print('animated_clicked')\n", 115 | " \n", 116 | "def on_Quit_Button(b):\n", 117 | " print('Quit_clicked')\n", 118 | " save_obj(lookup , 'work_done')\n", 119 | " clear_output()\n", 120 | "\n", 121 | "Nude_button = widgets.Button(description= \"Nude\")\n", 122 | "Porn_button = widgets.Button(description= \"Porn\")\n", 123 | "Animated_button = widgets.Button(description= \"Animated\")\n", 124 | "Quit_Button = widgets.Button(description= \"Quit\")\n", 125 | "\n", 126 | "Nude_button.on_click(on_nude_Button)\n", 127 | "Porn_button.on_click(on_Porn_Button)\n", 128 | "Animated_button.on_click(on_animated_Button)\n", 129 | "Quit_Button.on_click(on_Quit_Button)\n", 130 | "\n", 131 | "\n", 132 | "hb= widgets.HBox([ Nude_button , Porn_button , Animated_button , Quit_Button ])\n", 133 | "\n", 134 | "\n", 135 | "i = 0 \n", 136 | "if i < len(imgs):\n", 137 | " display( Image(imgs[i]) )\n", 138 | " display(hb)\n", 139 | "\n", 140 | " \n", 141 | "\n", 142 | "\n", 143 | "\n", 144 | "\n", 145 | "\n", 146 | " \n", 147 | " " 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": null, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [] 156 | } 157 | ], 158 | "metadata": { 159 | "kernelspec": { 160 | "display_name": "Python 3", 161 | "language": "python", 162 | "name": "python3" 163 | }, 164 | "language_info": { 165 | "codemirror_mode": { 166 | "name": "ipython", 167 | "version": 3 168 | }, 169 | "file_extension": ".py", 170 | "mimetype": "text/x-python", 171 | "name": "python", 172 | "nbconvert_exporter": "python", 173 | "pygments_lexer": "ipython3", 174 | "version": "3.5.2" 175 | } 176 | }, 177 | "nbformat": 4, 178 | "nbformat_minor": 2 179 | } 180 | -------------------------------------------------------------------------------- /clean and prepare data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## As discussed in the report we need to discard bad examples. " 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import os\n", 17 | "import PIL\n", 18 | "from PIL import Image\n", 19 | "import glob\n", 20 | "import shutil\n", 21 | "\n", 22 | "def check_and_discard_data(data_dir , dump_dir):\n", 23 | " for class_name in os.listdir(data_dir):\n", 24 | " list_of_files_of_single_class = glob.glob(os.path.join(data_dir , class_name , '*') ) \n", 25 | " for file_name in list_of_files_of_single_class:\n", 26 | " if file_name.lower().endswith(( '.jpg', '.jpeg')):\n", 27 | " try:\n", 28 | " img=Image.open(file_name)\n", 29 | " img.verify()\n", 30 | " height , width = img.size\n", 31 | " if height < 80 or width < 80:\n", 32 | " print('dim less')\n", 33 | " shutil.move(file_name, dump_dir)\n", 34 | " except(IOError,SyntaxError)as e:\n", 35 | " print('io error')\n", 36 | " shutil.move(file_name, dump_dir)\n", 37 | " else:\n", 38 | " print('print other format')\n", 39 | " shutil.move( file_name , dump_dir)" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 1, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "check_and_discard_data('nsfw_datset' , 'dump')" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 4, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "data": { 58 | "text/plain": [ 59 | "16864" 60 | ] 61 | }, 62 | "execution_count": 4, 63 | "metadata": {}, 64 | "output_type": "execute_result" 65 | } 66 | ], 67 | "source": [ 68 | "len(os.listdir('dump'))" 69 | ] 70 | }, 71 | { 72 | "cell_type": "markdown", 73 | "metadata": {}, 74 | "source": [ 75 | "## Spliting the datset into train , test , validation" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 2, 81 | "metadata": {}, 82 | "outputs": [ 83 | { 84 | "name": "stdout", 85 | "output_type": "stream", 86 | "text": [ 87 | "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mos\u001b[39;49;00m\r\n", 88 | "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mnumpy\u001b[39;49;00m \u001b[34mas\u001b[39;49;00m \u001b[04m\u001b[36mnp\u001b[39;49;00m\r\n", 89 | "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mshutil\u001b[39;49;00m\r\n", 90 | "\r\n", 91 | "\u001b[37m# # Creating Train / Val / Test folders (One time use)\u001b[39;49;00m\r\n", 92 | "\r\n", 93 | "\u001b[34mdef\u001b[39;49;00m \u001b[32msplit_dataset_one_time\u001b[39;49;00m(root_dir , valid_size, test_size , train_path , valid_path , test_path):\r\n", 94 | " \r\n", 95 | "\r\n", 96 | " class_names = os.listdir(root_dir)\r\n", 97 | "\r\n", 98 | " \u001b[37m#print(class_names)\u001b[39;49;00m\r\n", 99 | " \u001b[34mfor\u001b[39;49;00m single_class \u001b[35min\u001b[39;49;00m class_names:\r\n", 100 | " \u001b[34mtry\u001b[39;49;00m :\r\n", 101 | "\r\n", 102 | "\r\n", 103 | " os.makedirs(os.path.join( train_path , single_class))\r\n", 104 | " os.makedirs(os.path.join( valid_path , single_class))\r\n", 105 | " os.makedirs(os.path.join( test_path , single_class))\r\n", 106 | "\r\n", 107 | "\r\n", 108 | " train_single_path = os.path.join( train_path , single_class)\r\n", 109 | " test_single_path = os.path.join( test_path , single_class)\r\n", 110 | " valid_single_path = os.path.join( valid_path , single_class)\r\n", 111 | " \u001b[34mexcept\u001b[39;49;00m FileExistsError:\r\n", 112 | " \u001b[34mpass\u001b[39;49;00m\r\n", 113 | " \r\n", 114 | " single_full_list = os.listdir(os.path.join(root_dir , single_class))\r\n", 115 | " \u001b[37m#print(single_full_list[0])\u001b[39;49;00m\r\n", 116 | "\r\n", 117 | " num_examples_per_class = \u001b[36mlen\u001b[39;49;00m(single_full_list)\r\n", 118 | " indices = \u001b[36mlist\u001b[39;49;00m(\u001b[36mrange\u001b[39;49;00m(num_examples_per_class))\r\n", 119 | " split_valid = \u001b[36mint\u001b[39;49;00m(np.floor(valid_size * num_examples_per_class))\r\n", 120 | " split_test = \u001b[36mint\u001b[39;49;00m(np.floor(test_size * num_examples_per_class)) + split_valid\r\n", 121 | " np.random.shuffle(indices)\r\n", 122 | " train_idx, valid_idx , test_idx = indices[split_test:], indices[:split_valid] , indices[split_valid:split_test]\r\n", 123 | "\r\n", 124 | " \u001b[37m#print(train_idx[:5])\u001b[39;49;00m\r\n", 125 | " single_full_list_np = np.array(single_full_list)\r\n", 126 | " train_list , valid_list , test_list = single_full_list_np[train_idx].tolist() , single_full_list_np[valid_idx].tolist() , single_full_list_np[test_idx].tolist()\r\n", 127 | "\r\n", 128 | " \u001b[37m#print(len(train_list) , len(valid_list) , len(test_list) , len(single_full_list))\u001b[39;49;00m\r\n", 129 | "\r\n", 130 | "\r\n", 131 | "\r\n", 132 | " \u001b[34mfor\u001b[39;49;00m name \u001b[35min\u001b[39;49;00m train_list:\r\n", 133 | " \u001b[37m#print(os.path.join(root_dir , single_class , name) , train_single_path)\u001b[39;49;00m\r\n", 134 | " shutil.move( os.path.join(root_dir , single_class , name) , train_single_path)\r\n", 135 | "\r\n", 136 | " \u001b[34mfor\u001b[39;49;00m name \u001b[35min\u001b[39;49;00m valid_list:\r\n", 137 | " \u001b[37m#print(os.path.join(root_dir , single_class , name) , valid_single_path)\u001b[39;49;00m\r\n", 138 | " shutil.move(os.path.join(root_dir , single_class , name) ,valid_single_path)\r\n", 139 | "\r\n", 140 | " \u001b[34mfor\u001b[39;49;00m name \u001b[35min\u001b[39;49;00m test_list:\r\n", 141 | " \u001b[37m#print(os.path.join(root_dir , single_class , name) , test_single_path)\u001b[39;49;00m\r\n", 142 | " shutil.move(os.path.join(root_dir , single_class , name), test_single_path)\r\n" 143 | ] 144 | } 145 | ], 146 | "source": [ 147 | "!pygmentize useful_scripts/useful_scripts/split_dataset.py" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 2, 153 | "metadata": {}, 154 | "outputs": [], 155 | "source": [ 156 | "from useful_scripts.useful_scripts.split_dataset import split_dataset_one_time" 157 | ] 158 | }, 159 | { 160 | "cell_type": "code", 161 | "execution_count": 3, 162 | "metadata": {}, 163 | "outputs": [], 164 | "source": [ 165 | "split_dataset_one_time('nsfw_datset' , 0.075 ,0.075 , 'nsfw_dataset_train' , 'nsfw_dataset_valid' , 'nsfw_dataset_test')" 166 | ] 167 | }, 168 | { 169 | "cell_type": "markdown", 170 | "metadata": {}, 171 | "source": [ 172 | "## Setting the im2rec tool in the machine. " 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 1, 178 | "metadata": {}, 179 | "outputs": [ 180 | { 181 | "name": "stdout", 182 | "output_type": "stream", 183 | "text": [ 184 | "env: IM2REC=/home/ec2-user/anaconda3/envs/amazonei_mxnet_p36/lib/python3.6/site-packages/mxnet/tools/im2rec.py\n" 185 | ] 186 | } 187 | ], 188 | "source": [ 189 | "import sys,os\n", 190 | "\n", 191 | "suffix='/mxnet/tools/im2rec.py'\n", 192 | "im2rec = list(filter( (lambda x: os.path.isfile(x + suffix )), sys.path))[0] + suffix\n", 193 | "%env IM2REC=$im2rec" 194 | ] 195 | }, 196 | { 197 | "cell_type": "markdown", 198 | "metadata": {}, 199 | "source": [ 200 | "## We need to create .lst file before creating .rec file Notice That test_ratio is kept at 0.0 because we have already split the dataset. Also The --resize is set 224 because image classification algorithm accepts image size (224,224)." 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": 2, 206 | "metadata": {}, 207 | "outputs": [ 208 | { 209 | "name": "stdout", 210 | "output_type": "stream", 211 | "text": [ 212 | "Creating LST files\n", 213 | "Label classes:\n", 214 | "animated 0\n", 215 | "nude 1\n", 216 | "porn 2\n", 217 | "safe_for_work 3\n", 218 | "semi_nude 4\n", 219 | "Creating RecordIO files\n", 220 | "Creating .rec file from /home/ec2-user/SageMaker/nsfw_dataset_valid.lst in /home/ec2-user/SageMaker\n", 221 | "time: 0.3074500560760498 count: 0\n", 222 | "time: 28.615877151489258 count: 1000\n", 223 | "time: 29.252779245376587 count: 2000\n", 224 | "time: 27.263898849487305 count: 3000\n", 225 | "time: 24.09006953239441 count: 4000\n", 226 | "time: 24.43202519416809 count: 5000\n", 227 | "time: 23.392404556274414 count: 6000\n", 228 | "time: 22.804959058761597 count: 7000\n", 229 | "time: 23.166723251342773 count: 8000\n", 230 | "time: 21.34347176551819 count: 9000\n", 231 | "time: 20.13561248779297 count: 10000\n", 232 | "time: 20.357985973358154 count: 11000\n", 233 | "-rw-rw-r-- 1 ec2-user ec2-user 387M Sep 11 18:49 nsfw_dataset_test.rec\n", 234 | "-rw-rw-r-- 1 ec2-user ec2-user 318M Sep 12 07:16 nsfw_dataset_valid.rec\n" 235 | ] 236 | } 237 | ], 238 | "source": [ 239 | "%%bash\n", 240 | "echo \"Creating LST files\"\n", 241 | "python $IM2REC --list --recursive --pass-through --test-ratio=0.0 --train-ratio=1.0 nsfw_dataset_valid nsfw_dataset_valid > nsfw_dataset_valid_classes\n", 242 | "\n", 243 | "echo \"Label classes:\"\n", 244 | "cat nsfw_dataset_valid_classes\n", 245 | "\n", 246 | "echo \"Creating RecordIO files\"\n", 247 | "python $IM2REC --num-thread=4 nsfw_dataset_valid.lst nsfw_dataset_valid/ --resize=224\n", 248 | "#python $IM2REC --num-thread=4 ${DATASET_NAME}_test.lst $DATASET_NAME\n", 249 | "ls -lh *.rec" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "metadata": {}, 255 | "source": [ 256 | "## upload the .rec file to s3." 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": 3, 262 | "metadata": {}, 263 | "outputs": [ 264 | { 265 | "name": "stdout", 266 | "output_type": "stream", 267 | "text": [ 268 | "upload: ./nsfw_dataset_valid.rec to s3://project-completion-udacity/nsfw_dataset/validation/nsfw_dataset_valid.rec\n" 269 | ] 270 | } 271 | ], 272 | "source": [ 273 | "!aws s3 cp nsfw_dataset_valid.rec s3://project-completion-udacity/nsfw_dataset/validation/" 274 | ] 275 | }, 276 | { 277 | "cell_type": "markdown", 278 | "metadata": {}, 279 | "source": [ 280 | "## We did for validation. Now doing for testing." 281 | ] 282 | }, 283 | { 284 | "cell_type": "code", 285 | "execution_count": 8, 286 | "metadata": {}, 287 | "outputs": [ 288 | { 289 | "name": "stdout", 290 | "output_type": "stream", 291 | "text": [ 292 | "Creating LST files\n", 293 | "Label classes:\n", 294 | "animated 0\n", 295 | "nude 1\n", 296 | "porn 2\n", 297 | "safe_for_work 3\n", 298 | "semi_nude 4\n", 299 | "Creating RecordIO files\n", 300 | "Creating .rec file from /home/ec2-user/SageMaker/nsfw_dataset_test.lst in /home/ec2-user/SageMaker\n", 301 | "time: 0.05958127975463867 count: 0\n", 302 | "time: 9.245877742767334 count: 1000\n", 303 | "time: 7.712120532989502 count: 2000\n", 304 | "time: 7.699880361557007 count: 3000\n", 305 | "time: 9.303281545639038 count: 4000\n", 306 | "time: 9.592679262161255 count: 5000\n", 307 | "time: 8.724056959152222 count: 6000\n", 308 | "time: 8.638715028762817 count: 7000\n", 309 | "time: 10.148676872253418 count: 8000\n", 310 | "time: 7.879869699478149 count: 9000\n", 311 | "time: 7.991632699966431 count: 10000\n", 312 | "time: 8.284252405166626 count: 11000\n", 313 | "-rw-rw-r-- 1 ec2-user ec2-user 387M Sep 11 18:49 nsfw_dataset_test.rec\n", 314 | "-rw-rw-r-- 1 ec2-user ec2-user 388M Sep 11 18:46 nsfw_dataset_valid.rec\n" 315 | ] 316 | } 317 | ], 318 | "source": [ 319 | "%%bash\n", 320 | "echo \"Creating LST files\"\n", 321 | "python $IM2REC --list --recursive --pass-through --test-ratio=0.0 --train-ratio=1.0 nsfw_dataset_test nsfw_dataset_test > nsfw_dataset_test_classes\n", 322 | "\n", 323 | "echo \"Label classes:\"\n", 324 | "cat nsfw_dataset_test_classes\n", 325 | "\n", 326 | "echo \"Creating RecordIO files\"\n", 327 | "python $IM2REC --num-thread=4 nsfw_dataset_test.lst nsfw_dataset_test/ --resize=256\n", 328 | "#python $IM2REC --num-thread=4 ${DATASET_NAME}_test.lst $DATASET_NAME\n", 329 | "ls -lh *.rec" 330 | ] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "execution_count": 9, 335 | "metadata": {}, 336 | "outputs": [ 337 | { 338 | "name": "stdout", 339 | "output_type": "stream", 340 | "text": [ 341 | "upload: ./nsfw_dataset_test.rec to s3://project-completion-udacity/nsfw_dataset/testing/nsfw_dataset_test.rec\n" 342 | ] 343 | } 344 | ], 345 | "source": [ 346 | "!aws s3 cp nsfw_dataset_test.rec s3://project-completion-udacity/nsfw_dataset/testing/" 347 | ] 348 | }, 349 | { 350 | "cell_type": "markdown", 351 | "metadata": {}, 352 | "source": [ 353 | "## Similarly doing this training ." 354 | ] 355 | }, 356 | { 357 | "cell_type": "code", 358 | "execution_count": 4, 359 | "metadata": {}, 360 | "outputs": [ 361 | { 362 | "name": "stdout", 363 | "output_type": "stream", 364 | "text": [ 365 | "Creating LST files\n", 366 | "Label classes:\n", 367 | "animated 0\n", 368 | "nude 1\n", 369 | "porn 2\n", 370 | "safe_for_work 3\n", 371 | "semi_nude 4\n", 372 | "Creating RecordIO files\n", 373 | "Creating .rec file from /home/ec2-user/SageMaker/nsfw_dataset_train.lst in /home/ec2-user/SageMaker\n", 374 | "imread read blank (None) image for file: /home/ec2-user/SageMaker/nsfw_dataset_train/porn/le8LkPs.jpg\n", 375 | "time: 0.046601057052612305 count: 0\n", 376 | "time: 18.696394205093384 count: 1000\n", 377 | "time: 17.1350200176239 count: 2000\n", 378 | "time: 17.429579734802246 count: 3000\n", 379 | "time: 15.988012313842773 count: 4000\n", 380 | "time: 15.514463901519775 count: 5000\n", 381 | "time: 16.88562774658203 count: 6000\n", 382 | "time: 14.923904657363892 count: 7000\n", 383 | "time: 15.456107378005981 count: 8000\n", 384 | "time: 14.62105655670166 count: 9000\n", 385 | "time: 15.861175060272217 count: 10000\n", 386 | "time: 12.652273893356323 count: 11000\n", 387 | "time: 12.256561517715454 count: 12000\n", 388 | "time: 15.415345668792725 count: 13000\n", 389 | "time: 12.36422848701477 count: 14000\n", 390 | "time: 13.456058979034424 count: 15000\n", 391 | "time: 12.073819398880005 count: 16000\n", 392 | "time: 12.335830688476562 count: 17000\n", 393 | "time: 11.697640180587769 count: 18000\n", 394 | "time: 12.603720903396606 count: 19000\n", 395 | "time: 12.044790506362915 count: 20000\n", 396 | "time: 10.525394201278687 count: 21000\n", 397 | "time: 10.050683975219727 count: 22000\n", 398 | "time: 11.899126768112183 count: 23000\n", 399 | "time: 11.846455335617065 count: 24000\n", 400 | "time: 10.586645364761353 count: 25000\n", 401 | "time: 12.133596897125244 count: 26000\n", 402 | "time: 9.788665056228638 count: 27000\n", 403 | "time: 8.77630352973938 count: 28000\n", 404 | "time: 11.108652353286743 count: 29000\n", 405 | "time: 9.29899525642395 count: 30000\n", 406 | "time: 8.131813526153564 count: 31000\n", 407 | "time: 8.894354820251465 count: 32000\n", 408 | "time: 12.37432050704956 count: 33000\n", 409 | "time: 7.808954477310181 count: 34000\n", 410 | "time: 7.941628694534302 count: 35000\n", 411 | "time: 10.267624855041504 count: 36000\n", 412 | "time: 8.453442811965942 count: 37000\n", 413 | "time: 10.05301833152771 count: 38000\n", 414 | "time: 8.793598651885986 count: 39000\n", 415 | "time: 10.400083780288696 count: 40000\n", 416 | "time: 7.682428359985352 count: 41000\n", 417 | "time: 8.213168621063232 count: 42000\n", 418 | "time: 11.412482500076294 count: 43000\n", 419 | "time: 8.707566261291504 count: 44000\n", 420 | "time: 8.02009630203247 count: 45000\n", 421 | "time: 8.045879125595093 count: 46000\n", 422 | "time: 9.501078844070435 count: 47000\n", 423 | "time: 7.770918846130371 count: 48000\n", 424 | "time: 8.169719696044922 count: 49000\n", 425 | "time: 8.926843404769897 count: 50000\n", 426 | "time: 9.74822998046875 count: 51000\n", 427 | "time: 6.555929183959961 count: 52000\n", 428 | "time: 8.067629337310791 count: 53000\n", 429 | "time: 8.084633827209473 count: 54000\n", 430 | "time: 10.004342794418335 count: 55000\n", 431 | "time: 7.856613397598267 count: 56000\n", 432 | "time: 8.123029708862305 count: 57000\n", 433 | "time: 8.658063650131226 count: 58000\n", 434 | "time: 9.630893230438232 count: 59000\n", 435 | "time: 7.653128623962402 count: 60000\n", 436 | "time: 8.164822578430176 count: 61000\n", 437 | "time: 8.797566413879395 count: 62000\n", 438 | "time: 8.291775941848755 count: 63000\n", 439 | "time: 8.04925537109375 count: 64000\n", 440 | "time: 8.506189346313477 count: 65000\n", 441 | "time: 10.95771312713623 count: 66000\n", 442 | "time: 7.351006031036377 count: 67000\n", 443 | "time: 7.488478899002075 count: 68000\n", 444 | "time: 9.657930850982666 count: 69000\n", 445 | "time: 10.078916072845459 count: 70000\n", 446 | "time: 7.4690022468566895 count: 71000\n", 447 | "time: 8.133487224578857 count: 72000\n", 448 | "time: 8.533123970031738 count: 73000\n", 449 | "time: 9.47664999961853 count: 74000\n", 450 | "time: 7.935559034347534 count: 75000\n", 451 | "time: 7.936443567276001 count: 76000\n", 452 | "time: 7.421195030212402 count: 77000\n", 453 | "time: 10.084851026535034 count: 78000\n", 454 | "time: 7.899816274642944 count: 79000\n", 455 | "time: 7.024080038070679 count: 80000\n", 456 | "time: 7.249448776245117 count: 81000\n", 457 | "time: 9.525150060653687 count: 82000\n", 458 | "time: 7.0800535678863525 count: 83000\n", 459 | "time: 7.291053056716919 count: 84000\n", 460 | "time: 7.127345085144043 count: 85000\n", 461 | "time: 11.072758436203003 count: 86000\n", 462 | "time: 8.476456880569458 count: 87000\n", 463 | "time: 6.555356502532959 count: 88000\n", 464 | "time: 7.437634706497192 count: 89000\n", 465 | "time: 9.70185112953186 count: 90000\n", 466 | "time: 7.41411566734314 count: 91000\n", 467 | "time: 6.791863918304443 count: 92000\n", 468 | "time: 7.763757705688477 count: 93000\n", 469 | "time: 10.760865449905396 count: 94000\n", 470 | "time: 8.289317607879639 count: 95000\n", 471 | "time: 6.7506866455078125 count: 96000\n", 472 | "time: 7.14838719367981 count: 97000\n", 473 | "time: 10.034961223602295 count: 98000\n", 474 | "time: 8.622971773147583 count: 99000\n", 475 | "time: 6.736122369766235 count: 100000\n", 476 | "time: 6.962104082107544 count: 101000\n", 477 | "time: 7.514451742172241 count: 102000\n", 478 | "time: 10.299906492233276 count: 103000\n", 479 | "time: 8.237797975540161 count: 104000\n", 480 | "time: 7.880961179733276 count: 105000\n", 481 | "time: 9.844894409179688 count: 106000\n", 482 | "time: 8.546481132507324 count: 107000\n", 483 | "time: 8.704168319702148 count: 108000\n", 484 | "time: 7.344848394393921 count: 109000\n", 485 | "time: 9.523374795913696 count: 110000\n", 486 | "time: 8.907413244247437 count: 111000\n", 487 | "time: 23.584630012512207 count: 112000\n", 488 | "time: 7.0646071434021 count: 113000\n", 489 | "time: 6.461396217346191 count: 114000\n", 490 | "time: 8.127644538879395 count: 115000\n", 491 | "time: 9.173459768295288 count: 116000\n", 492 | "time: 7.150416374206543 count: 117000\n", 493 | "time: 6.67113995552063 count: 118000\n", 494 | "time: 9.35042119026184 count: 119000\n", 495 | "time: 35.60696053504944 count: 120000\n", 496 | "time: 44.333606481552124 count: 121000\n", 497 | "time: 44.246965408325195 count: 122000\n", 498 | "time: 37.153340578079224 count: 123000\n", 499 | "time: 26.17804741859436 count: 124000\n", 500 | "time: 16.095215320587158 count: 125000\n", 501 | "time: 10.093717336654663 count: 126000\n", 502 | "-rw-rw-r-- 1 ec2-user ec2-user 387M Sep 11 18:49 nsfw_dataset_test.rec\n", 503 | "-rw-rw-r-- 1 ec2-user ec2-user 3.6G Sep 12 07:39 nsfw_dataset_train.rec\n", 504 | "-rw-rw-r-- 1 ec2-user ec2-user 318M Sep 12 07:16 nsfw_dataset_valid.rec\n" 505 | ] 506 | } 507 | ], 508 | "source": [ 509 | "%%bash\n", 510 | "echo \"Creating LST files\"\n", 511 | "python $IM2REC --list --recursive --pass-through --test-ratio=0.0 --train-ratio=1.0 nsfw_dataset_train nsfw_dataset_train > nsfw_dataset_train_classes\n", 512 | "\n", 513 | "echo \"Label classes:\"\n", 514 | "cat nsfw_dataset_train_classes\n", 515 | "\n", 516 | "echo \"Creating RecordIO files\"\n", 517 | "python $IM2REC --num-thread=4 nsfw_dataset_train.lst nsfw_dataset_train/ --resize=224\n", 518 | "#python $IM2REC --num-thread=4 ${DATASET_NAME}_test.lst $DATASET_NAME\n", 519 | "ls -lh *.rec" 520 | ] 521 | }, 522 | { 523 | "cell_type": "code", 524 | "execution_count": 5, 525 | "metadata": {}, 526 | "outputs": [ 527 | { 528 | "name": "stdout", 529 | "output_type": "stream", 530 | "text": [ 531 | "upload: ./nsfw_dataset_train.rec to s3://project-completion-udacity/nsfw_dataset/training/nsfw_dataset_train.rec\n" 532 | ] 533 | } 534 | ], 535 | "source": [ 536 | "!aws s3 cp nsfw_dataset_train.rec s3://project-completion-udacity/nsfw_dataset/training/" 537 | ] 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": null, 542 | "metadata": {}, 543 | "outputs": [], 544 | "source": [] 545 | } 546 | ], 547 | "metadata": { 548 | "kernelspec": { 549 | "display_name": "Python 3", 550 | "language": "python", 551 | "name": "python3" 552 | }, 553 | "language_info": { 554 | "codemirror_mode": { 555 | "name": "ipython", 556 | "version": 3 557 | }, 558 | "file_extension": ".py", 559 | "mimetype": "text/x-python", 560 | "name": "python", 561 | "nbconvert_exporter": "python", 562 | "pygments_lexer": "ipython3", 563 | "version": "3.5.2" 564 | } 565 | }, 566 | "nbformat": 4, 567 | "nbformat_minor": 2 568 | } 569 | -------------------------------------------------------------------------------- /demo/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepanshu-yadav/NSFW-Classifier/8ee516cf4dc9045f8cd94fd13a4b6fa2cc92a554/demo/favicon.ico -------------------------------------------------------------------------------- /demo/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |

Is this image nsfw ??

10 | 11 | 12 | 13 | 14 |

Upload the image and find out !!

15 |
16 | 17 |

18 | 19 |

20 | 21 |

22 | 23 |
24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /demo/nsfw.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | var API_ENDPOINT = "https://7ul4fiq2v1.execute-api.us-east-2.amazonaws.com/final_stage" 4 | 5 | document.getElementById('inp').onchange = function(e) { 6 | var img = new Image(); 7 | img.onload = draw; 8 | img.onerror = failed; 9 | img.src = URL.createObjectURL(this.files[0]); 10 | console.log(img.src); 11 | }; 12 | function draw() { 13 | var canvas = document.getElementById('canvas'); 14 | canvas.width = this.width; 15 | canvas.height = this.height; 16 | var ctx = canvas.getContext('2d'); 17 | ctx.drawImage(this, 0,0); 18 | console.log("painted"); 19 | } 20 | function failed() { 21 | alert("The provided file couldn't be loaded as an Image media"); 22 | }; 23 | 24 | document.getElementById("genre").onclick = function(){ 25 | var canvas = document.getElementById("canvas") 26 | var image = canvas.toDataURL() 27 | var inputData = {"data": image}; 28 | console.log(inputData); 29 | $.ajax({ 30 | url: API_ENDPOINT, 31 | type: 'POST', 32 | crossDomain: true, 33 | data: JSON.stringify(inputData), 34 | dataType: 'json', 35 | contentType: "application/json", 36 | success: function (response) { 37 | console.log(response); 38 | document.getElementById("genreReturned").textContent = response; 39 | }, 40 | }); 41 | } 42 | -------------------------------------------------------------------------------- /demo/scr.sh: -------------------------------------------------------------------------------- 1 | export URL='https://6ehywu4ung.execute-api.us-east-2.amazonaws.com/init' 2 | 3 | ## WE SUBMIT THE POSTER FROM THE THRILLER/CRIME MOVIE HOSTAGE 4 | export PIC='sun.jpg' 5 | 6 | ## WE SEND THE IMAGE OVER TO THE API 7 | (echo -n '{"data": "'; base64 $PIC; echo '"}') | curl -H "Content-Type: application/json" -d @- $URL 8 | 9 | -------------------------------------------------------------------------------- /demo/styles.css: -------------------------------------------------------------------------------- 1 | .buttons { 2 | border : solid 0px #e6b215; 3 | border-radius : 8px; 4 | moz-border-radius : 8px; 5 | font-size : 16px; 6 | color : #ffffff; 7 | padding : 5px 18px; 8 | background-color : #FF9900; 9 | cursor:pointer; 10 | } 11 | 12 | .buttons:hover { 13 | background-color:#ffc477; 14 | } 15 | 16 | .buttons:active { 17 | position:relative; 18 | top:1px; 19 | } 20 | 21 | span { 22 | font-size: 120%; 23 | font-weight: bold; 24 | } 25 | 26 | h1, 27 | canvas, 28 | h3 { 29 | text-align: center; 30 | } 31 | 32 | .input-container { 33 | text-align: center; 34 | } 35 | 36 | body { 37 | font-family: 'Lato'; 38 | } 39 | -------------------------------------------------------------------------------- /demo/sun.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepanshu-yadav/NSFW-Classifier/8ee516cf4dc9045f8cd94fd13a4b6fa2cc92a554/demo/sun.jpg -------------------------------------------------------------------------------- /lambda.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import json 3 | import numpy as np 4 | import base64, os, boto3, ast, json 5 | 6 | 7 | 8 | 9 | endpoint = 'myprojectcapstone' 10 | 11 | def format_response(message, status_code): 12 | return { 13 | 'statusCode': str(status_code), 14 | 'body': json.dumps(message), 15 | 'headers': { 16 | 'Content-Type': 'application/json', 17 | 'Access-Control-Allow-Origin': '*' 18 | } 19 | } 20 | 21 | 22 | 23 | 24 | def lambda_handler(event, context): 25 | try : 26 | body = json.loads(event['body']) 27 | image = base64.b64decode(body['data'].replace('data:image/png;base64,', '')) 28 | 29 | try : 30 | runtime = boto3.Session().client(service_name='sagemaker-runtime', region_name='us-east-2') 31 | response = runtime.invoke_endpoint(EndpointName=endpoint, ContentType='application/x-image', Body=image) 32 | print(response) 33 | 34 | try: 35 | probs = response['Body'].read() 36 | probs = json.loads(probs) 37 | #probs = ast.literal_eval(probs) 38 | #pred = probs.index(max(probs)) 39 | pred = np.argmax( np.array( probs ) ) 40 | if pred == 0: 41 | resp = 'Animated Nsfw' 42 | elif pred == 1: 43 | resp = 'Conatins Nudity' 44 | elif pred == 2: 45 | resp = 'Contains Porn' 46 | elif pred == 4: 47 | resp = 'Conatins semi Nudity' 48 | else : 49 | resp = 'Safe For viewing' 50 | return format_response(resp, 200) 51 | except: 52 | return format_response('Ouch! Something went wrong with loading json data from endpoint'+response['Body'].read() , 200) 53 | except : 54 | return format_response('Ouch! Something went wrong with endpoint' , 200) 55 | except : 56 | return format_response('Ouch! Something went wrong with decoding' , 200) 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /nsfw-training_built_in_aws.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## This Notebook contains code for tuning the hyper parameters and then training the model." 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import sagemaker\n", 17 | "from sagemaker import get_execution_role\n", 18 | "from sagemaker.amazon.amazon_estimator import get_image_uri\n", 19 | "\n", 20 | "role = get_execution_role()\n", 21 | "sess = sagemaker.Session()\n", 22 | "\n", 23 | "training_image = get_image_uri(sess.boto_region_name, 'image-classification', repo_version=\"latest\")" 24 | ] 25 | }, 26 | { 27 | "cell_type": "markdown", 28 | "metadata": {}, 29 | "source": [ 30 | "## we need training and validation data in the form of record io format.\n", 31 | "Notice s3train_path and s3validation_path contains location of training data and validation data in record io format." 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "\n", 41 | "s3train_path = 's3://project-completion-udacity/nsfw_dataset/training'\n", 42 | "s3validation_path = 's3://project-completion-udacity/nsfw_dataset/validation'\n", 43 | "\n", 44 | "train_data = sagemaker.session.s3_input(\n", 45 | " s3train_path, \n", 46 | " distribution='FullyReplicated', \n", 47 | " content_type='application/x-recordio', \n", 48 | " s3_data_type='S3Prefix'\n", 49 | ")\n", 50 | "\n", 51 | "validation_data = sagemaker.session.s3_input(\n", 52 | " s3validation_path, \n", 53 | " distribution='FullyReplicated', \n", 54 | " content_type='application/x-recordio', \n", 55 | " s3_data_type='S3Prefix'\n", 56 | ")\n", 57 | "\n", 58 | "data_channels = {'train': train_data, 'validation': validation_data}\n" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": {}, 64 | "source": [ 65 | "## Specifying the instance for training." 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 3, 71 | "metadata": {}, 72 | "outputs": [], 73 | "source": [ 74 | "bucket = 'project-completion-udacity'\n", 75 | "dataset_name = 'nsfw_dataset'\n", 76 | "\n", 77 | "\n", 78 | "s3_output_location = 's3://{}/{}/output'.format(bucket, dataset_name)\n", 79 | "\n", 80 | "image_classifier = sagemaker.estimator.Estimator(\n", 81 | " training_image,\n", 82 | " role, \n", 83 | " train_instance_count=1, \n", 84 | " train_instance_type='ml.p2.xlarge',\n", 85 | " output_path=s3_output_location,\n", 86 | " sagemaker_session=sess\n", 87 | ")" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": 4, 93 | "metadata": {}, 94 | "outputs": [ 95 | { 96 | "name": "stdout", 97 | "output_type": "stream", 98 | "text": [ 99 | "126254 5\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "num_classes=5\n", 105 | "\n", 106 | "num_training_samples=! cat nsfw_dataset_train.lst | wc -l\n", 107 | "num_training_samples = int(num_training_samples[0])\n", 108 | "print(num_training_samples , num_classes)" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "## Declaring base hyperparameters that we dont wish to tune. " 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 5, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "\n", 125 | "# # Learn more about the Sagemaker built-in Image Classifier hyperparameters here: https://docs.aws.amazon.com/sagemaker/latest/dg/IC-Hyperparameter.html\n", 126 | "\n", 127 | "# # These hyperparameters we won't want to change, as they define things like\n", 128 | "# # the size of the images we'll be sending for input, the number of training classes we have, etc.\n", 129 | "base_hyperparameters=dict(\n", 130 | " use_pretrained_model=1,\n", 131 | " image_shape='3,224,224',\n", 132 | " num_classes=num_classes,\n", 133 | " num_training_samples=num_training_samples,\n", 134 | " augmentation_type = 'crop_color_transform',\n", 135 | " epochs = 1\n", 136 | " \n", 137 | " \n", 138 | ")\n", 139 | "\n", 140 | "# # These are hyperparameters we may want to tune, as they can affect the model training success:\n", 141 | "hyperparameters={\n", 142 | " **base_hyperparameters, \n", 143 | " **dict(\n", 144 | " learning_rate=0.001,\n", 145 | " mini_batch_size=5,\n", 146 | " )\n", 147 | "}\n", 148 | "\n", 149 | "\n", 150 | "image_classifier.set_hyperparameters(**hyperparameters)\n", 151 | "\n", 152 | "# hyperparameters\n" 153 | ] 154 | }, 155 | { 156 | "cell_type": "markdown", 157 | "metadata": {}, 158 | "source": [ 159 | "## Declaring the hyperparameters we wish to tune. And then creating hyper parameter tuning job." 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": 13, 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "from sagemaker.tuner import HyperparameterTuner, IntegerParameter, CategoricalParameter, ContinuousParameter\n", 169 | "hyperparameter_ranges = {'optimizer': CategoricalParameter(['nag', 'adam']),\n", 170 | " 'learning_rate': ContinuousParameter(0.0001, 0.01),\n", 171 | " 'mini_batch_size': IntegerParameter(15, 32),\n", 172 | " }\n", 173 | "\n", 174 | "objective_metric_name = 'validation:accuracy'\n", 175 | "\n", 176 | "tuner = HyperparameterTuner(image_classifier,\n", 177 | " objective_metric_name,\n", 178 | " hyperparameter_ranges,\n", 179 | " max_jobs=5,\n", 180 | " max_parallel_jobs=1)\n", 181 | "\n", 182 | "tuner.fit(inputs=data_channels, logs=True, include_cls_metadata=False)\n" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": 8, 188 | "metadata": {}, 189 | "outputs": [], 190 | "source": [ 191 | "# best_image_classifier = sagemaker.estimator.Estimator.attach(tuner.best_training_job())\n" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "# %%time\n", 201 | "\n", 202 | "# import time\n", 203 | "# now = str(int(time.time()))\n", 204 | "# training_job_name = 'IC-' + dataset_name.replace('_', '-') + '-' + now\n", 205 | "\n", 206 | "# image_classifier.fit(inputs=data_channels, job_name=training_job_name, logs=True)\n", 207 | "\n", 208 | "# job = image_classifier.latest_training_job\n", 209 | "# model_path = f\"{base_dir}/{job.name}\"\n", 210 | "\n", 211 | "# print(f\"\\n\\n Finished training! The model is available for download at: {image_classifier.output_path}/{job.name}/output/model.tar.gz\")\n" 212 | ] 213 | } 214 | ], 215 | "metadata": { 216 | "kernelspec": { 217 | "display_name": "Python 3", 218 | "language": "python", 219 | "name": "python3" 220 | }, 221 | "language_info": { 222 | "codemirror_mode": { 223 | "name": "ipython", 224 | "version": 3 225 | }, 226 | "file_extension": ".py", 227 | "mimetype": "text/x-python", 228 | "name": "python", 229 | "nbconvert_exporter": "python", 230 | "pygments_lexer": "ipython3", 231 | "version": "3.5.2" 232 | } 233 | }, 234 | "nbformat": 4, 235 | "nbformat_minor": 2 236 | } 237 | -------------------------------------------------------------------------------- /nsfw_deploy.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Contains code for deploying the trained model" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "import sagemaker\n", 17 | "from sagemaker import get_execution_role\n", 18 | "from sagemaker.amazon.amazon_estimator import get_image_uri\n", 19 | "\n", 20 | "role = get_execution_role()\n", 21 | "sess = sagemaker.Session()\n", 22 | "\n", 23 | "training_image = get_image_uri(sess.boto_region_name, 'image-classification', repo_version=\"latest\")\n" 24 | ] 25 | }, 26 | { 27 | "cell_type": "code", 28 | "execution_count": 2, 29 | "metadata": {}, 30 | "outputs": [], 31 | "source": [ 32 | "# bucket = 'project-completion-udacity'\n", 33 | "# dataset_name = 'nsfw_dataset'\n", 34 | "\n", 35 | "\n", 36 | "# s3_output_location = 's3://project-completion-udacity/nsfw_dataset/output'\n", 37 | "\n", 38 | "# image_classifier = sagemaker.estimator.Estimator(\n", 39 | "# training_image,\n", 40 | "# role, \n", 41 | "# train_instance_count = 1, \n", 42 | "# train_instance_type = 'ml.p2.xlarge',\n", 43 | "# output_path=s3_output_location,\n", 44 | "# sagemaker_session=sess\n", 45 | "# )" 46 | ] 47 | }, 48 | { 49 | "cell_type": "markdown", 50 | "metadata": {}, 51 | "source": [ 52 | "# the only change you need to make is to change job_name with the name of your own training job." 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 3, 58 | "metadata": {}, 59 | "outputs": [ 60 | { 61 | "name": "stdout", 62 | "output_type": "stream", 63 | "text": [ 64 | "DEMO-full-image-classification-model-2019-09-11-06-17-19\n", 65 | "s3://project-completion-udacity/nsfw_dataset/output/IC-nsfw-dataset-1568090782/output/model.tar.gz\n", 66 | "arn:aws:sagemaker:us-east-2:733184320490:model/demo-full-image-classification-model-2019-09-11-06-17-19\n" 67 | ] 68 | } 69 | ], 70 | "source": [ 71 | "\n", 72 | "\n", 73 | "job_name = 'IC-nsfw-dataset-1568090782'\n", 74 | "import boto3\n", 75 | "from time import gmtime, strftime\n", 76 | "import time\n", 77 | "sage = boto3.Session().client(service_name='sagemaker') \n", 78 | "timestamp = time.strftime('-%Y-%m-%d-%H-%M-%S', time.gmtime())\n", 79 | "model_name=\"DEMO-full-image-classification-model\" + timestamp\n", 80 | "print(model_name)\n", 81 | "info = sage.describe_training_job(TrainingJobName=job_name)\n", 82 | "model_data = info['ModelArtifacts']['S3ModelArtifacts']\n", 83 | "print(model_data)\n", 84 | "\n", 85 | "hosting_image = get_image_uri(boto3.Session().region_name, 'image-classification')\n", 86 | "\n", 87 | "primary_container = {\n", 88 | " 'Image': hosting_image,\n", 89 | " 'ModelDataUrl': model_data,\n", 90 | "}\n", 91 | "\n", 92 | "create_model_response = sage.create_model(\n", 93 | " ModelName = model_name,\n", 94 | " ExecutionRoleArn = role,\n", 95 | " PrimaryContainer = primary_container)\n", 96 | "\n", 97 | "print(create_model_response['ModelArn'])" 98 | ] 99 | }, 100 | { 101 | "cell_type": "markdown", 102 | "metadata": {}, 103 | "source": [ 104 | "

For endpoint configuration" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": null, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "# %%time\n", 114 | "# # Deploying a model to an endpoint takes a few minutes to complete\n", 115 | "\n", 116 | "# deployed_endpoint = image_classifier.deploy(\n", 117 | "# initial_instance_count = 1,\n", 118 | "# instance_type = 'ml.t2.medium'\n", 119 | "# )\n", 120 | "\n", 121 | "from time import gmtime, strftime\n", 122 | "\n", 123 | "timestamp = time.strftime('-%Y-%m-%d-%H-%M-%S', time.gmtime())\n", 124 | "endpoint_config_name = 'my-endpoint' + timestamp\n", 125 | "endpoint_config_response = sage.create_endpoint_config(\n", 126 | " EndpointConfigName = endpoint_config_name,\n", 127 | " ProductionVariants=[{\n", 128 | " 'InstanceType':'ml.t2.medium',\n", 129 | " 'InitialInstanceCount':1,\n", 130 | " 'ModelName':model_name,\n", 131 | " 'VariantName':'AllTraffic'}])\n", 132 | "\n", 133 | "print('Endpoint configuration name: {}'.format(endpoint_config_name))\n", 134 | "print('Endpoint configuration arn: {}'.format(endpoint_config_response['EndpointConfigArn']))" 135 | ] 136 | }, 137 | { 138 | "cell_type": "markdown", 139 | "metadata": {}, 140 | "source": [ 141 | "## code for lauanching the endpoint." 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": 5, 147 | "metadata": {}, 148 | "outputs": [ 149 | { 150 | "name": "stdout", 151 | "output_type": "stream", 152 | "text": [ 153 | "Endpoint name: myprojectcapstone\n", 154 | "EndpointArn = arn:aws:sagemaker:us-east-2:733184320490:endpoint/myprojectcapstone\n" 155 | ] 156 | } 157 | ], 158 | "source": [ 159 | "endpoint_name = 'myprojectcapstone'\n", 160 | "print('Endpoint name: {}'.format('myprojectcapstone'))\n", 161 | "\n", 162 | "import sagemaker as sm\n", 163 | "\n", 164 | "endpoint_params = {\n", 165 | " 'EndpointName': endpoint_name,\n", 166 | " 'EndpointConfigName': endpoint_config_name,\n", 167 | "}\n", 168 | "endpoint_response = sess.sagemaker_client.create_endpoint(**endpoint_params)\n", 169 | "print('EndpointArn = {}'.format(endpoint_response['EndpointArn']))" 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": 6, 175 | "metadata": {}, 176 | "outputs": [ 177 | { 178 | "name": "stdout", 179 | "output_type": "stream", 180 | "text": [ 181 | "------------------------------------------------------------------------------------------------------------!" 182 | ] 183 | } 184 | ], 185 | "source": [ 186 | "endpoint_dec = sess.wait_for_endpoint(endpoint_name)" 187 | ] 188 | }, 189 | { 190 | "cell_type": "markdown", 191 | "metadata": {}, 192 | "source": [ 193 | "## Testing the deployed endpoint" 194 | ] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "execution_count": 7, 199 | "metadata": {}, 200 | "outputs": [], 201 | "source": [ 202 | "import json\n", 203 | "import numpy as np\n", 204 | "import os\n", 205 | "\n", 206 | "def classify_deployed(file_name, classes , endpoint_name):\n", 207 | " payload = None\n", 208 | " with open(file_name, 'rb') as f:\n", 209 | " payload = f.read()\n", 210 | " payload = bytearray(payload)\n", 211 | "\n", 212 | " response = sess.sagemaker_runtime_client.invoke_endpoint(\n", 213 | " EndpointName = endpoint_name,\n", 214 | " ContentType = 'application/x-image',\n", 215 | " Body = payload)\n", 216 | " \n", 217 | " #print(response)\n", 218 | " result = response['Body'].read()\n", 219 | " result = json.loads(result)\n", 220 | " print(result)\n", 221 | " best_prob_index = np.argmax( np.array( result ) )\n", 222 | " print(best_prob_index)\n", 223 | " return (classes[best_prob_index], result[best_prob_index])" 224 | ] 225 | }, 226 | { 227 | "cell_type": "code", 228 | "execution_count": 8, 229 | "metadata": {}, 230 | "outputs": [ 231 | { 232 | "name": "stdout", 233 | "output_type": "stream", 234 | "text": [ 235 | "[0.0002461728290654719, 0.0017670468660071492, 0.00032288278453052044, 0.9976606369018555, 3.218596020815312e-06]\n", 236 | "3\n", 237 | "safe_for_work 0.9976606369018555\n" 238 | ] 239 | } 240 | ], 241 | "source": [ 242 | "file_name = 'flowes/test/daisy/10555826524_423eb8bf71_n.jpg'\n", 243 | "classes = ['animated' , 'nude' ,'porn' , 'safe_for_work' , 'semi_nude']\n", 244 | "\n", 245 | "class_predicted , class_score = classify_deployed(file_name, classes , endpoint_name)\n", 246 | "print(class_predicted , class_score)\n" 247 | ] 248 | }, 249 | { 250 | "cell_type": "markdown", 251 | "metadata": {}, 252 | "source": [ 253 | "## Deleting the endpoint." 254 | ] 255 | }, 256 | { 257 | "cell_type": "code", 258 | "execution_count": 9, 259 | "metadata": {}, 260 | "outputs": [ 261 | { 262 | "data": { 263 | "text/plain": [ 264 | "{'ResponseMetadata': {'RequestId': '75b459e9-b20b-466b-aab3-17afe9b8fa29',\n", 265 | " 'HTTPStatusCode': 200,\n", 266 | " 'HTTPHeaders': {'x-amzn-requestid': '75b459e9-b20b-466b-aab3-17afe9b8fa29',\n", 267 | " 'content-type': 'application/x-amz-json-1.1',\n", 268 | " 'content-length': '0',\n", 269 | " 'date': 'Wed, 11 Sep 2019 06:30:04 GMT'},\n", 270 | " 'RetryAttempts': 0}}" 271 | ] 272 | }, 273 | "execution_count": 9, 274 | "metadata": {}, 275 | "output_type": "execute_result" 276 | } 277 | ], 278 | "source": [ 279 | "sess.sagemaker_client.delete_endpoint(EndpointName = endpoint_name)" 280 | ] 281 | }, 282 | { 283 | "cell_type": "code", 284 | "execution_count": null, 285 | "metadata": {}, 286 | "outputs": [], 287 | "source": [] 288 | } 289 | ], 290 | "metadata": { 291 | "kernelspec": { 292 | "display_name": "Python 3", 293 | "language": "python", 294 | "name": "python3" 295 | }, 296 | "language_info": { 297 | "codemirror_mode": { 298 | "name": "ipython", 299 | "version": 3 300 | }, 301 | "file_extension": ".py", 302 | "mimetype": "text/x-python", 303 | "name": "python", 304 | "nbconvert_exporter": "python", 305 | "pygments_lexer": "ipython3", 306 | "version": "3.5.2" 307 | } 308 | }, 309 | "nbformat": 4, 310 | "nbformat_minor": 2 311 | } 312 | -------------------------------------------------------------------------------- /results_bench.csv: -------------------------------------------------------------------------------- 1 | Image_path,ground_truth_label,Predicted 2 | benchmark_images/animated/11.jpg,0,0 3 | benchmark_images/animated/12.jpg,0,0 4 | benchmark_images/animated/14.jpg,0,0 5 | benchmark_images/animated/3.jpg,0,2 6 | benchmark_images/animated/6.jpg,0,0 7 | benchmark_images/animated/7.jpg,0,0 8 | benchmark_images/animated/0.jpg,0,0 9 | benchmark_images/animated/4.jpg,0,0 10 | benchmark_images/animated/13.jpg,0,2 11 | benchmark_images/animated/8.jpg,0,0 12 | benchmark_images/animated/2.jpg,0,0 13 | benchmark_images/animated/10.jpg,0,0 14 | benchmark_images/animated/17.jpg,0,0 15 | benchmark_images/animated/5.jpg,0,2 16 | benchmark_images/animated/18.jpg,0,0 17 | benchmark_images/nude/15.jpg,1,2 18 | benchmark_images/nude/11.jpg,1,2 19 | benchmark_images/nude/12.jpg,1,4 20 | benchmark_images/nude/3.jpg,1,2 21 | benchmark_images/nude/16.jpg,1,4 22 | benchmark_images/nude/6.jpg,1,1 23 | benchmark_images/nude/0.jpg,1,4 24 | benchmark_images/nude/8.jpg,1,2 25 | benchmark_images/nude/2.jpg,1,2 26 | benchmark_images/nude/19.jpg,1,2 27 | benchmark_images/nude/9.jpg,1,2 28 | benchmark_images/nude/17.jpg,1,4 29 | benchmark_images/nude/5.jpg,1,2 30 | benchmark_images/nude/1.jpg,1,2 31 | benchmark_images/porn/15.jpg,2,4 32 | benchmark_images/porn/11.jpg,2,1 33 | benchmark_images/porn/12.jpg,2,2 34 | benchmark_images/porn/3.jpg,2,4 35 | benchmark_images/porn/16.jpg,2,2 36 | benchmark_images/porn/4.jpg,2,2 37 | benchmark_images/porn/13.jpg,2,4 38 | benchmark_images/porn/8.jpg,2,2 39 | benchmark_images/porn/9.jpg,2,4 40 | benchmark_images/porn/17.jpg,2,2 41 | benchmark_images/porn/5.jpg,2,2 42 | benchmark_images/porn/1.jpg,2,2 43 | benchmark_images/safe_for_work/15.jpg,3,2 44 | benchmark_images/safe_for_work/11.jpg,3,0 45 | benchmark_images/safe_for_work/12.jpg,3,1 46 | benchmark_images/safe_for_work/14.jpg,3,4 47 | benchmark_images/safe_for_work/3.jpg,3,0 48 | benchmark_images/safe_for_work/16.jpg,3,4 49 | benchmark_images/safe_for_work/6.jpg,3,4 50 | benchmark_images/safe_for_work/7.jpg,3,1 51 | benchmark_images/safe_for_work/0.jpg,3,4 52 | benchmark_images/safe_for_work/4.jpg,3,0 53 | benchmark_images/safe_for_work/13.jpg,3,4 54 | benchmark_images/safe_for_work/8.jpg,3,4 55 | benchmark_images/safe_for_work/2.jpg,3,4 56 | benchmark_images/safe_for_work/19.jpg,3,4 57 | benchmark_images/safe_for_work/9.jpg,3,2 58 | benchmark_images/safe_for_work/10.jpg,3,2 59 | benchmark_images/safe_for_work/17.jpg,3,4 60 | benchmark_images/safe_for_work/5.jpg,3,4 61 | benchmark_images/safe_for_work/1.jpg,3,4 62 | benchmark_images/safe_for_work/18.jpg,3,4 63 | benchmark_images/semi_nude/15.jpg,4,4 64 | benchmark_images/semi_nude/11.jpg,4,4 65 | benchmark_images/semi_nude/14.jpg,4,1 66 | benchmark_images/semi_nude/3.jpg,4,4 67 | benchmark_images/semi_nude/16.jpg,4,4 68 | benchmark_images/semi_nude/6.jpg,4,4 69 | benchmark_images/semi_nude/7.jpg,4,4 70 | benchmark_images/semi_nude/0.jpg,4,4 71 | benchmark_images/semi_nude/4.jpg,4,4 72 | benchmark_images/semi_nude/8.jpg,4,4 73 | benchmark_images/semi_nude/2.jpg,4,2 74 | benchmark_images/semi_nude/19.jpg,4,4 75 | benchmark_images/semi_nude/9.jpg,4,2 76 | benchmark_images/semi_nude/10.jpg,4,4 77 | benchmark_images/semi_nude/17.jpg,4,4 78 | benchmark_images/semi_nude/5.jpg,4,4 79 | benchmark_images/semi_nude/1.jpg,4,4 80 | benchmark_images/semi_nude/18.jpg,4,4 81 | -------------------------------------------------------------------------------- /useful_scripts/useful_scripts/csv_maker.py: -------------------------------------------------------------------------------- 1 | import os 2 | import csv 3 | import re 4 | import numpy as np 5 | 6 | def give_prediction_label(result_string): 7 | #x = re.match("[(.*)]", result_string) 8 | x = result_string[result_string.find('[')+1 : result_string.find(']')].split(',') 9 | x_float = [ float(xi) for xi in x ] 10 | ind = x_float.index(max(x_float)) 11 | 12 | return ind 13 | 14 | 15 | 16 | def make_results_csv( results_directory , csv_path , csv_file_name , eval_image_path ): 17 | with open(os.path.join(csv_path , csv_file_name), mode='w') as file: 18 | writer = csv.writer(file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 19 | writer.writerow(['Image_path', 'ground_truth_label', 'Predicted']) 20 | i = 0 21 | sorted_list = sorted(os.listdir(results_directory)) 22 | for single_class in sorted_list: 23 | list_of_files = os.listdir(os.path.join( results_directory , single_class )) 24 | for img_result in list_of_files: 25 | result_file = open( os.path.join( results_directory , single_class , img_result ) , 'r') 26 | result_string = result_file.read() 27 | prediction_val = give_prediction_label(result_string) 28 | #print(result_string.split(',')) 29 | label = str(i) 30 | img_path = os.path.join( eval_image_path , single_class ,img_result[:-4] ) 31 | writer.writerow([img_path, label , prediction_val ]) 32 | result_file.close() 33 | #print(i) 34 | i += 1 35 | #make_results_csv('analyze/analyze' , 'dump' , 'results.csv' , 'nsfw_dataset_test' ) -------------------------------------------------------------------------------- /useful_scripts/useful_scripts/example.py: -------------------------------------------------------------------------------- 1 | """ 2 | A simple example of hooking the keyboard on Linux using pyxhook 3 | 4 | Any key pressed prints out the keys values, program terminates when spacebar 5 | is pressed. 6 | """ 7 | from __future__ import print_function 8 | 9 | # Libraries we need 10 | import pyxhook 11 | import time 12 | 13 | 14 | import sys, termios, tty, os, time 15 | import autopy 16 | 17 | import os, os.path 18 | 19 | try : 20 | os.makedirs('nude') 21 | except FileExistsError as e : 22 | pass 23 | 24 | try : 25 | os.makedirs('semi_nude') 26 | except FileExistsError as e : 27 | pass 28 | 29 | try : 30 | os.makedirs('animated') 31 | except FileExistsError as e : 32 | pass 33 | 34 | try : 35 | os.makedirs('porn') 36 | except FileExistsError as e : 37 | pass 38 | 39 | def getch(): 40 | fd = sys.stdin.fileno() 41 | old_settings = termios.tcgetattr(fd) 42 | try: 43 | tty.setraw(sys.stdin.fileno()) 44 | ch = sys.stdin.read(1) 45 | 46 | finally: 47 | termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) 48 | return ch 49 | 50 | button_delay = 0.2 51 | 52 | 53 | DIR = 'porn' 54 | n_porn = len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) 55 | 56 | DIR = 'semi_nude' 57 | n_semi_nude = len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) 58 | 59 | DIR = 'nude' 60 | n_nude = len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) 61 | 62 | DIR = 'animated' 63 | n_animated = len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) 64 | 65 | print(n_nude , n_semi_nude , n_porn , n_animated) 66 | 67 | 68 | 69 | 70 | 71 | """ 72 | while True: 73 | char = getch() 74 | 75 | if (char == "p"): 76 | print("Stop!") 77 | exit(0) 78 | 79 | if (char == "a"): 80 | print("Left pressed") 81 | n_nude += 1 82 | bmp = autopy.bitmap.capture_screen() 83 | bmp.save('nude/nude{}.png'.format(n_nude)) 84 | time.sleep(button_delay) 85 | 86 | elif (char == "d"): 87 | print("Right pressed") 88 | n_semi_nude += 1 89 | bmp = autopy.bitmap.capture_screen() 90 | bmp.save('semi_nude/semi_nude{}.png'.format(n_semi_nude)) 91 | time.sleep(button_delay) 92 | 93 | elif (char == "w"): 94 | print("Up pressed") 95 | n_animated += 1 96 | bmp = autopy.bitmap.capture_screen() 97 | bmp.save('animated/animated{}.png'.format(n_animated)) 98 | time.sleep(button_delay) 99 | 100 | elif (char == "s"): 101 | print("Down pressed") 102 | n_porn += 1 103 | bmp = autopy.bitmap.capture_screen() 104 | bmp.save('porn/porn{}.png'.format(n_porn)) 105 | time.sleep(button_delay) 106 | 107 | elif (char == "1"): 108 | print("Number 1 pressed") 109 | time.sleep(button_delay) 110 | """ 111 | 112 | 113 | 114 | # This function is called every time a key is presssed 115 | def kbevent(event): 116 | global running 117 | global n_nude 118 | global n_semi_nude 119 | global n_animated 120 | global n_porn 121 | global button_delay 122 | # print key info 123 | #print(event) 124 | 125 | # If the ascii value matches spacebar, terminate the while loop 126 | if event.Ascii == 113: 127 | running = False 128 | print('quitting') 129 | elif event.Ascii == 110: 130 | print("Left pressed") 131 | n_nude += 1 132 | bmp = autopy.bitmap.capture_screen() 133 | bmp.save('nude/nude{}.png'.format(n_nude)) 134 | time.sleep(button_delay) 135 | elif event.Ascii == 115: 136 | print("Right pressed") 137 | n_semi_nude += 1 138 | bmp = autopy.bitmap.capture_screen() 139 | bmp.save('semi_nude/semi_nude{}.png'.format(n_semi_nude)) 140 | time.sleep(button_delay) 141 | elif event.Ascii == 112: 142 | print("up pressed") 143 | n_porn += 1 144 | bmp = autopy.bitmap.capture_screen() 145 | bmp.save('porn/porn{}.png'.format(n_porn)) 146 | time.sleep(button_delay) 147 | elif event.Ascii == 97: 148 | print("down pressed") 149 | n_animated += 1 150 | bmp = autopy.bitmap.capture_screen() 151 | bmp.save('animated/animated{}.png'.format(n_animated)) 152 | time.sleep(button_delay) 153 | #Create hookmanager 154 | hookman = pyxhook.HookManager() 155 | # Define our callback to fire when a key is pressed down 156 | hookman.KeyDown = kbevent 157 | # Hook the keyboard 158 | hookman.HookKeyboard() 159 | # Start our listener 160 | hookman.start() 161 | 162 | # Create a loop to keep the application running 163 | running = True 164 | 165 | button_delay = 0.2 166 | 167 | 168 | DIR = 'porn' 169 | n_porn = len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) 170 | 171 | DIR = 'semi_nude' 172 | n_semi_nude = len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) 173 | 174 | DIR = 'nude' 175 | n_nude = len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) 176 | 177 | DIR = 'animated' 178 | n_animated = len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]) 179 | 180 | #print(n_nude , n_semi_nude , n_porn , n_animated) 181 | 182 | while running: 183 | time.sleep(0.1) 184 | 185 | # Close the listener when we are done 186 | hookman.cancel() 187 | -------------------------------------------------------------------------------- /useful_scripts/useful_scripts/image_dataset_handler.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | from torch.utils.data.dataset import Dataset 4 | from torch.utils.data import DataLoader 5 | import numpy as np 6 | import os 7 | from PIL import Image 8 | 9 | 10 | 11 | 12 | def create_list_file_in_directory(datadir): 13 | i = 0 14 | os.remove(os.path.join(datadir,'data.txt')) 15 | for single_label in os.listdir(datadir): 16 | single_full_list = os.listdir(os.path.join(datadir , single_label)) 17 | for file_name in single_full_list: 18 | file = open( os.path.join(datadir,'data.txt') ,'a') 19 | file.write( '{} {} \n'.format(os.path.join( single_label , file_name) , str(i) ) ) 20 | file.close() 21 | i += 1 22 | 23 | 24 | 25 | 26 | class DriveData(Dataset): 27 | __xs = [] 28 | __ys = [] 29 | 30 | def __init__(self, folder_dataset, transform=None , resize_tuple = (224, 224)): 31 | self.transform = transform 32 | self.resize_tuple = resize_tuple 33 | # Open and load text file including the whole training data 34 | with open( os.path.join(folder_dataset , "data.txt" ) ) as f: 35 | for line in f: 36 | # Image path 37 | self.__xs.append( os.path.join( folder_dataset ,line.split()[0]) ) 38 | # Steering wheel label 39 | self.__ys.append(np.float32(line.split()[1])) 40 | 41 | # Override to give PyTorch access to any image on the dataset 42 | def __getitem__(self, index): 43 | img = Image.open(self.__xs[index]) 44 | img = img.convert('RGB') 45 | #img= img.resize( self.resize_tuple ) 46 | if self.transform is not None: 47 | img = self.transform(img) 48 | 49 | # Convert image and label to torch tensors 50 | img = torch.from_numpy(np.asarray(img)) 51 | label = torch.from_numpy(np.asarray(self.__ys[index]).reshape([1,1])) 52 | return img, label 53 | 54 | # Override to give PyTorch size of dataset 55 | def __len__(self): 56 | return len(self.__xs) 57 | 58 | 59 | def build_image_dataset(datadir , batch_size=10, shuffle=True, num_workers=1 , transform=None , resize_tuple = (224, 224) ): 60 | create_list_file_in_directory(datadir) 61 | dset_created = DriveData(datadir,transform , resize_tuple) 62 | 63 | #img , label = dset_train.__getitem__(0) 64 | 65 | data_loader = DataLoader(dset_created, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) 66 | return data_loader 67 | -------------------------------------------------------------------------------- /useful_scripts/useful_scripts/image_edit.py: -------------------------------------------------------------------------------- 1 | 2 | #from PIL import Image 3 | 4 | #img =Image.open('semi_nude/semi_nude34.png') 5 | #img.show() 6 | 7 | #import matplotlib.pyplot as plt 8 | #img= plt.imread('porn/porn1.png') 9 | 10 | #fig, ax = plt.subplots() 11 | #im = ax.imshow(img) 12 | 13 | #plt.show() 14 | 15 | 16 | import matplotlib.pyplot as plt 17 | import numpy as np 18 | import time 19 | from datetime import timedelta 20 | import os 21 | import glob , pickle 22 | import sys 23 | # Functions and classes for loading and using the Inception model. 24 | #import inception 25 | from PIL import Image 26 | # We use Pretty Tensor to define the new classifier. 27 | #import prettytensor as pt 28 | 29 | 30 | def get_data(datadir, img_rows , img_cols , crop_tuple=(68,77,1230,714) ): 31 | # datadir = args.data 32 | # assume each image is 512x256 split to left and right 33 | # print(datadir) 34 | imgs = glob.glob(os.path.join(datadir, '*.png')) 35 | print(len(imgs)) 36 | # imgs = glob.glob(datadir) 37 | # print(len(imgs)) 38 | for file in imgs: 39 | img = Image.open(file) 40 | width, height = img.size 41 | if(height > img_rows and width > img_cols): 42 | # img = img.resize((img_cols, img_rows), Image.LANCZOS) 43 | img = img.crop(crop_tuple) 44 | img = img.resize((img_cols, img_rows), Image.NONE) 45 | img.save(file , optimize=True,quality=95) 46 | for file in imgs: 47 | img = Image.open(file) 48 | img.save( file[:-4]+'.jpg' ) 49 | os.remove(file) 50 | 51 | def crop(image_path, coords, saved_location): 52 | image_obj = Image.open(image_path) 53 | cropped_image = image_obj.crop(coords) 54 | cropped_image.save(saved_location) 55 | cropped_image.show() 56 | 57 | 58 | 59 | if __name__ == '__main__': 60 | get_data('semi_nude', 400 , 600 ) 61 | 62 | 63 | -------------------------------------------------------------------------------- /useful_scripts/useful_scripts/split_dataset.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import shutil 4 | 5 | # # Creating Train / Val / Test folders (One time use) 6 | 7 | def split_dataset_one_time(root_dir , valid_size, test_size , train_path , valid_path , test_path): 8 | 9 | 10 | class_names = os.listdir(root_dir) 11 | 12 | #print(class_names) 13 | for single_class in class_names: 14 | try : 15 | 16 | 17 | os.makedirs(os.path.join( train_path , single_class)) 18 | os.makedirs(os.path.join( valid_path , single_class)) 19 | os.makedirs(os.path.join( test_path , single_class)) 20 | 21 | 22 | train_single_path = os.path.join( train_path , single_class) 23 | test_single_path = os.path.join( test_path , single_class) 24 | valid_single_path = os.path.join( valid_path , single_class) 25 | except FileExistsError: 26 | pass 27 | 28 | single_full_list = os.listdir(os.path.join(root_dir , single_class)) 29 | #print(single_full_list[0]) 30 | 31 | num_examples_per_class = len(single_full_list) 32 | indices = list(range(num_examples_per_class)) 33 | split_valid = int(np.floor(valid_size * num_examples_per_class)) 34 | split_test = int(np.floor(test_size * num_examples_per_class)) + split_valid 35 | np.random.shuffle(indices) 36 | train_idx, valid_idx , test_idx = indices[split_test:], indices[:split_valid] , indices[split_valid:split_test] 37 | 38 | #print(train_idx[:5]) 39 | single_full_list_np = np.array(single_full_list) 40 | train_list , valid_list , test_list = single_full_list_np[train_idx].tolist() , single_full_list_np[valid_idx].tolist() , single_full_list_np[test_idx].tolist() 41 | 42 | #print(len(train_list) , len(valid_list) , len(test_list) , len(single_full_list)) 43 | 44 | 45 | 46 | for name in train_list: 47 | #print(os.path.join(root_dir , single_class , name) , train_single_path) 48 | shutil.move( os.path.join(root_dir , single_class , name) , train_single_path) 49 | 50 | for name in valid_list: 51 | #print(os.path.join(root_dir , single_class , name) , valid_single_path) 52 | shutil.move(os.path.join(root_dir , single_class , name) ,valid_single_path) 53 | 54 | for name in test_list: 55 | #print(os.path.join(root_dir , single_class , name) , test_single_path) 56 | shutil.move(os.path.join(root_dir , single_class , name), test_single_path) 57 | 58 | --------------------------------------------------------------------------------