├── src ├── sample │ ├── test.png │ ├── data │ │ ├── test │ │ │ ├── neutral │ │ │ │ ├── neutral_1.png │ │ │ │ ├── neutral_11.png │ │ │ │ ├── neutral_12.png │ │ │ │ ├── neutral_3.png │ │ │ │ ├── neutral_4.png │ │ │ │ ├── neutral_5.png │ │ │ │ ├── neutral_6.png │ │ │ │ ├── neutral_7.png │ │ │ │ ├── neutral_8.png │ │ │ │ └── neutral_9.png │ │ │ ├── negative │ │ │ │ ├── negative_1.png │ │ │ │ ├── negative_11.png │ │ │ │ ├── negative_12.png │ │ │ │ ├── negative_13.png │ │ │ │ ├── negative_15.png │ │ │ │ ├── negative_3.png │ │ │ │ ├── negative_4.png │ │ │ │ ├── negative_5.png │ │ │ │ ├── negative_6.png │ │ │ │ ├── negative_7.png │ │ │ │ ├── negative_8.png │ │ │ │ └── negative_9.png │ │ │ └── positive │ │ │ │ ├── positive_1.png │ │ │ │ ├── positive_11.png │ │ │ │ ├── positive_12.png │ │ │ │ ├── positive_13.png │ │ │ │ ├── positive_15.png │ │ │ │ ├── positive_3.png │ │ │ │ ├── positive_4.png │ │ │ │ ├── positive_5.png │ │ │ │ ├── positive_6.png │ │ │ │ ├── positive_7.png │ │ │ │ ├── positive_8.png │ │ │ │ └── positive_9.png │ │ ├── train │ │ │ ├── neutral │ │ │ │ ├── neutral_2.png │ │ │ │ ├── neutral_10.png │ │ │ │ └── neutral_13.png │ │ │ ├── negative │ │ │ │ ├── negative_10.png │ │ │ │ ├── negative_14.png │ │ │ │ └── negative_2.png │ │ │ └── positive │ │ │ │ ├── positive_10.png │ │ │ │ ├── positive_14.png │ │ │ │ └── positive_2.png │ │ └── README.md │ ├── README.md │ ├── cv │ │ └── im_capture.py │ ├── config.py │ ├── run.py │ └── ml │ │ └── ml_model.py └── README.md ├── requirements.txt ├── LICENSE ├── .gitignore └── README.md /src/sample/test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/test.png -------------------------------------------------------------------------------- /src/sample/data/test/neutral/neutral_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/neutral/neutral_1.png -------------------------------------------------------------------------------- /src/sample/data/test/neutral/neutral_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/neutral/neutral_11.png -------------------------------------------------------------------------------- /src/sample/data/test/neutral/neutral_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/neutral/neutral_12.png -------------------------------------------------------------------------------- /src/sample/data/test/neutral/neutral_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/neutral/neutral_3.png -------------------------------------------------------------------------------- /src/sample/data/test/neutral/neutral_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/neutral/neutral_4.png -------------------------------------------------------------------------------- /src/sample/data/test/neutral/neutral_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/neutral/neutral_5.png -------------------------------------------------------------------------------- /src/sample/data/test/neutral/neutral_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/neutral/neutral_6.png -------------------------------------------------------------------------------- /src/sample/data/test/neutral/neutral_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/neutral/neutral_7.png -------------------------------------------------------------------------------- /src/sample/data/test/neutral/neutral_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/neutral/neutral_8.png -------------------------------------------------------------------------------- /src/sample/data/test/neutral/neutral_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/neutral/neutral_9.png -------------------------------------------------------------------------------- /src/sample/data/train/neutral/neutral_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/train/neutral/neutral_2.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_1.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_11.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_12.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_13.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_15.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_3.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_4.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_5.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_6.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_7.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_8.png -------------------------------------------------------------------------------- /src/sample/data/test/negative/negative_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/negative/negative_9.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_1.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_11.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_12.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_13.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_15.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_3.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_4.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_5.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_6.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_7.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_8.png -------------------------------------------------------------------------------- /src/sample/data/test/positive/positive_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/test/positive/positive_9.png -------------------------------------------------------------------------------- /src/sample/data/train/negative/negative_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/train/negative/negative_10.png -------------------------------------------------------------------------------- /src/sample/data/train/negative/negative_14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/train/negative/negative_14.png -------------------------------------------------------------------------------- /src/sample/data/train/negative/negative_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/train/negative/negative_2.png -------------------------------------------------------------------------------- /src/sample/data/train/neutral/neutral_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/train/neutral/neutral_10.png -------------------------------------------------------------------------------- /src/sample/data/train/neutral/neutral_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/train/neutral/neutral_13.png -------------------------------------------------------------------------------- /src/sample/data/train/positive/positive_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/train/positive/positive_10.png -------------------------------------------------------------------------------- /src/sample/data/train/positive/positive_14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/train/positive/positive_14.png -------------------------------------------------------------------------------- /src/sample/data/train/positive/positive_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/powerhouseofthecell/machine_feeling/HEAD/src/sample/data/train/positive/positive_2.png -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | # The Code :hushed: 2 | Here it is, the code you've been looking to get your hands on. Before you dive in, just know that the sample code is functional - running run.py from the sample/ directory will deliver a product...that may or may not work well. As we walk through the process, if you notice an improvement you'd like to make (adjusting the machine learning model, for example), feel free to do so. Experiment with what changes you can make on how well the model learns. -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | backports.weakref==1.0rc1 2 | bleach==1.5.0 3 | Django==1.11.6 4 | h5py==2.7.0 5 | html5lib==0.9999999 6 | image==1.5.16 7 | Keras==2.0.6 8 | Markdown==2.6.8 9 | mss==3.0.1 10 | numpy==1.13.1 11 | olefile==0.44 12 | opencv-python==3.3.0.10 13 | pandas==0.20.3 14 | Pillow==4.3.0 15 | protobuf==3.3.0 16 | python-dateutil==2.6.1 17 | pytz==2017.2 18 | PyYAML==3.12 19 | scipy==0.19.1 20 | six==1.10.0 21 | tensorflow==1.2.1 22 | Theano==0.9.0 23 | Werkzeug==0.12.2 24 | -------------------------------------------------------------------------------- /src/sample/README.md: -------------------------------------------------------------------------------- 1 | # The Sample Code :grin:! 2 | In this directory, there is sample code for the software we may want to build. There is a lot of stuff to look at, but please don't feel overwhelmed. Everything has been commented and README'd as much as was thought reasonable. This was to make the experience a little less terrifying. 3 | 4 | ----- 5 | 6 | #### A Short Tour :astonished: 7 | So, looking to our left :point_left:, we have the wonderful wildlands of computer vision (under the directory "cv/"), and to the right :point_right:, we have the fiery plains of machine learning ("ml/"). Looking straight ahead :point_up_2:, you'll see the actual data from which we will pull. And underneath your seats is the mastermind behind it all - "run.py". This file contains the actual application code that will allow you to take pictures of (and classify) emoji! 8 | 9 | ----- 10 | 11 | #### Explore :sunglasses:! 12 | Go and take a look around! -------------------------------------------------------------------------------- /src/sample/data/README.md: -------------------------------------------------------------------------------- 1 | # The (Not So Big) Data :sweat_smile: 2 | This is the directory in which our data is stored. If we were Google, or Apple, or Amazon, or Microsoft, we might have billions of data points. Instead, we are a college student with little sleep, and so we have about 46 data points. Even with this "limitation," you'll find that we're able to do a surprisingly large amount. Feel free to peruse the data at will! (This isn't a museum, you are welcome to touch/edit the data) 3 | 4 | ----- 5 | ### Train vs Test :zzz: 6 | Basically, we use the training dataset to try and teach our baby :baby: machine some patterns, and then use the testing one to test its accuracy. Ideally, data in the testing dataset should not have ever been seen before by the machine (i.e. in training). Within each of the folders of this directory, is a folder or *class* with the images belonging to that class inside. If you don't believe me, feel free to go take a look! -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Nicholas Wong 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | # custom ignore 104 | .vscode/ 105 | *.h5 -------------------------------------------------------------------------------- /src/sample/cv/im_capture.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | from PIL import ImageGrab 4 | 5 | # set some global variables 6 | is_clicked = False 7 | coors = list() 8 | loop = True 9 | filename = 'tmp' 10 | 11 | def run(filepath='tmp'): 12 | global img, loop, filename 13 | 14 | # set the file path if it was passed 15 | filename = filepath 16 | 17 | try: 18 | # create a window to hold the feed, and set up the mouse callback 19 | cv2.namedWindow('Feed', cv2.WINDOW_NORMAL) 20 | cv2.setMouseCallback('Feed', click_and_crop) 21 | 22 | # infinitely monitor the screen (albeit somewhat slowly) 23 | while loop: 24 | # grabs a screenshot of the entire screen 25 | raw_grab = ImageGrab.grab() 26 | 27 | # converts that screenshot to a NumPy array (and array of numbers) 28 | img = np.array(raw_grab) 29 | 30 | # show the image on the display window we created earlier 31 | cv2.imshow('Feed', img) 32 | 33 | # if the user quits as with pressing 'q' 34 | if cv2.waitKey(25) & 0xFF == ord('q'): 35 | cv2.destroyAllWindows() 36 | break 37 | 38 | # close any open windows 39 | cv2.destroyAllWindows() 40 | 41 | # some basic error handling 42 | except KeyboardInterrupt: 43 | cv2.destroyAllWindows() 44 | 45 | except Exception as err: 46 | cv2.destroyAllWindows() 47 | print(err) 48 | exit(1) 49 | 50 | # a mouse callback that allows our mouse events to be registered and perform some actions 51 | def click_and_crop(event, x, y, flags, param): 52 | global coors, is_clicked, loop 53 | 54 | # if the button is pressed 55 | if event == cv2.EVENT_LBUTTONDOWN and not is_clicked: 56 | coors.append((x, y)) 57 | is_clicked = True 58 | 59 | # if the button is released 60 | elif event == cv2.EVENT_LBUTTONUP and is_clicked: 61 | coors.append((x, y)) 62 | is_clicked = False 63 | 64 | # if we have two coordinates 65 | if len(coors) == 2 and not is_clicked: 66 | # take the selection and write it to file 67 | cv2.imwrite(filename + '.png', img[coors[0][1]:coors[1][1], coors[0][0]:coors[1][0]]) 68 | 69 | # stop looping 70 | loop = False 71 | -------------------------------------------------------------------------------- /src/sample/config.py: -------------------------------------------------------------------------------- 1 | ############################################ 2 | ################ CONFIG #################### 3 | ############################################ 4 | # this file holds all of the configuration # 5 | # variables for this project. # 6 | ############################################ 7 | ############################################ 8 | ############################################ 9 | 10 | from os import listdir 11 | 12 | # the available model types to work with 13 | available_model_types = ['mul_class'] 14 | 15 | 16 | # Data Generator Parameters 17 | ## rotation range for images, in degrees 18 | rot_range = 0 19 | 20 | ## how much to shift the width and height, as proportions 21 | w_shift = 0.2 22 | h_shift = 0.1 23 | 24 | ## how much to shear images (see Internet for specifics) 25 | shear = 0.01 26 | 27 | ## horizontal and vertical flipping of images 28 | h_flip = False 29 | v_flip = False 30 | 31 | 32 | # Data Parameters 33 | ## the dimensions of the images, rows then cols 34 | target_dim1 = 200 35 | target_dim2 = 200 36 | 37 | ## how many images to generate at a time 38 | batch_size = 200 39 | 40 | ## the type of classifier, if applicable 41 | classification_type = 'categorical' 42 | 43 | 44 | # Model Parameters 45 | ## activation distributions for the layers 46 | activations = ['relu', 'softmax'] 47 | 48 | ## dimensionality of a filter to be run over the image 49 | filter_dim = (3, 3) 50 | 51 | # the number of classes found in test and train directories 52 | num_classes = len(listdir('data/train')) - 1 53 | 54 | 55 | # Compilation Parameters - see Keras docs for lists 56 | ## the metric for loss, found in Keras documentation 57 | l_type = classification_type + '_crossentropy' 58 | 59 | ## the optimizer, a tool for how weights are decided 60 | opt = 'adam' 61 | 62 | ## the metrics against which things are judged, can be several 63 | met = ['accuracy'] 64 | 65 | 66 | # Train Parameters 67 | ## the number of steps to take in a given round (this will later be divided by the batch size) 68 | step_num = 4000 69 | 70 | ## the number of rounds to run 71 | epoch_num = 15 72 | 73 | ## number of workers to unleash on doing calculations 74 | worker_num = 4 75 | 76 | ## whether or not to use multiprocessing 77 | multiprocessing = False 78 | 79 | 80 | # Evaluation Parameters 81 | ## the number of evaluation steps to run 82 | eval_steps = 10 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # :smiley: machine_feeling :smiley: 2 | An Emoji-ful introduction to computer vision and machine learning. 3 | 4 | ----- 5 | 6 | ### Introduction :smirk: 7 | This small project is intended as a beginning to some of the fundamental ideas of machine learning and computer vision. Specifically, this is a look at the problem of categorizing images (along with some potential solutions). 8 | 9 | Our goal in this project is to walk through how something like this may work. 10 | 11 | ----- 12 | 13 | ### Goal :relaxed: 14 | The problem herein is that we want to be able to identify whether the emoji we see on the Internet is positive :blush:, neutral :neutral_face:, or negative :angry:. In this (somewhat contrived) scenario, we, the enterprising young computer scientists, have decided to build software to perform several tasks in accomplishing this goal: 15 | 1. We need a way to "take a picture" or photograph something within our computer (i.e. something we see on the Internet). We could do this by using our phone or a screenshot software. But we are grade-A hipsters, so we'll be using OpenCV, a computer vision (*buzzword!*) platform, to build this ourselves. 16 | 2. This picture should then be plopped into some machine-learning algorithm (*yay! more buzzwords!*) that will tell us (hopefully), which if our emoji is looking positive, negative or somewhere in between. 17 | 18 | ----- 19 | 20 | ### Some Notes :grin: 21 | - Yes, this is an arguably absurdly contrived example on which to learn some basic machine learning and computer vision. 22 | - Yes, there are likely better ways to do this than using machine learning, especially since we can usually tell what emotion an emoji is already 23 | - Given the above two things, we designed the project this way because it seemed fun and entertaining 24 | - Please do not take this project in any way, shape, or form as a representation of *all* that either machine learning or computer vision are capable - this barely scratches the surface of either field 25 | 26 | ----- 27 | 28 | ### To Get Started 29 | - First, I would recommend using some form of dedicated environment, as with virtualenv, but if you don't wish to, that's ok too 30 | - The rest is as easy as installing the requirements with ```pip install -r requirements.txt``` 31 | - Then, navigate to the directory containing run (assuming you're in the machine_feeling/ directory, ```cd src; cd sample``` and run "run.py" ```./run.py -h``` 32 | - If you receive a message about your ability to run that file, go ahead and change the permissions ```chmod +x run.py``` and you should be able to run it 33 | - PLEASE NOTE, DUE TO A BUG WITH KERAS, YOU WILL LIKELY GET AN ERROR ABOUT SUBTRACTION AND NEGATIVE SIZES: 34 | - the way I recommend is modifying the file at ```~/.keras/keras.json``` where it reads ```"image_dim_ordering": "something here"```, and change that row to read ```"image_dim_ordering": "tf"``` 35 | - YOU MAY ALSO GET AN ERROR ABOUT INPUT DIMENSIONS, IN WHICH CASE YOU SHOULD CHANGE THE MODEL SUCH THAT THE LAYERS PASS ON THE CORRECT DIMENSIONALITY TO THE LAYERS AFTER THEM 36 | - All bugs should be fixed for distro code, please do let me know if anything doesn't work! 37 | 38 | ----- 39 | 40 | ### Conclusion :heart: 41 | We sincerely hope that you enjoy working through this project and go on to pursue your interest in both machine learning and computer vision! 42 | 43 | If you find a bug, a typo, or simply wish to contribute, please do make a pull request with your edit! 44 | 45 | Sincerely, 46 | 47 | 48 | potc 49 | -------------------------------------------------------------------------------- /src/sample/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | from os.path import isfile 5 | from os import remove, listdir 6 | 7 | # initialize an argument parser 8 | ap = argparse.ArgumentParser() 9 | 10 | # add an argument for output name, else saves to default name 11 | ap.add_argument( 12 | '-o', 13 | '--output', 14 | metavar='OUTPUT', 15 | type=str, 16 | help='the filename to write the model to' 17 | ) 18 | 19 | # add an argument for how many rounds of training to undergo 20 | ap.add_argument( 21 | '-r', 22 | '--rounds', 23 | metavar='ROUNDS', 24 | default=1, 25 | type=int, 26 | help='the number of rounds of training to undergo' 27 | ) 28 | 29 | # add an argument for loading in a model 30 | ap.add_argument( 31 | '-l', 32 | '--load', 33 | metavar='LOAD', 34 | type=str, 35 | help='the filepath of an existing model which you want to load' 36 | ) 37 | 38 | ap.add_argument( 39 | '-i', 40 | '--img', 41 | metavar='IMG', 42 | type=str, 43 | default='tmp', 44 | help='the filepath to save screenshots to' 45 | ) 46 | 47 | # add an argument for using a pretrained model 48 | ap.add_argument('--pretrained', action='store_true', help='use a pretrained model') 49 | 50 | # add an argument for if you've loaded a model, still training it again 51 | ap.add_argument('-t', action='store_true', help='force training of the model') 52 | 53 | # add an argument for automatically saving the model 54 | ap.add_argument('-y', action='store_true', help='autosave the model') 55 | 56 | args = vars(ap.parse_args()) 57 | 58 | # import the model after parsing arguments, since it takes a bit 59 | from ml.ml_model import Model 60 | 61 | # instantiate a model for the whole file to use 62 | m = Model() 63 | 64 | # get the model built and ready to go 65 | def initialize(): 66 | m.load_data('data') 67 | 68 | if args['pretrained']: 69 | m.load_model('', pretrained=True) 70 | 71 | elif not args['load']: 72 | m.build_model() 73 | 74 | else: 75 | m.load_model(args['load']) 76 | 77 | # train the model n rounds 78 | def train_model(n): 79 | # only train if forced or if nothing was loaded 80 | if (args['t'] or not args['load']) and not args['pretrained']: 81 | for i in range(n): 82 | m.train() 83 | 84 | # save the model 85 | def save_model(): 86 | # in case the user wanted to save the model automatically 87 | if args['y']: 88 | ans = 'y' 89 | else: 90 | ans = '' 91 | 92 | # check with the user and see if they want to save 93 | while not (ans == 'y' or ans == 'n'): 94 | ans = input('Do you want to save the model? (y/N) ').lower() 95 | 96 | if ans == 'y': 97 | m.save(save_path=args['output']) 98 | 99 | def run_im_capture(): 100 | # import computer vision after the machine learning has gone well 101 | import cv.im_capture as im 102 | 103 | # runs the image capture and lets you save the screenshot 104 | im.run(filepath=args['img']) 105 | 106 | 107 | def predict(): 108 | # we need some extra packages 109 | from keras.preprocessing.image import img_to_array, load_img 110 | from config import target_dim1, target_dim2 111 | import numpy as np 112 | 113 | # if the image exists, and we're using a pretrained model 114 | if isfile(args['img'] + '.png') and args['pretrained']: 115 | image = load_img(args['img'] + '.png', target_size=(299, 299)) 116 | 117 | elif isfile(args['img'] + '.png'): 118 | image = load_img(args['img'] + '.png', target_size=(target_dim1, target_dim2)) 119 | 120 | # convert the image to a proper numpy array 121 | image = img_to_array(image) 122 | 123 | # add a dimension to this array so that it fits what our model expects 124 | image = np.expand_dims(image, axis=0) 125 | 126 | # actually run the prediction 127 | m.predict(image) 128 | 129 | # remove the temporary file we created 130 | remove(args['img'] + '.png') 131 | 132 | # a method for actually evaluating the model 133 | def evaluate(): 134 | m.evaluate() 135 | return m.evaluation 136 | 137 | # TODO: Make this into some form of loop for interacting live with the model, a la an interpreter 138 | # run the actual file 139 | if __name__ == '__main__': 140 | initialize() 141 | 142 | train_model(args['rounds']) 143 | 144 | save_model() 145 | 146 | run_im_capture() 147 | 148 | predict() 149 | 150 | for k in m.prediction: 151 | print('It is {} with {}% likelihood!'.format(k, 100 * m.prediction[k])) 152 | 153 | # single letter variable names seemed apropos here, may need to change later 154 | e = evaluate() 155 | 156 | print('On evaluation, metrics were as follows:') 157 | for k in e: 158 | print('\t{}: {}'.format(k, e[k] * 100)) 159 | 160 | 161 | 162 | 163 | -------------------------------------------------------------------------------- /src/sample/ml/ml_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime as d 3 | 4 | ####################################################################### 5 | ################ THE MACHINE LEARNING MODEL CLASS ##################### 6 | ####################################################################### 7 | #### Feel free to play around with how this class is built as you #### 8 | #### see fit. It was designed as a wrapper around Keras' nice, #### 9 | #### built-in classes, in order to provide an extremely high level #### 10 | #### interface. If there are things you wish to modify, feel free. #### 11 | ####################################################################### 12 | 13 | # will hide TensorFlow warnings, comment or remove to have those warnings back 14 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' 15 | 16 | # import the pieces from keras to make the model work 17 | from keras.preprocessing.image import ImageDataGenerator 18 | from keras.models import Sequential, load_model 19 | from keras.layers import AlphaDropout, GaussianNoise, Conv2D, MaxPooling2D, Activation, Dropout, Flatten, Dense 20 | from keras import optimizers 21 | 22 | from keras.applications.inception_v3 import preprocess_input 23 | from keras.applications import imagenet_utils 24 | 25 | # import our configuration variables 26 | from config import * 27 | 28 | # define a decorator that makes sure we don't try to run something like a prediction without a model 29 | def model_required(f): 30 | def wrapper(*args, **kwargs): 31 | if args[0].model: 32 | f(*args, **kwargs) 33 | else: 34 | print('[-] Please compile the model using the "build_model" method or load a model before attempting this') 35 | return wrapper 36 | 37 | # also a decorator, makes sure we don't try to train a model without data 38 | def data_required(f): 39 | def wrapper(*args, **kwargs): 40 | if args[0].loaded: 41 | f(*args, **kwargs) 42 | else: 43 | print('[-] Please load data using the "load_data" method before attempting this') 44 | return wrapper 45 | 46 | # the class definition 47 | class Model(): 48 | # initialization of the class object, defaults to a multi-class classifier 49 | def __init__(self, model_type='mul_class'): 50 | self.model_type = model_type 51 | self.model = None 52 | self.loaded = False 53 | 54 | # if the selected model is not available, say as much 55 | if not model_type in available_model_types: 56 | print( 57 | '[-] Invalid model type input - {} - please use one of the following: {}'.format( 58 | model_type, 59 | ' '.join([ x for x in available_model_types ]) 60 | ) 61 | ) 62 | 63 | # a method for loading the model if it has already been created and saved 64 | def load_model(self, model_path, pretrained=False): 65 | self.pretrained = pretrained 66 | 67 | if not pretrained: 68 | try: 69 | if os.path.isfile(model_path): 70 | self.model = load_model(model_path) 71 | print('[+] Model loading complete') 72 | 73 | else: 74 | print('[-] Model loading incomplete, could not find model - {}'.format(model_path)) 75 | 76 | except Exception as err: 77 | print('[-] Model loading unsuccessful, please check your model file:') 78 | print(err) 79 | else: 80 | from keras.applications import InceptionV3 81 | self.model = InceptionV3(weights='imagenet') 82 | 83 | # a "begin" marker to time how long it takes (in real time) to compile 84 | start_compile = d.now() 85 | 86 | # actually compile the model 87 | self.model.compile( 88 | loss=l_type, 89 | optimizer=opt, 90 | metrics=met 91 | ) 92 | 93 | # a calculation of the compile time, in seconds 94 | compile_time = (d.now() - start_compile).total_seconds() 95 | 96 | print('[+] Model successfully compiled in {:.3f} seconds'.format(compile_time)) 97 | 98 | 99 | # a method for loading in the data given path (and many optional arguments) 100 | # note, this data path should point to a folder with the data 101 | def load_data(self, data_path, data_type='img'): 102 | # check that the data exists and has two subfolders, 'test' and 'train' 103 | if os.path.isdir(data_path) and os.path.isdir(data_path + '/train/') and os.path.isdir(data_path + '/test/'): 104 | if data_type == 'img': 105 | # for images, we can increase the size/variety of our data set using generators, courtesy of Keras 106 | train_gen = ImageDataGenerator( 107 | rotation_range=rot_range, 108 | width_shift_range=w_shift, 109 | height_shift_range=h_shift, 110 | rescale=1./255, 111 | shear_range=shear, 112 | fill_mode='nearest', 113 | horizontal_flip=h_flip, 114 | vertical_flip=v_flip 115 | ) 116 | 117 | test_gen = ImageDataGenerator(rescale=1./255) 118 | 119 | # uses the generator to make data, uses parameters from config 120 | self.train_data = train_gen.flow_from_directory( 121 | data_path + '/train/', 122 | target_size=(target_dim1, target_dim2), 123 | batch_size=batch_size, 124 | class_mode=classification_type 125 | ) 126 | 127 | self.test_data = test_gen.flow_from_directory( 128 | data_path + '/test/', 129 | target_size=(target_dim1, target_dim2), 130 | batch_size=batch_size, 131 | class_mode=classification_type 132 | ) 133 | 134 | print('[+] Data successfully loaded') 135 | self.loaded = True 136 | 137 | else: 138 | print('[-] Datatype {} not yet supported.'.format(str(data_type))) 139 | 140 | else: 141 | print('[-] The path provided is not a folder containing "train" and "test" or does not exist, please try again.') 142 | 143 | # method for building the actual model 144 | def build_model(self, r_params=False): 145 | self.pretrained = False 146 | 147 | # define the model type, from Keras 148 | model = Sequential() 149 | 150 | if self.model_type == 'mul_class': 151 | model.add(Conv2D( 152 | 32, 153 | filter_dim, 154 | input_shape=(target_dim1, target_dim2, 3), 155 | activation=activations[0] 156 | )) 157 | 158 | ## mid layers of our model, where a lot of our tinkering will occur 159 | model.add(MaxPooling2D(pool_size=(2, 2))) 160 | model.add(Dropout(0.5)) 161 | 162 | model.add(Flatten()) 163 | 164 | model.add(Dense(16, activation='relu')) 165 | model.add(Dropout(0.5)) 166 | 167 | # final layer of the model, outputs actual classification 168 | model.add(Dense(num_classes, activation=activations[-1])) 169 | 170 | # a "begin" marker to time how long it takes (in real time) to compile 171 | start_compile = d.now() 172 | 173 | # actually compile the model 174 | model.compile( 175 | loss=l_type, 176 | optimizer=opt, 177 | metrics=met 178 | ) 179 | 180 | # a calculation of the compile time, in seconds 181 | compile_time = (d.now() - start_compile).total_seconds() 182 | 183 | self.model = model 184 | 185 | print('[+] Model successfully compiled in {:.3f} seconds'.format(compile_time)) 186 | 187 | 188 | # a method for actually training the model on the supplied data 189 | # TODO: maybe allow override of config stuff? 190 | @model_required 191 | @data_required 192 | def train(self): 193 | # we cannot train a pretrained model on our dataset 194 | if not self.pretrained: 195 | try: 196 | # again, a variable for timing 197 | fit_start = d.now() 198 | 199 | # fit the model to the data 200 | self.model.fit_generator( 201 | self.train_data, 202 | steps_per_epoch=step_num // batch_size, 203 | epochs=epoch_num, 204 | validation_data=self.test_data, 205 | validation_steps=step_num // batch_size, 206 | workers=worker_num, 207 | use_multiprocessing=multiprocessing 208 | ) 209 | 210 | # another time calculation, but for fitting, in seconds 211 | fit_time = (d.now() - fit_start).total_seconds() / 60 212 | 213 | print('[+] Training completed in {:.3f} minutes'.format(fit_time)) 214 | 215 | except KeyboardInterrupt: 216 | print('\nUser aborted...') 217 | 218 | # method for predicting on an input data piece 219 | @model_required 220 | def predict(self, raw_input): 221 | try: 222 | if not self.pretrained: 223 | 224 | pred = self.model.predict(raw_input) 225 | 226 | # get the list of labels 227 | labels = listdir('data/train') 228 | 229 | # remove hidden files from the labels list 230 | while labels[0].startswith('.'): 231 | labels.pop(0) 232 | 233 | # initialize a dictionary for storing label to probability mappings 234 | pmap = dict() 235 | for i in range(len(labels)): 236 | pmap[labels[i]] = list(pred[0])[i] 237 | 238 | self.prediction = pmap 239 | print('[+] Prediction successfully completed') 240 | 241 | else: 242 | # preprocess the image for the pretrained net 243 | image = preprocess_input(raw_input) 244 | 245 | # make predictions 246 | prediction = self.model.predict(image) 247 | preds = imagenet_utils.decode_predictions(prediction) 248 | 249 | # create a dictionary to store the top five predictions with their probabilities 250 | p = dict() 251 | for (i, (imagenetID, label, prob)) in enumerate(preds[0]): 252 | p[label] = prob 253 | 254 | self.prediction = p 255 | print('[+] Prediction successfully completed') 256 | 257 | except ValueError as err: 258 | print('[-] Prediction failed, please check the input shape:') 259 | print(err) 260 | 261 | 262 | 263 | # method for evaluating the model 264 | @model_required 265 | @data_required 266 | def evaluate(self, steps=eval_steps): 267 | try: 268 | print('[+] Beginning evaluation of model...') 269 | 270 | # begin timer for checking evaluation speed 271 | eval_start = d.now() 272 | 273 | if not self.pretrained: 274 | evaln = self.model.evaluate_generator(self.test_data, steps) 275 | 276 | self.evaluation = dict() 277 | for i in range(len(self.model.metrics_names)): 278 | self.evaluation[self.model.metrics_names[i]] = evaln[i] 279 | 280 | # another time calculation, but for evaluating, in seconds 281 | eval_time = (d.now() - eval_start).total_seconds() / 60 282 | 283 | print('[+] Evaluation successful! ({} seconds)'.format(eval_time)) 284 | except RuntimeError: 285 | print('[-] Evaluation of the model failed - check that it was compiled') 286 | 287 | # method for saving the model to file 288 | @model_required 289 | def save(self, save_path=None): 290 | if not save_path: 291 | save_path = 'Model_{}'.format(d.now().strftime('%A%B%-d%Y%H%M%S')) 292 | try: 293 | self.model.save(save_path + '.h5') 294 | print('[+] Model successfully saved to "{}"'.format(os.path.abspath(save_path))) 295 | 296 | except Exception as err: 297 | print('[-] Model could not be saved:') 298 | print(err) 299 | --------------------------------------------------------------------------------