├── .gitignore ├── LICENSE ├── README.md ├── build.bat ├── downloadtfjs.sh ├── examples ├── AddImageTrainingData.jpg ├── AddNumbersTrainingData.jpg ├── AddTextTrainingData.jpg ├── AppInProject.jpg ├── CirclesAndRectangles_Project │ └── CirclesAndRectangles.aia ├── ClassifyImage.jpg ├── ClassifyNumbers.jpg ├── ClassifyText.jpg ├── GetModelStatus.jpg ├── GotClassification.jpg ├── GotError.jpg ├── GotStatus.jpg ├── MakeMeHappy_Project │ ├── MakeMeHappy.aia │ └── worksheet-makemehappy-appinventor.pdf ├── PyProject.jpg ├── README.md ├── SmartClassroom_Project │ └── SmartClassroom.aia ├── TrainNewModel.jpg ├── download.png ├── ml4k-image.PNG ├── ml4k-numbers.PNG ├── ml4k-text.PNG └── set_key.png ├── makefile ├── release ├── build_aix.py └── com.kylecorry.ml4k │ ├── aiwebres │ └── ml4k.png │ ├── assets │ ├── api.txt │ ├── group1-shard1of1 │ ├── group10-shard1of1 │ ├── group11-shard1of1 │ ├── group12-shard1of1 │ ├── group13-shard1of1 │ ├── group14-shard1of1 │ ├── group15-shard1of1 │ ├── group16-shard1of1 │ ├── group17-shard1of1 │ ├── group18-shard1of1 │ ├── group19-shard1of1 │ ├── group2-shard1of1 │ ├── group20-shard1of1 │ ├── group21-shard1of1 │ ├── group22-shard1of1 │ ├── group23-shard1of1 │ ├── group24-shard1of1 │ ├── group25-shard1of1 │ ├── group26-shard1of1 │ ├── group27-shard1of1 │ ├── group28-shard1of1 │ ├── group29-shard1of1 │ ├── group3-shard1of1 │ ├── group30-shard1of1 │ ├── group31-shard1of1 │ ├── group32-shard1of1 │ ├── group33-shard1of1 │ ├── group34-shard1of1 │ ├── group35-shard1of1 │ ├── group36-shard1of1 │ ├── group37-shard1of1 │ ├── group38-shard1of1 │ ├── group39-shard1of1 │ ├── group4-shard1of1 │ ├── group40-shard1of1 │ ├── group41-shard1of1 │ ├── group42-shard1of1 │ ├── group43-shard1of1 │ ├── group44-shard1of1 │ ├── group45-shard1of1 │ ├── group46-shard1of1 │ ├── group47-shard1of1 │ ├── group48-shard1of1 │ ├── group49-shard1of1 │ ├── group5-shard1of1 │ ├── group50-shard1of1 │ ├── group51-shard1of1 │ ├── group52-shard1of1 │ ├── group53-shard1of1 │ ├── group54-shard1of1 │ ├── group55-shard1of1 │ ├── group6-shard1of1 │ ├── group7-shard1of1 │ ├── group8-shard1of1 │ ├── group9-shard1of1 │ ├── ml4k.html │ ├── ml4k.js │ ├── model.json │ ├── promisepool.js │ └── tf.min.js │ ├── classes.jar │ ├── component.json │ ├── components.json │ ├── extension.properties │ └── files │ ├── AndroidRuntime.jar │ ├── component_build_info.json │ └── component_build_infos.json └── src └── com └── kylecorry └── ml4k ├── APIErrorResponse.java ├── APIResponse.java ├── Base64Encoder.java ├── Classification.java ├── HttpImpl.java ├── HttpStrategy.java ├── ImageEncoder.java ├── ImageResizer.java ├── JSONUtils.java ├── ML4K.java ├── ML4KComponent.java ├── ML4KException.java ├── ML4KWebPage.java ├── ModelStatus.java ├── aiwebres └── ml4k.png └── assets ├── api.txt ├── ml4k.html └── ml4k.js /.gitignore: -------------------------------------------------------------------------------- 1 | *.aix 2 | _build/ 3 | dist/ 4 | build/ 5 | src/com/kylecorry/ml4k/assets/model.json 6 | src/com/kylecorry/ml4k/assets/tf.min.js 7 | src/com/kylecorry/ml4k/assets/group*-shard1of1 8 | src/com/kylecorry/ml4k/assets/promisepool.js 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Kyle Corry 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ML4K AppInventor Extension 2 | Use machine learning in AppInventor, with easy training using text, images, or numbers through the [Machine Learning for Kids](https://machinelearningforkids.co.uk/) website. 3 | 4 | [![Download (.aix)](examples/download.png)](https://github.com/kylecorry31/ML4K-AI-Extension/releases/download/v1.2.1/ML4K.aix) 5 | 6 | ## Example 7 | 8 | ### Text classification 9 | ![ML4K Text Example](examples/ml4k-text.PNG) 10 | 11 | ### Image classification 12 | ![ML4K Image Example](examples/ml4k-image.PNG) 13 | 14 | Note: The ClassifyImage block takes the path to an image, which you can get from the Selected property of an ImagePicker. 15 | 16 | ### Numbers classification 17 | ![ML4K Number Example](examples/ml4k-numbers.PNG) 18 | 19 | ### Live examples 20 | See example .aia projects in the examples directory of this repo (Created by [Joe Mazzone](https://github.com/MrMazzone)). Look at the code blocks for where to add your API key (API key is not included), they are set in the click method - though you can set it anywhere you choose as long as it is before the classification occurs. 21 | 22 | ## Installation 23 | Download the latest extension file (.aix) from the [releases](https://github.com/kylecorry31/ML4K-AI-Extension/releases) page and follow section "2. How to use extensions components" of [this website](http://ai2.appinventor.mit.edu/reference/other/extensions.html) to add the extension to your App Inventor project. 24 | 25 | ## Guide 26 | 27 | --- 28 | **If you received this extension from the ML4K website, your API key will be set for you and you don't need the block to set it - skip to step 1 and 2.** 29 | 30 | --- 31 | 32 | 1. After installing the extension, you need to get an API key, which can be obtained from [Machine Learning for Kids](https://machinelearningforkids.co.uk/). This API Key is not the IBM Watson API keys used to create your Machine Learning for Kids account. This key is created by Machine Learning for Kids and is specific to your project. It can be found on the App Inventor project page in the unique URL for your project. 33 | 34 | ![](examples/AppInProject.jpg) 35 | 36 |        Or, the key can be found on the Python project page. 37 | 38 | ![](examples/PyProject.jpg) 39 | 40 |       **__Do not use the keys displayed in the images above. Use the keys from your project pages.__** 41 | 42 | 2. Copy and paste the API Key into the ML4K component’s “Key” property on the Designer screen or use the "set Key" block on the Blocks screen. Note: API Key must be set before you can use any of the ML4K extension blocks for classification. If you choose to set the key using the “set Key” block, be sure to set the key in the Screen.Initialize event or any time before you use a classification method (purple block). 43 | 44 | ![Set Key](examples/set_key.png) 45 | 46 | 3. Classify the text, image, or numbers: 47 | a. If classifying text, use the "ClassifyText" block with the text to classify. 48 | ![](examples/ClassifyText.jpg) 49 | b. If classifying images, use the "ClassifyImage" block with the image path to classify. 50 | ![](examples/ClassifyImage.jpg) 51 | c. If classifying numbers, use the "ClassifyNumbers" block with a list of numbers to classify. 52 | ![](examples/ClassifyNumbers.jpg) 53 | 54 | 4. Use the "GotClassification" event block to retrieve the classification once it is completed. 55 | ![](examples/GotClassification.jpg) 56 | 57 | 5. Use the "GotError" event block to retrieve any errors which occur during classification. 58 | ![](examples/GotError.jpg) 59 | 60 | 6. Add data to your machine learning project with code: 61 | a. To add data to a text project use the "AddTextTrainingData" block, identifying the text data to add and label to add it to. 62 | ![](examples/AddTextTrainingData.jpg) 63 | b. To add data to an image project use the "AddImageTrainingData" block, identifying the image file to add and label to add it to. 64 | ![](examples/AddImageTrainingData.jpg) 65 | c. To add data to a numbers project use the "AddNumbersTrainingData" block, identifying the list of numbers and label to add it to. 66 | ![](examples/AddNumbersTrainingData.jpg) 67 | 68 | 7. Use the "GetModelStatus" block to see if your model is ready to use or still in the process of training. 69 | ![](examples/GetModelStatus.jpg) 70 | 71 | 8. Use the "GotStatus" event block to retrieve the status of the model. 72 | ![](examples/GotStatus.jpg) 73 | * statusCode 2 - "Ready" - The model is trained and ready to use. 74 | * statusCode 1 - "Training in progress" - The model is still training and cannot be used. 75 | * statusCode 0 - Something went wrong (or there isn't a model) - the 'message' variable will contain information on the issue. 76 | 77 | 9. Use the "TrainNewModel" block to train your machine learning model with App Inventor. 78 | ![](examples/TrainNewModel.jpg) 79 | * Note: Using this block is optional for text or numbers projects, as models can be trained on the Machine Learning for Kids server from the "Learn & Test" page. However for Image projects, using this block is essential and must be used to create a model before an ML model can be used (because models are created and used on the mobile device) 80 | 81 | ### Handling Errors 82 | Upon an error, the "GotError" event block will be called with the error that occurred. Please use this event block fro debugging. 83 | ![](examples/GotError.jpg) 84 | 85 | ## Building with preset API key 86 | To build the extension, open a terminal and navigate to the release folder. Run the build_aix.py script, passing in the API key. 87 | 88 | ```Shell 89 | cd release 90 | python build_aix.py 91 | ``` 92 | 93 | This will generate a ML4K.aix file which contains a preset API key. 94 | 95 | To do this without the Python script, the file com.kylecorry.ml4k/assets/api.txt needs to be modified to have the API key in it. Then the whole folder (com.kylecorry.ml4k directory needs to be present in the top level of the zip file) needs to be zipped and renamed to have the .aix extension instead of .zip. 96 | 97 | ## Building from source 98 | 99 | ### Unix (Linux / Mac) 100 | To build the extension from sources, you can use the makefile. 101 | 102 | #### Requirements 103 | - make 104 | - ant 105 | - git 106 | 107 | #### Building 108 | ```shell 109 | make 110 | ``` 111 | 112 | This will create a directory called \_build/dist which will contain the .aix file (without an API key). 113 | 114 | ### Windows 115 | To build the extension from sources, you can use the build.bat. 116 | 117 | #### Requirements 118 | - ant 119 | - git 120 | 121 | #### Building 122 | ```shell 123 | .\build.bat 124 | ``` 125 | 126 | This will create a directory called \_build/dist which will contain the .aix file (without an API key). 127 | 128 | ## FAQ 129 | **The extension crashes while using an emulator**: See [this issue](https://github.com/kylecorry31/ML4K-AI-Extension/issues/35) 130 | 131 | ## License 132 | This project is licensed under the [MIT License](LICENSE). 133 | 134 | ## Featured projects 135 | Here are some community contributed projects which demonstrate awesome uses of the ML4K extension. All credits belong to the creators of these projects: 136 | - [eFAP: Emotion Detector for Angry Players](https://youtu.be/LZ760BT8QmM) 137 | 138 | ## Credits 139 | * [The Machine Learning for Kids](https://github.com/IBM/taxinomitis) for providing the training tool and API endpoint. 140 | * [Kyle Corry (Me)](https://github.com/kylecorry31) for project planning, programming the extension, and testing. 141 | * [Joe Mazzone](https://github.com/MrMazzone) for project planning, testing, and examples. 142 | 143 | ## Contribute 144 | Please feel free to contribute to this extension, or if you find an issue be sure to report it under issues. 145 | -------------------------------------------------------------------------------- /build.bat: -------------------------------------------------------------------------------- 1 | if not exist "_build" mkdir _build 2 | cd _build 3 | if exist "dist" cd dist && del *.aix && cd .. 4 | if not exist "appinventor-sources" git clone https://github.com/mit-cml/appinventor-sources 5 | robocopy ..\src\com\ appinventor-sources\appinventor\components\src\com\ /e 6 | cd appinventor-sources/appinventor 7 | cmd.exe /c ant extensions 8 | cd ..\.. 9 | if not exist "dist" mkdir dist 10 | xcopy appinventor-sources\appinventor\components\build\extensions\com.kylecorry.ml4k.aix dist\ /Y 11 | rename dist\com.kylecorry.ml4k.aix ML4K.aix 12 | -------------------------------------------------------------------------------- /downloadtfjs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -f "../src/com/kylecorry/ml4k/assets/model.json" ]; then 4 | echo "Downloading tfjs files..." 5 | curl https://storage.googleapis.com/tfjs-models/tfjs/mobilenet_v1_0.25_224/model.json -o ../src/com/kylecorry/ml4k/assets/model.json 6 | for i in $(seq 1 55) 7 | do 8 | curl https://storage.googleapis.com/tfjs-models/tfjs/mobilenet_v1_0.25_224/group$i-shard1of1 -o ../src/com/kylecorry/ml4k/assets/group$i-shard1of1 9 | done 10 | npm install @tensorflow/tfjs@3.14.0 11 | cp node_modules/@tensorflow/tfjs/dist/tf.min.js ../src/com/kylecorry/ml4k/assets/. 12 | npm install @ricokahler/pool 13 | cp node_modules/@ricokahler/pool/dist/index.js ../src/com/kylecorry/ml4k/assets/promisepool.js 14 | fi -------------------------------------------------------------------------------- /examples/AddImageTrainingData.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/AddImageTrainingData.jpg -------------------------------------------------------------------------------- /examples/AddNumbersTrainingData.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/AddNumbersTrainingData.jpg -------------------------------------------------------------------------------- /examples/AddTextTrainingData.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/AddTextTrainingData.jpg -------------------------------------------------------------------------------- /examples/AppInProject.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/AppInProject.jpg -------------------------------------------------------------------------------- /examples/CirclesAndRectangles_Project/CirclesAndRectangles.aia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/CirclesAndRectangles_Project/CirclesAndRectangles.aia -------------------------------------------------------------------------------- /examples/ClassifyImage.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/ClassifyImage.jpg -------------------------------------------------------------------------------- /examples/ClassifyNumbers.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/ClassifyNumbers.jpg -------------------------------------------------------------------------------- /examples/ClassifyText.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/ClassifyText.jpg -------------------------------------------------------------------------------- /examples/GetModelStatus.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/GetModelStatus.jpg -------------------------------------------------------------------------------- /examples/GotClassification.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/GotClassification.jpg -------------------------------------------------------------------------------- /examples/GotError.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/GotError.jpg -------------------------------------------------------------------------------- /examples/GotStatus.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/GotStatus.jpg -------------------------------------------------------------------------------- /examples/MakeMeHappy_Project/MakeMeHappy.aia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/MakeMeHappy_Project/MakeMeHappy.aia -------------------------------------------------------------------------------- /examples/MakeMeHappy_Project/worksheet-makemehappy-appinventor.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/MakeMeHappy_Project/worksheet-makemehappy-appinventor.pdf -------------------------------------------------------------------------------- /examples/PyProject.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/PyProject.jpg -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Using Project Examples 2 | 3 | After importing the example project .aia file into App Inventor, you __MUST__ add your project's Machine Learning for Kids API key to the example in order to use the example. That means you must create your own project and add your own examples on the Machine Learning for Kids site (https://machinelearningforkids.co.uk). This API Key is not the IBM Watson API keys used to create your Machine Learning for Kids account. This key is created by Machine Learning for Kids and is specific to your project. 4 | 5 | The API key can be found on the App Inventor project page in the unique URL for your project. 6 | 7 | ![](AppInProject.jpg) 8 | 9 | Or, the key can be found on the Python project page. 10 | 11 | ![](PyProject.jpg) 12 | 13 | **__Do not use the keys displayed in the images above. Use the keys from your project pages.__** 14 | 15 | Copy and paste the API Key into the ML4K component’s “Key” property on the Designer screen or use the "set Key" block on the Blocks screen. Note: API Key must be set before you can use any of the ML4K extension blocks for classification. If you choose to set the key using the “set Key” block, be sure to set the key in the Screen.Initialize event or any time before you use a classification method (purple block). 16 | 17 | ![Set Key](set_key.png) 18 | -------------------------------------------------------------------------------- /examples/SmartClassroom_Project/SmartClassroom.aia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/SmartClassroom_Project/SmartClassroom.aia -------------------------------------------------------------------------------- /examples/TrainNewModel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/TrainNewModel.jpg -------------------------------------------------------------------------------- /examples/download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/download.png -------------------------------------------------------------------------------- /examples/ml4k-image.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/ml4k-image.PNG -------------------------------------------------------------------------------- /examples/ml4k-numbers.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/ml4k-numbers.PNG -------------------------------------------------------------------------------- /examples/ml4k-text.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/ml4k-text.PNG -------------------------------------------------------------------------------- /examples/set_key.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/examples/set_key.png -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | all: 2 | mkdir -p _build 3 | cd _build && \ 4 | if [ ! -d "appinventor-sources" ]; then \ 5 | git clone https://github.com/mit-cml/appinventor-sources; \ 6 | fi && \ 7 | cp -r ../src/com/kylecorry appinventor-sources/appinventor/components/src/com/ && \ 8 | ../downloadtfjs.sh && \ 9 | cd appinventor-sources/appinventor && \ 10 | ant extensions && \ 11 | cd ../.. && \ 12 | mkdir -p dist && \ 13 | cp appinventor-sources/appinventor/components/build/extensions/com.kylecorry.ml4k.aix dist/ML4K.aix 14 | 15 | clean: 16 | rm -rf _build 17 | -------------------------------------------------------------------------------- /release/build_aix.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import shutil 3 | import os 4 | 5 | if __name__ == '__main__': 6 | if len(sys.argv) != 2: 7 | print("Usage: build_aix.py ") 8 | exit(1) 9 | api_key = sys.argv[1] 10 | 11 | with open('com.kylecorry.ml4k/assets/api.txt', 'w+') as f: 12 | f.write(api_key) 13 | 14 | if os.path.exists('ML4K.aix'): 15 | os.remove('ML4K.aix') 16 | shutil.make_archive('com.kylecorry.ml4k', 'zip', base_dir = 'com.kylecorry.ml4k') 17 | os.rename('com.kylecorry.ml4k.zip', 'ML4K.aix') 18 | -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/aiwebres/ml4k.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/aiwebres/ml4k.png -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/api.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/api.txt -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group1-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group1-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group10-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group10-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group11-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group11-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group12-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group12-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group13-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group13-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group14-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group14-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group15-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group15-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group16-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group16-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group17-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group17-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group18-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group18-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group19-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group19-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group2-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group2-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group20-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group20-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group21-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group21-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group22-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group22-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group23-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group23-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group24-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group24-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group25-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group25-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group26-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group26-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group27-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group27-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group28-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group28-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group29-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group29-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group3-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group3-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group30-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group30-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group31-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group31-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group32-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group32-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group33-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group33-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group34-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group34-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group35-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group35-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group36-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group36-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group37-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group37-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group38-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group38-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group39-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group39-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group4-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group4-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group40-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group40-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group41-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group41-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group42-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group42-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group43-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group43-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group44-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group44-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group45-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group45-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group46-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group46-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group47-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group47-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group48-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group48-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group49-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group49-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group5-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group5-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group50-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group50-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group51-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group51-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group52-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group52-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group53-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group53-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group54-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group54-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group55-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group55-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group6-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group6-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group7-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group7-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group8-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group8-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/group9-shard1of1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/assets/group9-shard1of1 -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/ml4k.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/ml4k.js: -------------------------------------------------------------------------------- 1 | // --------------------------------------------------------------------- 2 | 3 | var _ML4K_IMG_WIDTH = 224; 4 | var _ML4K_IMG_HEIGHT = 224; 5 | 6 | var _ML4K_MODEL_TYPE = 'images'; 7 | 8 | // --------------------------------------------------------------------- 9 | 10 | var _ml4kBaseModel; 11 | var _ml4kTransferModel; 12 | 13 | var _ml4kModelClasses; 14 | 15 | var _ml4kUsingRestoredModel = false; 16 | 17 | // --------------------------------------------------------------------- 18 | 19 | function _ml4kFetchJson(urlString) { 20 | return fetch(urlString) 21 | .then(function (resp) { 22 | if (resp.ok) { 23 | return resp.json(); 24 | } 25 | else { 26 | console.log(resp); 27 | throw new Error('Error in communicating with Machine Learning for Kids server'); 28 | } 29 | }); 30 | } 31 | 32 | function _ml4kFetchData(urlString) { 33 | return fetch(urlString) 34 | .then(function (resp) { 35 | if (resp.ok) { 36 | return resp.arrayBuffer(); 37 | } 38 | else { 39 | console.log(resp); 40 | throw new Error('Error in downloading image data'); 41 | } 42 | }); 43 | } 44 | 45 | // --------------------------------------------------------------------- 46 | 47 | function _ml4kGetModelDbLocation(modeltype, scratchkey) { 48 | return 'indexeddb://ml4k-models-' + 49 | modeltype + '-' + 50 | scratchkey.replace(/-/g, ''); 51 | } 52 | 53 | function _ml4kLoadModel(modeltype, scratchkey) { 54 | console.log('loading model'); 55 | var savelocation = _ml4kGetModelDbLocation(modeltype, scratchkey); 56 | return tf.loadLayersModel(savelocation) 57 | .then(function (resp) { 58 | console.log('loaded model'); 59 | 60 | if (window.localStorage) { 61 | var modelMetadataStr = window.localStorage.getItem('ml4k-modelinfo-' + modeltype + '-' + scratchkey); 62 | _ml4kModelClasses = JSON.parse(modelMetadataStr).classes; 63 | } 64 | else { 65 | console.log('Unable to access local storage'); 66 | } 67 | 68 | ML4KJavaInterface.setModelStatus('Available', 100); 69 | _ml4kUsingRestoredModel = true; 70 | return resp; 71 | }) 72 | .catch(function (err) { 73 | console.log('failed to load model'); 74 | console.log(err); 75 | }); 76 | } 77 | 78 | function _ml4kSaveModel(modeltype, scratchkey, modelClasses, transferModel) { 79 | console.log('saving model'); 80 | var savelocation = _ml4kGetModelDbLocation(modeltype, scratchkey); 81 | return transferModel.save(savelocation) 82 | .then(function () { 83 | if (window.localStorage) { 84 | window.localStorage.setItem('ml4k-modelinfo-' + modeltype + '-' + scratchkey, 85 | JSON.stringify({ classes : modelClasses })); 86 | } 87 | else { 88 | console.log('unable to save model metadata'); 89 | } 90 | }) 91 | .catch(function (err) { 92 | console.log('failed to save model'); 93 | console.log(err); 94 | }); 95 | } 96 | 97 | // --------------------------------------------------------------------- 98 | 99 | 100 | function _ml4kPrepareMobilenet() { 101 | return tf.loadLayersModel('https://machinelearningforkids.co.uk/appinventor-assets/model.json') 102 | .then(function (pretrainedModel) { 103 | var activationLayer = pretrainedModel.getLayer('conv_pw_13_relu'); 104 | return tf.model({ 105 | inputs : pretrainedModel.inputs, 106 | outputs: activationLayer.output 107 | }); 108 | }) 109 | .catch(function (err) { 110 | console.log('failed to prepare mobilenet'); 111 | console.log(err); 112 | throw err; 113 | }); 114 | } 115 | 116 | function _ml4kPrepareTransferLearningModel(modifiedMobilenet, numClasses) { 117 | var model = tf.sequential({ 118 | layers : [ 119 | tf.layers.flatten({ 120 | inputShape : modifiedMobilenet.outputs[0].shape.slice(1) 121 | }), 122 | tf.layers.dense({ 123 | units : 100, 124 | activation : 'relu', 125 | kernelInitializer : 'varianceScaling', 126 | useBias : true 127 | }), 128 | tf.layers.dense({ 129 | units : numClasses, 130 | activation : 'softmax', 131 | kernelInitializer : 'varianceScaling', 132 | useBias : false 133 | }) 134 | ] 135 | }); 136 | 137 | model.compile({ 138 | optimizer : tf.train.adam(0.0001), 139 | loss : 'categoricalCrossentropy' 140 | }); 141 | 142 | return model; 143 | } 144 | 145 | function _ml4kTrainModel(baseModel, transferModel, scratchkey, trainingdata) { 146 | ML4KJavaInterface.setModelStatus('Training', 0); 147 | 148 | _ml4kModelClasses = trainingdata.labels; 149 | 150 | var xs; 151 | var ys; 152 | 153 | for (var i=0; i < trainingdata.imagedata.length; i++) { 154 | var trainingdataitem = trainingdata.imagedata[i]; 155 | 156 | var labelIdx = _ml4kModelClasses.indexOf(trainingdataitem.label); 157 | 158 | var xval = baseModel.predict(trainingdataitem.tensor); 159 | var yval = tf.tidy(function () { 160 | return tf.oneHot(tf.tensor1d([ labelIdx ]).toInt(), _ml4kModelClasses.length); 161 | }); 162 | 163 | if (i === 0) { 164 | xs = xval; 165 | ys = yval; 166 | } 167 | else { 168 | var oldxs = xs; 169 | var oldys = ys; 170 | xs = oldxs.concat(xval, 0); 171 | ys = oldys.concat(yval, 0); 172 | 173 | oldxs.dispose(); 174 | oldys.dispose(); 175 | } 176 | } 177 | 178 | var epochs = 10; 179 | if (trainingdata.imagedata.length > 55) { 180 | epochs = 15; 181 | } 182 | 183 | transferModel.fit(xs, ys, { 184 | batchSize : 10, 185 | epochs : epochs, 186 | callbacks : { 187 | onEpochEnd : function (epoch, logs) { 188 | console.log('epoch ' + epoch + ' loss ' + logs.loss); 189 | if (epochs === 15) { 190 | ML4KJavaInterface.setModelStatus('Training', (epoch + 1) * 7); 191 | } 192 | else { 193 | ML4KJavaInterface.setModelStatus('Training', (epoch + 1) * 10); 194 | } 195 | }, 196 | onTrainEnd : function () { 197 | return _ml4kSaveModel(_ML4K_MODEL_TYPE, scratchkey, _ml4kModelClasses, transferModel) 198 | .then(function () { 199 | console.log('training complete'); 200 | ML4KJavaInterface.setModelStatus('Available', 100); 201 | 202 | _ml4kUsingRestoredModel = false; 203 | }); 204 | } 205 | } 206 | }); 207 | } 208 | 209 | // --------------------------------------------------------------------- 210 | 211 | function _ml4kCreateTensorForImageData(imageid, imagedata, imagelabel) { 212 | return new Promise(function (resolve, reject) { 213 | var imageDataBlob = URL.createObjectURL(new Blob([ imagedata ])); 214 | 215 | var hiddenImg = document.createElement('img'); 216 | hiddenImg.id = '_ml4k_' + imageid; 217 | hiddenImg.width = _ML4K_IMG_WIDTH; 218 | hiddenImg.height = _ML4K_IMG_HEIGHT; 219 | hiddenImg.onerror = function (err) { 220 | console.log('Failed to load image', imageid); 221 | console.log(err); 222 | return reject(err); 223 | }; 224 | hiddenImg.onload = function () { 225 | var tensorData = tf.tidy(function () { 226 | return tf.browser.fromPixels(hiddenImg) 227 | .expandDims(0) 228 | .toFloat() 229 | .div(127) 230 | .sub(1); 231 | }); 232 | 233 | resolve({ id : imageid, label : imagelabel, tensor : tensorData }); 234 | 235 | URL.revokeObjectURL(imageDataBlob); 236 | }; 237 | 238 | hiddenImg.src = imageDataBlob; 239 | }); 240 | } 241 | 242 | function _ml4kCreateTensorForTestImage(imagedata) { 243 | return new Promise(function (resolve, reject) { 244 | var hiddenImg = document.createElement('img'); 245 | hiddenImg.id = '_ml4k_test_' + Date.now(); 246 | hiddenImg.width = _ML4K_IMG_WIDTH; 247 | hiddenImg.height = _ML4K_IMG_HEIGHT; 248 | hiddenImg.onerror = function (err) { 249 | console.log('Failed to load image'); 250 | console.log(imagedata); 251 | return reject(err); 252 | }; 253 | hiddenImg.onload = function () { 254 | var tensorData = tf.tidy(function () { 255 | return tf.browser.fromPixels(hiddenImg) 256 | .expandDims(0) 257 | .toFloat() 258 | .div(127) 259 | .sub(1); 260 | }); 261 | 262 | resolve(tensorData); 263 | }; 264 | hiddenImg.src = 'data:image/jpg;base64,' + imagedata; 265 | }); 266 | } 267 | 268 | function _ml4kGetImageData(imageid, imageurl, imagelabel) { 269 | console.log('getImageData : ' + imageid + ' : ' + imageurl); 270 | return _ml4kFetchData(imageurl) 271 | .then(function (imagedata) { 272 | return _ml4kCreateTensorForImageData(imageid, imagedata, imagelabel); 273 | }) 274 | .catch(function (err) { 275 | console.log(err); 276 | throw new Error('Unable to process training image at ' + imageurl); 277 | }); 278 | } 279 | 280 | function _ml4kGetTrainingImages(scratchkey) { 281 | console.log('getting training images'); 282 | var labels = new Set(); 283 | return _ml4kFetchJson('https://machinelearningforkids.co.uk/api/scratch/' + scratchkey + '/train?proxy=true') 284 | .then(function (imagesList) { 285 | return pool({ 286 | collection: imagesList, 287 | maxConcurrency: 10, 288 | task: function (imageInfo) { 289 | labels.add(imageInfo.label); 290 | return _ml4kGetImageData(imageInfo.id, imageInfo.imageurl, imageInfo.label); 291 | } 292 | }); 293 | }) 294 | .then(function (trainingimages) { 295 | return { imagedata : trainingimages, labels : Array.from(labels) }; 296 | }); 297 | } 298 | 299 | // --------------------------------------------------------------------- 300 | 301 | function _ml4kSortByConfidence(a, b) { 302 | if (a.confidence < b.confidence) { 303 | return 1; 304 | } 305 | else if (a.confidence > b.confidence) { 306 | return -1; 307 | } 308 | else { 309 | return 0; 310 | } 311 | } 312 | 313 | 314 | function _ml4kTestImageDataTensor(testTensor) { 315 | var baseModelOutput = _ml4kBaseModel.predict(testTensor); 316 | var transferModelOutput = _ml4kTransferModel.predict(baseModelOutput); 317 | 318 | return transferModelOutput.data() 319 | .then(function (output) { 320 | console.log(JSON.stringify(_ml4kModelClasses)); 321 | console.log(JSON.stringify(output)); 322 | 323 | if (output.length !== _ml4kModelClasses.length) { 324 | console.log('unexpected output from model', output); 325 | return []; 326 | } 327 | 328 | var scores = _ml4kModelClasses.map(function (label, idx) { 329 | return { 330 | class_name : label, 331 | confidence : 100 * output[idx] 332 | }; 333 | }).sort(_ml4kSortByConfidence); 334 | return scores; 335 | }) 336 | .catch(function (err) { 337 | console.log('failed to test image'); 338 | console.log(err); 339 | return []; 340 | }); 341 | } 342 | 343 | function ml4kClassifyImage(imagedata) { 344 | return _ml4kCreateTensorForTestImage(imagedata) 345 | .then(function (tensor) { 346 | return _ml4kTestImageDataTensor(tensor); 347 | }) 348 | .then(function (modelOutput) { 349 | if (modelOutput.length > 0) { 350 | ML4KJavaInterface.classifyResponse(modelOutput[0].class_name, modelOutput[0].confidence); 351 | } 352 | else { 353 | ML4KJavaInterface.classifyResponse("Unknown", 0.1); 354 | } 355 | }) 356 | .catch(function (err) { 357 | console.log('failed to classify image'); 358 | console.log(err); 359 | ML4KJavaInterface.classifyResponse("Unknown", 0.1); 360 | }); 361 | } 362 | 363 | // --------------------------------------------------------------------- 364 | 365 | function ml4kTrainNewModel(scratchkey) { 366 | return _ml4kGetTrainingImages(scratchkey) 367 | .then(function (trainingdata) { 368 | if (_ml4kUsingRestoredModel || !_ml4kTransferModel) { 369 | _ml4kTransferModel = _ml4kPrepareTransferLearningModel(_ml4kBaseModel, trainingdata.labels.length); 370 | } 371 | return _ml4kTrainModel(_ml4kBaseModel, _ml4kTransferModel, scratchkey, trainingdata); 372 | }) 373 | .catch(function (err) { 374 | console.log('model training failure'); 375 | console.log(err); 376 | ML4KJavaInterface.setModelStatus('Failed', 0); 377 | ML4KJavaInterface.returnErrorMessage(err.message); 378 | }); 379 | } 380 | 381 | 382 | 383 | 384 | function ml4kOnStart(scratchkey) { 385 | if (tf && tf.enableProdMode) { 386 | tf.enableProdMode(); 387 | } 388 | console.log(tf.version); 389 | 390 | return _ml4kPrepareMobilenet() 391 | .then(function (preparedBaseModel) { 392 | _ml4kBaseModel = preparedBaseModel; 393 | 394 | if (scratchkey) { 395 | return _ml4kLoadModel(_ML4K_MODEL_TYPE, scratchkey); 396 | } 397 | }) 398 | .then(function (loadedmodel) { 399 | if (loadedmodel) { 400 | _ml4kTransferModel = loadedmodel; 401 | } 402 | 403 | ML4KJavaInterface.setReady(true); 404 | }); 405 | } 406 | -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/model.json: -------------------------------------------------------------------------------- 1 | {"modelTopology": {"keras_version": "2.1.4", "model_config": {"class_name": "Model", "config": {"layers": [{"class_name": "InputLayer", "inbound_nodes": [], "config": {"dtype": "float32", "batch_input_shape": [null, 224, 224, 3], "name": "input_1", "sparse": false}, "name": "input_1"}, {"class_name": "Conv2D", "inbound_nodes": [[["input_1", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [2, 2], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 8, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv1"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv1", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv1_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv1_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv1_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv1_relu"}, "name": "conv1_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv1_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_1", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [1, 1], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_1"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_1", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_1_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_1_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_1_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_1_relu"}, "name": "conv_dw_1_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_1_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_1", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 16, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_1"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_1", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_1_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_1_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_1_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_1_relu"}, "name": "conv_pw_1_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_1_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_2", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [2, 2], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_2"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_2", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_2_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_2_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_2_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_2_relu"}, "name": "conv_dw_2_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_2_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_2", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 32, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_2"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_2", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_2_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_2_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_2_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_2_relu"}, "name": "conv_pw_2_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_2_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_3", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [1, 1], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_3"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_3", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_3_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_3_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_3_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_3_relu"}, "name": "conv_dw_3_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_3_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_3", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 32, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_3"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_3", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_3_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_3_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_3_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_3_relu"}, "name": "conv_pw_3_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_3_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_4", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [2, 2], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_4"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_4", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_4_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_4_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_4_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_4_relu"}, "name": "conv_dw_4_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_4_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_4", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 64, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_4"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_4", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_4_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_4_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_4_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_4_relu"}, "name": "conv_pw_4_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_4_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_5", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [1, 1], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_5"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_5", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_5_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_5_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_5_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_5_relu"}, "name": "conv_dw_5_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_5_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_5", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 64, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_5"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_5", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_5_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_5_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_5_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_5_relu"}, "name": "conv_pw_5_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_5_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_6", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [2, 2], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_6"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_6", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_6_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_6_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_6_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_6_relu"}, "name": "conv_dw_6_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_6_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_6", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 128, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_6"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_6", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_6_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_6_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_6_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_6_relu"}, "name": "conv_pw_6_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_6_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_7", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [1, 1], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_7"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_7", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_7_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_7_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_7_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_7_relu"}, "name": "conv_dw_7_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_7_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_7", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 128, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_7"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_7", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_7_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_7_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_7_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_7_relu"}, "name": "conv_pw_7_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_7_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_8", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [1, 1], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_8"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_8", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_8_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_8_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_8_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_8_relu"}, "name": "conv_dw_8_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_8_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_8", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 128, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_8"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_8", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_8_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_8_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_8_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_8_relu"}, "name": "conv_pw_8_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_8_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_9", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [1, 1], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_9"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_9", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_9_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_9_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_9_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_9_relu"}, "name": "conv_dw_9_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_9_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_9", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 128, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_9"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_9", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_9_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_9_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_9_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_9_relu"}, "name": "conv_pw_9_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_9_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_10", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [1, 1], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_10"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_10", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_10_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_10_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_10_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_10_relu"}, "name": "conv_dw_10_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_10_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_10", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 128, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_10"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_10", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_10_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_10_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_10_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_10_relu"}, "name": "conv_pw_10_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_10_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_11", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [1, 1], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_11"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_11", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_11_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_11_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_11_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_11_relu"}, "name": "conv_dw_11_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_11_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_11", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 128, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_11"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_11", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_11_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_11_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_11_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_11_relu"}, "name": "conv_pw_11_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_11_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_12", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [2, 2], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_12"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_12", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_12_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_12_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_12_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_12_relu"}, "name": "conv_dw_12_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_12_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_12", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 256, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_12"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_12", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_12_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_12_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_12_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_12_relu"}, "name": "conv_pw_12_relu"}, {"class_name": "DepthwiseConv2D", "inbound_nodes": [[["conv_pw_12_relu", 0, 0, {}]]], "config": {"padding": "same", "depth_multiplier": 1, "name": "conv_dw_13", "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "depthwise_constraint": null, "strides": [1, 1], "dilation_rate": [1, 1], "depthwise_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "depthwise_regularizer": null, "use_bias": false, "activity_regularizer": null, "kernel_size": [3, 3]}, "name": "conv_dw_13"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_dw_13", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_dw_13_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_dw_13_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_dw_13_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_dw_13_relu"}, "name": "conv_dw_13_relu"}, {"class_name": "Conv2D", "inbound_nodes": [[["conv_dw_13_relu", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_pw_13", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 256, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": false, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_pw_13"}, {"class_name": "BatchNormalization", "inbound_nodes": [[["conv_pw_13", 0, 0, {}]]], "config": {"gamma_initializer": {"class_name": "Ones", "config": {}}, "moving_mean_initializer": {"class_name": "Zeros", "config": {}}, "name": "conv_pw_13_bn", "epsilon": 0.001, "trainable": true, "center": true, "moving_variance_initializer": {"class_name": "Ones", "config": {}}, "beta_initializer": {"class_name": "Zeros", "config": {}}, "scale": true, "gamma_regularizer": null, "gamma_constraint": null, "beta_constraint": null, "beta_regularizer": null, "momentum": 0.99, "axis": -1}, "name": "conv_pw_13_bn"}, {"class_name": "Activation", "inbound_nodes": [[["conv_pw_13_bn", 0, 0, {}]]], "config": {"activation": "relu6", "trainable": true, "name": "conv_pw_13_relu"}, "name": "conv_pw_13_relu"}, {"class_name": "GlobalAveragePooling2D", "inbound_nodes": [[["conv_pw_13_relu", 0, 0, {}]]], "config": {"trainable": true, "name": "global_average_pooling2d_1", "data_format": "channels_last"}, "name": "global_average_pooling2d_1"}, {"class_name": "Reshape", "inbound_nodes": [[["global_average_pooling2d_1", 0, 0, {}]]], "config": {"target_shape": [1, 1, 256], "trainable": true, "name": "reshape_1"}, "name": "reshape_1"}, {"class_name": "Dropout", "inbound_nodes": [[["reshape_1", 0, 0, {}]]], "config": {"rate": 0.001, "noise_shape": null, "trainable": true, "seed": null, "name": "dropout"}, "name": "dropout"}, {"class_name": "Conv2D", "inbound_nodes": [[["dropout", 0, 0, {}]]], "config": {"kernel_initializer": {"class_name": "VarianceScaling", "config": {"distribution": "uniform", "scale": 1.0, "seed": null, "mode": "fan_avg"}}, "name": "conv_preds", "kernel_constraint": null, "bias_regularizer": null, "bias_constraint": null, "activation": "linear", "trainable": true, "data_format": "channels_last", "padding": "same", "strides": [1, 1], "dilation_rate": [1, 1], "kernel_regularizer": null, "filters": 1000, "bias_initializer": {"class_name": "Zeros", "config": {}}, "use_bias": true, "activity_regularizer": null, "kernel_size": [1, 1]}, "name": "conv_preds"}, {"class_name": "Activation", "inbound_nodes": [[["conv_preds", 0, 0, {}]]], "config": {"activation": "softmax", "trainable": true, "name": "act_softmax"}, "name": "act_softmax"}, {"class_name": "Reshape", "inbound_nodes": [[["act_softmax", 0, 0, {}]]], "config": {"target_shape": [1000], "trainable": true, "name": "reshape_2"}, "name": "reshape_2"}], "input_layers": [["input_1", 0, 0]], "name": "mobilenet_0.25_224", "output_layers": [["reshape_2", 0, 0]]}}, "backend": "tensorflow"}, "weightsManifest": [{"paths": ["group1-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 3, 8], "name": "conv1/kernel"}]}, {"paths": ["group2-shard1of1"], "weights": [{"dtype": "float32", "shape": [8], "name": "conv1_bn/gamma"}, {"dtype": "float32", "shape": [8], "name": "conv1_bn/beta"}, {"dtype": "float32", "shape": [8], "name": "conv1_bn/moving_mean"}, {"dtype": "float32", "shape": [8], "name": "conv1_bn/moving_variance"}]}, {"paths": ["group3-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 8, 1], "name": "conv_dw_1/depthwise_kernel"}]}, {"paths": ["group4-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 128, 1], "name": "conv_dw_10/depthwise_kernel"}]}, {"paths": ["group5-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_dw_10_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_10_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_10_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_10_bn/moving_variance"}]}, {"paths": ["group6-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 128, 1], "name": "conv_dw_11/depthwise_kernel"}]}, {"paths": ["group7-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_dw_11_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_11_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_11_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_11_bn/moving_variance"}]}, {"paths": ["group8-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 128, 1], "name": "conv_dw_12/depthwise_kernel"}]}, {"paths": ["group9-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_dw_12_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_12_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_12_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_12_bn/moving_variance"}]}, {"paths": ["group10-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 256, 1], "name": "conv_dw_13/depthwise_kernel"}]}, {"paths": ["group11-shard1of1"], "weights": [{"dtype": "float32", "shape": [256], "name": "conv_dw_13_bn/gamma"}, {"dtype": "float32", "shape": [256], "name": "conv_dw_13_bn/beta"}, {"dtype": "float32", "shape": [256], "name": "conv_dw_13_bn/moving_mean"}, {"dtype": "float32", "shape": [256], "name": "conv_dw_13_bn/moving_variance"}]}, {"paths": ["group12-shard1of1"], "weights": [{"dtype": "float32", "shape": [8], "name": "conv_dw_1_bn/gamma"}, {"dtype": "float32", "shape": [8], "name": "conv_dw_1_bn/beta"}, {"dtype": "float32", "shape": [8], "name": "conv_dw_1_bn/moving_mean"}, {"dtype": "float32", "shape": [8], "name": "conv_dw_1_bn/moving_variance"}]}, {"paths": ["group13-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 16, 1], "name": "conv_dw_2/depthwise_kernel"}]}, {"paths": ["group14-shard1of1"], "weights": [{"dtype": "float32", "shape": [16], "name": "conv_dw_2_bn/gamma"}, {"dtype": "float32", "shape": [16], "name": "conv_dw_2_bn/beta"}, {"dtype": "float32", "shape": [16], "name": "conv_dw_2_bn/moving_mean"}, {"dtype": "float32", "shape": [16], "name": "conv_dw_2_bn/moving_variance"}]}, {"paths": ["group15-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 32, 1], "name": "conv_dw_3/depthwise_kernel"}]}, {"paths": ["group16-shard1of1"], "weights": [{"dtype": "float32", "shape": [32], "name": "conv_dw_3_bn/gamma"}, {"dtype": "float32", "shape": [32], "name": "conv_dw_3_bn/beta"}, {"dtype": "float32", "shape": [32], "name": "conv_dw_3_bn/moving_mean"}, {"dtype": "float32", "shape": [32], "name": "conv_dw_3_bn/moving_variance"}]}, {"paths": ["group17-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 32, 1], "name": "conv_dw_4/depthwise_kernel"}]}, {"paths": ["group18-shard1of1"], "weights": [{"dtype": "float32", "shape": [32], "name": "conv_dw_4_bn/gamma"}, {"dtype": "float32", "shape": [32], "name": "conv_dw_4_bn/beta"}, {"dtype": "float32", "shape": [32], "name": "conv_dw_4_bn/moving_mean"}, {"dtype": "float32", "shape": [32], "name": "conv_dw_4_bn/moving_variance"}]}, {"paths": ["group19-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 64, 1], "name": "conv_dw_5/depthwise_kernel"}]}, {"paths": ["group20-shard1of1"], "weights": [{"dtype": "float32", "shape": [64], "name": "conv_dw_5_bn/gamma"}, {"dtype": "float32", "shape": [64], "name": "conv_dw_5_bn/beta"}, {"dtype": "float32", "shape": [64], "name": "conv_dw_5_bn/moving_mean"}, {"dtype": "float32", "shape": [64], "name": "conv_dw_5_bn/moving_variance"}]}, {"paths": ["group21-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 64, 1], "name": "conv_dw_6/depthwise_kernel"}]}, {"paths": ["group22-shard1of1"], "weights": [{"dtype": "float32", "shape": [64], "name": "conv_dw_6_bn/gamma"}, {"dtype": "float32", "shape": [64], "name": "conv_dw_6_bn/beta"}, {"dtype": "float32", "shape": [64], "name": "conv_dw_6_bn/moving_mean"}, {"dtype": "float32", "shape": [64], "name": "conv_dw_6_bn/moving_variance"}]}, {"paths": ["group23-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 128, 1], "name": "conv_dw_7/depthwise_kernel"}]}, {"paths": ["group24-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_dw_7_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_7_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_7_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_7_bn/moving_variance"}]}, {"paths": ["group25-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 128, 1], "name": "conv_dw_8/depthwise_kernel"}]}, {"paths": ["group26-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_dw_8_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_8_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_8_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_8_bn/moving_variance"}]}, {"paths": ["group27-shard1of1"], "weights": [{"dtype": "float32", "shape": [3, 3, 128, 1], "name": "conv_dw_9/depthwise_kernel"}]}, {"paths": ["group28-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_dw_9_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_9_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_9_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_dw_9_bn/moving_variance"}]}, {"paths": ["group29-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 256, 1000], "name": "conv_preds/kernel"}, {"dtype": "float32", "shape": [1000], "name": "conv_preds/bias"}]}, {"paths": ["group30-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 8, 16], "name": "conv_pw_1/kernel"}]}, {"paths": ["group31-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 128, 128], "name": "conv_pw_10/kernel"}]}, {"paths": ["group32-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_pw_10_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_10_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_10_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_10_bn/moving_variance"}]}, {"paths": ["group33-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 128, 128], "name": "conv_pw_11/kernel"}]}, {"paths": ["group34-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_pw_11_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_11_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_11_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_11_bn/moving_variance"}]}, {"paths": ["group35-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 128, 256], "name": "conv_pw_12/kernel"}]}, {"paths": ["group36-shard1of1"], "weights": [{"dtype": "float32", "shape": [256], "name": "conv_pw_12_bn/gamma"}, {"dtype": "float32", "shape": [256], "name": "conv_pw_12_bn/beta"}, {"dtype": "float32", "shape": [256], "name": "conv_pw_12_bn/moving_mean"}, {"dtype": "float32", "shape": [256], "name": "conv_pw_12_bn/moving_variance"}]}, {"paths": ["group37-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 256, 256], "name": "conv_pw_13/kernel"}]}, {"paths": ["group38-shard1of1"], "weights": [{"dtype": "float32", "shape": [256], "name": "conv_pw_13_bn/gamma"}, {"dtype": "float32", "shape": [256], "name": "conv_pw_13_bn/beta"}, {"dtype": "float32", "shape": [256], "name": "conv_pw_13_bn/moving_mean"}, {"dtype": "float32", "shape": [256], "name": "conv_pw_13_bn/moving_variance"}]}, {"paths": ["group39-shard1of1"], "weights": [{"dtype": "float32", "shape": [16], "name": "conv_pw_1_bn/gamma"}, {"dtype": "float32", "shape": [16], "name": "conv_pw_1_bn/beta"}, {"dtype": "float32", "shape": [16], "name": "conv_pw_1_bn/moving_mean"}, {"dtype": "float32", "shape": [16], "name": "conv_pw_1_bn/moving_variance"}]}, {"paths": ["group40-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 16, 32], "name": "conv_pw_2/kernel"}]}, {"paths": ["group41-shard1of1"], "weights": [{"dtype": "float32", "shape": [32], "name": "conv_pw_2_bn/gamma"}, {"dtype": "float32", "shape": [32], "name": "conv_pw_2_bn/beta"}, {"dtype": "float32", "shape": [32], "name": "conv_pw_2_bn/moving_mean"}, {"dtype": "float32", "shape": [32], "name": "conv_pw_2_bn/moving_variance"}]}, {"paths": ["group42-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 32, 32], "name": "conv_pw_3/kernel"}]}, {"paths": ["group43-shard1of1"], "weights": [{"dtype": "float32", "shape": [32], "name": "conv_pw_3_bn/gamma"}, {"dtype": "float32", "shape": [32], "name": "conv_pw_3_bn/beta"}, {"dtype": "float32", "shape": [32], "name": "conv_pw_3_bn/moving_mean"}, {"dtype": "float32", "shape": [32], "name": "conv_pw_3_bn/moving_variance"}]}, {"paths": ["group44-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 32, 64], "name": "conv_pw_4/kernel"}]}, {"paths": ["group45-shard1of1"], "weights": [{"dtype": "float32", "shape": [64], "name": "conv_pw_4_bn/gamma"}, {"dtype": "float32", "shape": [64], "name": "conv_pw_4_bn/beta"}, {"dtype": "float32", "shape": [64], "name": "conv_pw_4_bn/moving_mean"}, {"dtype": "float32", "shape": [64], "name": "conv_pw_4_bn/moving_variance"}]}, {"paths": ["group46-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 64, 64], "name": "conv_pw_5/kernel"}]}, {"paths": ["group47-shard1of1"], "weights": [{"dtype": "float32", "shape": [64], "name": "conv_pw_5_bn/gamma"}, {"dtype": "float32", "shape": [64], "name": "conv_pw_5_bn/beta"}, {"dtype": "float32", "shape": [64], "name": "conv_pw_5_bn/moving_mean"}, {"dtype": "float32", "shape": [64], "name": "conv_pw_5_bn/moving_variance"}]}, {"paths": ["group48-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 64, 128], "name": "conv_pw_6/kernel"}]}, {"paths": ["group49-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_pw_6_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_6_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_6_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_6_bn/moving_variance"}]}, {"paths": ["group50-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 128, 128], "name": "conv_pw_7/kernel"}]}, {"paths": ["group51-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_pw_7_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_7_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_7_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_7_bn/moving_variance"}]}, {"paths": ["group52-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 128, 128], "name": "conv_pw_8/kernel"}]}, {"paths": ["group53-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_pw_8_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_8_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_8_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_8_bn/moving_variance"}]}, {"paths": ["group54-shard1of1"], "weights": [{"dtype": "float32", "shape": [1, 1, 128, 128], "name": "conv_pw_9/kernel"}]}, {"paths": ["group55-shard1of1"], "weights": [{"dtype": "float32", "shape": [128], "name": "conv_pw_9_bn/gamma"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_9_bn/beta"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_9_bn/moving_mean"}, {"dtype": "float32", "shape": [128], "name": "conv_pw_9_bn/moving_variance"}]}]} -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/assets/promisepool.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Like `Promise.all` but you can specify how many concurrent tasks you want at once. 3 | */ 4 | async function pool({ 5 | collection, 6 | task, 7 | maxConcurrency 8 | }) { 9 | if (!maxConcurrency) { 10 | return Promise.all(collection.map((item, i) => task(item, i))); 11 | } 12 | 13 | if (!collection.length) { 14 | return []; 15 | } 16 | 17 | const results = []; 18 | const mutableCollection = collection.slice().map((t, i) => [t, i]); 19 | let available = maxConcurrency; 20 | let done = false; 21 | let globalResolve; 22 | let globalReject; 23 | const finalPromise = new Promise((resolve, reject) => { 24 | globalResolve = resolve; 25 | globalReject = reject; 26 | }); 27 | const listeners = new Set(); 28 | 29 | function notify() { 30 | for (const listener of listeners) { 31 | listener(); 32 | } 33 | } 34 | 35 | function ready() { 36 | return new Promise(resolve => { 37 | const listener = () => { 38 | if (done) { 39 | listeners.delete(listener); 40 | resolve(); 41 | } else if (available > 0) { 42 | listeners.delete(listener); 43 | available -= 1; 44 | resolve(); 45 | } 46 | }; 47 | 48 | listeners.add(listener); 49 | notify(); 50 | }); 51 | } 52 | 53 | while (true) { 54 | const value = mutableCollection.shift(); 55 | if (!value) break; 56 | if (done) break; 57 | const [t, i] = value; 58 | await ready(); 59 | task(t, i).then(r => { 60 | results.push([r, i]); 61 | available += 1; 62 | 63 | if (results.length === collection.length) { 64 | done = true; 65 | globalResolve(); 66 | } 67 | }).catch(e => { 68 | done = true; 69 | globalReject(e); 70 | }).finally(notify); 71 | } 72 | 73 | await finalPromise; 74 | return results.slice().sort(([, a], [, b]) => a - b).map(([r]) => r); 75 | } 76 | 77 | module.exports = pool; 78 | //# sourceMappingURL=index.js.map 79 | -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/classes.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/classes.jar -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/component.json: -------------------------------------------------------------------------------- 1 | { 2 | "categoryString": "EXTENSION", 3 | "dateBuilt": "2021-09-19T07:05:24-0400", 4 | "nonVisible": "true", 5 | "iconName": "aiwebres/ml4k.png", 6 | "methods": [ 7 | { 8 | "deprecated": "false", 9 | "name": "AddImageTrainingData", 10 | "description": "Adds an image training data to the model", 11 | "params": [ 12 | { 13 | "name": "label", 14 | "type": "text" 15 | }, 16 | { 17 | "name": "path", 18 | "type": "text" 19 | } 20 | ] 21 | }, 22 | { 23 | "deprecated": "false", 24 | "name": "AddNumbersTrainingData", 25 | "description": "Adds numbers training data to the model", 26 | "params": [ 27 | { 28 | "name": "label", 29 | "type": "text" 30 | }, 31 | { 32 | "name": "numbers", 33 | "type": "list" 34 | } 35 | ] 36 | }, 37 | { 38 | "deprecated": "false", 39 | "name": "AddTextTrainingData", 40 | "description": "Adds a text training data to the model", 41 | "params": [ 42 | { 43 | "name": "label", 44 | "type": "text" 45 | }, 46 | { 47 | "name": "text", 48 | "type": "text" 49 | } 50 | ] 51 | }, 52 | { 53 | "deprecated": "false", 54 | "name": "ClassifyImage", 55 | "description": "Get the classification for the image.", 56 | "params": [{ 57 | "name": "path", 58 | "type": "text" 59 | }] 60 | }, 61 | { 62 | "deprecated": "false", 63 | "name": "ClassifyNumbers", 64 | "description": "Get the classification for the numbers.", 65 | "params": [{ 66 | "name": "numbers", 67 | "type": "list" 68 | }] 69 | }, 70 | { 71 | "deprecated": "false", 72 | "name": "ClassifyText", 73 | "description": "Get the classification for the text.", 74 | "params": [{ 75 | "name": "data", 76 | "type": "text" 77 | }] 78 | }, 79 | { 80 | "deprecated": "false", 81 | "name": "GetModelStatus", 82 | "description": "Gets the status of the model", 83 | "params": [] 84 | }, 85 | { 86 | "deprecated": "false", 87 | "name": "TrainNewModel", 88 | "description": "Train new machine learning model", 89 | "params": [] 90 | } 91 | ], 92 | "blockProperties": [{ 93 | "rw": "read-write", 94 | "deprecated": "false", 95 | "name": "Key", 96 | "description": "The API key for the ML4K app.", 97 | "type": "text" 98 | }], 99 | "helpUrl": "", 100 | "licenseName": "", 101 | "type": "com.kylecorry.ml4k.ML4KComponent", 102 | "androidMinSdk": 7, 103 | "version": "5", 104 | "external": "true", 105 | "showOnPalette": "true", 106 | "assets": [ 107 | "group22-shard1of1", 108 | "group18-shard1of1", 109 | "group48-shard1of1", 110 | "group52-shard1of1", 111 | "group14-shard1of1", 112 | "group33-shard1of1", 113 | "promisepool.js", 114 | "group41-shard1of1", 115 | "group44-shard1of1", 116 | "group4-shard1of1", 117 | "group25-shard1of1", 118 | "group10-shard1of1", 119 | "group7-shard1of1", 120 | "group36-shard1of1", 121 | "group17-shard1of1", 122 | "group53-shard1of1", 123 | "group1-shard1of1", 124 | "group40-shard1of1", 125 | "group8-shard1of1", 126 | "api.txt", 127 | "group15-shard1of1", 128 | "group28-shard1of1", 129 | "group47-shard1of1", 130 | "group34-shard1of1", 131 | "group21-shard1of1", 132 | "group30-shard1of1", 133 | "ml4k.html", 134 | "group37-shard1of1", 135 | "group43-shard1of1", 136 | "group24-shard1of1", 137 | "group11-shard1of1", 138 | "ml4k.js", 139 | "group9-shard1of1", 140 | "group54-shard1of1", 141 | "group16-shard1of1", 142 | "group20-shard1of1", 143 | "group35-shard1of1", 144 | "group46-shard1of1", 145 | "model.json", 146 | "group27-shard1of1", 147 | "group12-shard1of1", 148 | "group5-shard1of1", 149 | "group50-shard1of1", 150 | "group38-shard1of1", 151 | "group2-shard1of1", 152 | "group23-shard1of1", 153 | "group31-shard1of1", 154 | "group49-shard1of1", 155 | "group55-shard1of1", 156 | "group42-shard1of1", 157 | "group19-shard1of1", 158 | "group39-shard1of1", 159 | "tf.min.js", 160 | "group26-shard1of1", 161 | "group13-shard1of1", 162 | "group3-shard1of1", 163 | "group32-shard1of1", 164 | "group6-shard1of1", 165 | "group45-shard1of1", 166 | "group51-shard1of1", 167 | "group29-shard1of1" 168 | ], 169 | "name": "ML4KComponent", 170 | "helpString": "This provides an interface for the Machine Learning for Kids website.", 171 | "properties": [{ 172 | "defaultValue": "", 173 | "name": "Key", 174 | "editorArgs": [], 175 | "editorType": "string" 176 | }], 177 | "events": [ 178 | { 179 | "deprecated": "false", 180 | "name": "GotClassification", 181 | "description": "Event indicating that a classification has finished.", 182 | "params": [ 183 | { 184 | "name": "data", 185 | "type": "text" 186 | }, 187 | { 188 | "name": "classification", 189 | "type": "text" 190 | }, 191 | { 192 | "name": "confidence", 193 | "type": "number" 194 | } 195 | ] 196 | }, 197 | { 198 | "deprecated": "false", 199 | "name": "GotError", 200 | "description": "Event indicating that a classification got an error.", 201 | "params": [ 202 | { 203 | "name": "data", 204 | "type": "text" 205 | }, 206 | { 207 | "name": "error", 208 | "type": "text" 209 | } 210 | ] 211 | }, 212 | { 213 | "deprecated": "false", 214 | "name": "GotStatus", 215 | "description": "Event fired when the status check completes.", 216 | "params": [ 217 | { 218 | "name": "statusCode", 219 | "type": "number" 220 | }, 221 | { 222 | "name": "message", 223 | "type": "text" 224 | } 225 | ] 226 | } 227 | ] 228 | } -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/components.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "categoryString": "EXTENSION", 3 | "dateBuilt": "2021-09-19T07:05:24-0400", 4 | "nonVisible": "true", 5 | "iconName": "aiwebres/ml4k.png", 6 | "methods": [ 7 | { 8 | "deprecated": "false", 9 | "name": "AddImageTrainingData", 10 | "description": "Adds an image training data to the model", 11 | "params": [ 12 | { 13 | "name": "label", 14 | "type": "text" 15 | }, 16 | { 17 | "name": "path", 18 | "type": "text" 19 | } 20 | ] 21 | }, 22 | { 23 | "deprecated": "false", 24 | "name": "AddNumbersTrainingData", 25 | "description": "Adds numbers training data to the model", 26 | "params": [ 27 | { 28 | "name": "label", 29 | "type": "text" 30 | }, 31 | { 32 | "name": "numbers", 33 | "type": "list" 34 | } 35 | ] 36 | }, 37 | { 38 | "deprecated": "false", 39 | "name": "AddTextTrainingData", 40 | "description": "Adds a text training data to the model", 41 | "params": [ 42 | { 43 | "name": "label", 44 | "type": "text" 45 | }, 46 | { 47 | "name": "text", 48 | "type": "text" 49 | } 50 | ] 51 | }, 52 | { 53 | "deprecated": "false", 54 | "name": "ClassifyImage", 55 | "description": "Get the classification for the image.", 56 | "params": [{ 57 | "name": "path", 58 | "type": "text" 59 | }] 60 | }, 61 | { 62 | "deprecated": "false", 63 | "name": "ClassifyNumbers", 64 | "description": "Get the classification for the numbers.", 65 | "params": [{ 66 | "name": "numbers", 67 | "type": "list" 68 | }] 69 | }, 70 | { 71 | "deprecated": "false", 72 | "name": "ClassifyText", 73 | "description": "Get the classification for the text.", 74 | "params": [{ 75 | "name": "data", 76 | "type": "text" 77 | }] 78 | }, 79 | { 80 | "deprecated": "false", 81 | "name": "GetModelStatus", 82 | "description": "Gets the status of the model", 83 | "params": [] 84 | }, 85 | { 86 | "deprecated": "false", 87 | "name": "TrainNewModel", 88 | "description": "Train new machine learning model", 89 | "params": [] 90 | } 91 | ], 92 | "blockProperties": [{ 93 | "rw": "read-write", 94 | "deprecated": "false", 95 | "name": "Key", 96 | "description": "The API key for the ML4K app.", 97 | "type": "text" 98 | }], 99 | "helpUrl": "", 100 | "licenseName": "", 101 | "type": "com.kylecorry.ml4k.ML4KComponent", 102 | "androidMinSdk": 7, 103 | "version": "5", 104 | "external": "true", 105 | "showOnPalette": "true", 106 | "assets": [ 107 | "group22-shard1of1", 108 | "group18-shard1of1", 109 | "group48-shard1of1", 110 | "group52-shard1of1", 111 | "group14-shard1of1", 112 | "group33-shard1of1", 113 | "promisepool.js", 114 | "group41-shard1of1", 115 | "group44-shard1of1", 116 | "group4-shard1of1", 117 | "group25-shard1of1", 118 | "group10-shard1of1", 119 | "group7-shard1of1", 120 | "group36-shard1of1", 121 | "group17-shard1of1", 122 | "group53-shard1of1", 123 | "group1-shard1of1", 124 | "group40-shard1of1", 125 | "group8-shard1of1", 126 | "api.txt", 127 | "group15-shard1of1", 128 | "group28-shard1of1", 129 | "group47-shard1of1", 130 | "group34-shard1of1", 131 | "group21-shard1of1", 132 | "group30-shard1of1", 133 | "ml4k.html", 134 | "group37-shard1of1", 135 | "group43-shard1of1", 136 | "group24-shard1of1", 137 | "group11-shard1of1", 138 | "ml4k.js", 139 | "group9-shard1of1", 140 | "group54-shard1of1", 141 | "group16-shard1of1", 142 | "group20-shard1of1", 143 | "group35-shard1of1", 144 | "group46-shard1of1", 145 | "model.json", 146 | "group27-shard1of1", 147 | "group12-shard1of1", 148 | "group5-shard1of1", 149 | "group50-shard1of1", 150 | "group38-shard1of1", 151 | "group2-shard1of1", 152 | "group23-shard1of1", 153 | "group31-shard1of1", 154 | "group49-shard1of1", 155 | "group55-shard1of1", 156 | "group42-shard1of1", 157 | "group19-shard1of1", 158 | "group39-shard1of1", 159 | "tf.min.js", 160 | "group26-shard1of1", 161 | "group13-shard1of1", 162 | "group3-shard1of1", 163 | "group32-shard1of1", 164 | "group6-shard1of1", 165 | "group45-shard1of1", 166 | "group51-shard1of1", 167 | "group29-shard1of1" 168 | ], 169 | "name": "ML4KComponent", 170 | "helpString": "This provides an interface for the Machine Learning for Kids website.", 171 | "properties": [{ 172 | "defaultValue": "", 173 | "name": "Key", 174 | "editorArgs": [], 175 | "editorType": "string" 176 | }], 177 | "events": [ 178 | { 179 | "deprecated": "false", 180 | "name": "GotClassification", 181 | "description": "Event indicating that a classification has finished.", 182 | "params": [ 183 | { 184 | "name": "data", 185 | "type": "text" 186 | }, 187 | { 188 | "name": "classification", 189 | "type": "text" 190 | }, 191 | { 192 | "name": "confidence", 193 | "type": "number" 194 | } 195 | ] 196 | }, 197 | { 198 | "deprecated": "false", 199 | "name": "GotError", 200 | "description": "Event indicating that a classification got an error.", 201 | "params": [ 202 | { 203 | "name": "data", 204 | "type": "text" 205 | }, 206 | { 207 | "name": "error", 208 | "type": "text" 209 | } 210 | ] 211 | }, 212 | { 213 | "deprecated": "false", 214 | "name": "GotStatus", 215 | "description": "Event fired when the status check completes.", 216 | "params": [ 217 | { 218 | "name": "statusCode", 219 | "type": "number" 220 | }, 221 | { 222 | "name": "message", 223 | "type": "text" 224 | } 225 | ] 226 | } 227 | ] 228 | }] -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/extension.properties: -------------------------------------------------------------------------------- 1 | type=external 2 | -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/files/AndroidRuntime.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/release/com.kylecorry.ml4k/files/AndroidRuntime.jar -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/files/component_build_info.json: -------------------------------------------------------------------------------- 1 | {"contentProviders":[],"metadata":[],"broadcastReceivers":[],"broadcastReceiver":[],"libraries":[],"services":[],"type":"com.kylecorry.ml4k.ML4KComponent","androidMinSdk":["7"],"activityMetadata":[],"assets":["group22-shard1of1","group18-shard1of1","group48-shard1of1","group52-shard1of1","group14-shard1of1","group33-shard1of1","promisepool.js","group41-shard1of1","group44-shard1of1","group4-shard1of1","group25-shard1of1","group10-shard1of1","group7-shard1of1","group36-shard1of1","group17-shard1of1","group53-shard1of1","group1-shard1of1","group40-shard1of1","group8-shard1of1","api.txt","group15-shard1of1","group28-shard1of1","group47-shard1of1","group34-shard1of1","group21-shard1of1","group30-shard1of1","ml4k.html","group37-shard1of1","group43-shard1of1","group24-shard1of1","group11-shard1of1","ml4k.js","group9-shard1of1","group54-shard1of1","group16-shard1of1","group20-shard1of1","group35-shard1of1","group46-shard1of1","model.json","group27-shard1of1","group12-shard1of1","group5-shard1of1","group50-shard1of1","group38-shard1of1","group2-shard1of1","group23-shard1of1","group31-shard1of1","group49-shard1of1","group55-shard1of1","group42-shard1of1","group19-shard1of1","group39-shard1of1","tf.min.js","group26-shard1of1","group13-shard1of1","group3-shard1of1","group32-shard1of1","group6-shard1of1","group45-shard1of1","group51-shard1of1","group29-shard1of1"],"native":[],"permissions":["android.permission.INTERNET","android.permission.READ_EXTERNAL_STORAGE","android.permission.WRITE_EXTERNAL_STORAGE"],"activities":[]} -------------------------------------------------------------------------------- /release/com.kylecorry.ml4k/files/component_build_infos.json: -------------------------------------------------------------------------------- 1 | [{"contentProviders":[],"metadata":[],"broadcastReceivers":[],"broadcastReceiver":[],"libraries":[],"services":[],"type":"com.kylecorry.ml4k.ML4KComponent","androidMinSdk":["7"],"activityMetadata":[],"assets":["group22-shard1of1","group18-shard1of1","group48-shard1of1","group52-shard1of1","group14-shard1of1","group33-shard1of1","promisepool.js","group41-shard1of1","group44-shard1of1","group4-shard1of1","group25-shard1of1","group10-shard1of1","group7-shard1of1","group36-shard1of1","group17-shard1of1","group53-shard1of1","group1-shard1of1","group40-shard1of1","group8-shard1of1","api.txt","group15-shard1of1","group28-shard1of1","group47-shard1of1","group34-shard1of1","group21-shard1of1","group30-shard1of1","ml4k.html","group37-shard1of1","group43-shard1of1","group24-shard1of1","group11-shard1of1","ml4k.js","group9-shard1of1","group54-shard1of1","group16-shard1of1","group20-shard1of1","group35-shard1of1","group46-shard1of1","model.json","group27-shard1of1","group12-shard1of1","group5-shard1of1","group50-shard1of1","group38-shard1of1","group2-shard1of1","group23-shard1of1","group31-shard1of1","group49-shard1of1","group55-shard1of1","group42-shard1of1","group19-shard1of1","group39-shard1of1","tf.min.js","group26-shard1of1","group13-shard1of1","group3-shard1of1","group32-shard1of1","group6-shard1of1","group45-shard1of1","group51-shard1of1","group29-shard1of1"],"native":[],"permissions":["android.permission.INTERNET","android.permission.READ_EXTERNAL_STORAGE","android.permission.WRITE_EXTERNAL_STORAGE"],"activities":[]}] -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/APIErrorResponse.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | class APIErrorResponse { 4 | private String error; 5 | public APIErrorResponse(String error) { 6 | this.error = error; 7 | } 8 | public String getError() { 9 | return this.error; 10 | } 11 | 12 | public static APIErrorResponse fromJson(String json) { 13 | String error = JSONUtils.readStringProperty(json, "error"); 14 | 15 | if (error == null){ 16 | return null; 17 | } 18 | 19 | if (error.equals("Missing data")) { 20 | error = "Empty or invalid data sent to Machine Learning for Kids"; 21 | } 22 | else if (error.equals("Scratch key not found")) { 23 | error = "Machine Learning for Kids API Key not recognised"; 24 | } 25 | 26 | return new APIErrorResponse(error); 27 | } 28 | } -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/APIResponse.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.net.HttpURLConnection; 6 | import java.util.Scanner; 7 | 8 | public class APIResponse { 9 | private String responseMessage; 10 | private int responseCode; 11 | private String body; 12 | 13 | APIResponse(int responseCode, String responseMessage, String body) { 14 | this.responseCode = responseCode; 15 | this.responseMessage = responseMessage; 16 | this.body = body; 17 | } 18 | 19 | public static APIResponse fromConnection(HttpURLConnection conn) throws IOException { 20 | final int responseCode = conn.getResponseCode(); 21 | final String responseMessage = conn.getResponseMessage(); 22 | 23 | String body; 24 | 25 | if (responseCode == HttpURLConnection.HTTP_OK) { 26 | body = read(conn.getInputStream()); 27 | } else { 28 | body = read(conn.getErrorStream()); 29 | } 30 | 31 | return new APIResponse(responseCode, responseMessage, body); 32 | } 33 | 34 | public String getResponseMessage() { 35 | return responseMessage; 36 | } 37 | 38 | public int getResponseCode() { 39 | return responseCode; 40 | } 41 | 42 | public String getBody() { 43 | return body; 44 | } 45 | 46 | public boolean isOK() { 47 | return responseCode == HttpURLConnection.HTTP_OK; 48 | } 49 | 50 | public boolean isNotFound() { 51 | return responseCode == HttpURLConnection.HTTP_NOT_FOUND; 52 | } 53 | 54 | public boolean isError() { 55 | return !isOK(); 56 | } 57 | 58 | /** 59 | * Read an input stream to a String. 60 | * 61 | * @param is The input stream. 62 | * @return The data from the input stream as a String. 63 | */ 64 | private static String read(InputStream is) { 65 | if (is == null){ 66 | return ""; 67 | } 68 | Scanner scanner = new Scanner(is); 69 | 70 | StringBuilder sb = new StringBuilder(); 71 | 72 | while (scanner.hasNextLine()) { 73 | sb.append(scanner.nextLine()); 74 | } 75 | 76 | return sb.toString(); 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/Base64Encoder.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | class Base64Encoder { 4 | 5 | /** 6 | * Encode a string into base64 format. 7 | * @param s The string to encode. 8 | * @return The input string, encoded into a base64 string. 9 | */ 10 | public static String encode(String s){ 11 | return encode(s.getBytes()); 12 | } 13 | 14 | /** 15 | * Encode a byte array into base64 format. 16 | * @param bytes The byte array to encode. 17 | * @return The input byte array, encoded into a base64 string. 18 | */ 19 | public static String encode(byte[] bytes){ 20 | 21 | StringBuilder s = new StringBuilder(); 22 | 23 | for (int i = 0; i < bytes.length; i+=3) { 24 | byte b1 = bytes[i]; 25 | byte b2 = i+1 < bytes.length ? bytes[i+1] : 0; 26 | byte b3 = i+2 < bytes.length ? bytes[i+2] : 0; 27 | 28 | int current = packBytesIntoInt(byteToUnsignedInt(b1), byteToUnsignedInt(b2), byteToUnsignedInt(b3)); 29 | s.append(lookup(byteToUnsignedInt(read6Bits(0, current)))); 30 | s.append(lookup(byteToUnsignedInt(read6Bits(1, current)))); 31 | if (i + 1 < bytes.length) { 32 | s.append(lookup(byteToUnsignedInt(read6Bits(2, current)))); 33 | } 34 | if (i + 2 < bytes.length){ 35 | s.append(lookup(byteToUnsignedInt(read6Bits(3, current)))); 36 | } 37 | } 38 | 39 | int eqsNeeded = (bytes.length % 3); 40 | if (eqsNeeded == 2){ 41 | s.append('='); 42 | } else if (eqsNeeded == 1){ 43 | s.append("=="); 44 | } 45 | 46 | return s.toString(); 47 | } 48 | 49 | /** 50 | * Convert a byte to an unsigned int. 51 | * @param b The byte to convert. 52 | * @return The unsigned int version of the byte. 53 | */ 54 | static int byteToUnsignedInt(byte b){ 55 | return ((int) b) & 0xFF; 56 | } 57 | 58 | /** 59 | * Pack 3 bytes into an integer, from left to right. 60 | * Byte 1 Byte 2 Byte 3 Zeros 61 | * 11111111222222223333333300000000 62 | * @param byte1 The left most byte 63 | * @param byte2 The middle byte 64 | * @param byte3 The right most byte 65 | * @return An integer containing all three bytes and right padded with zeros. 66 | */ 67 | static int packBytesIntoInt(int byte1, int byte2, int byte3){ 68 | return ((byte1 << 24) + (byte2 << 16) + (byte3 << 8)); 69 | } 70 | 71 | /** 72 | * Read 6 bits from an integer. 73 | * @param index The index to read bits from [0, 1, 2, 3]. 74 | * @param values The integer to get bits from. 75 | * @return The 6 bits from the values int, left padded with 2 zeros. 76 | */ 77 | static byte read6Bits(int index, int values){ 78 | if (index >= 4 || index < 0){ 79 | return 0; 80 | } 81 | int i = index * 6; 82 | int mask = 0b111111 << (26 - i); 83 | return (byte) ((values & mask) >> (26 - i) & 0b00111111); 84 | } 85 | 86 | /** 87 | * Lookup a value in a base64 table. 88 | * @param val The value to lookup [0, 63] 89 | * @return The character associated with the value in base64. 90 | */ 91 | private static char lookup(int val){ 92 | if (val <= 25){ 93 | return (char) ('A' + val); 94 | } else if (val <= 51){ 95 | return (char) ('a' + val - 26); 96 | } else if (val <= 61){ 97 | return (char) ('0' + val - 52); 98 | } else if (val == 62){ 99 | return '+'; 100 | } else { 101 | return '/'; 102 | } 103 | } 104 | 105 | } -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/Classification.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | class Classification { 4 | private String classification; 5 | private double confidence; 6 | 7 | private Classification(String classification, double confidence) { 8 | this.classification = classification; 9 | this.confidence = confidence; 10 | } 11 | 12 | /** 13 | * Loads a classification from JSON 14 | * @param json the JSON 15 | * @return the classification 16 | * @throws ML4KException if the JSON is invalid 17 | */ 18 | public static Classification fromJson(String json) throws ML4KException { 19 | 20 | if (json == null){ 21 | throw new ML4KException("JSON is not valid: " + json); 22 | } 23 | 24 | String className = JSONUtils.readStringProperty(json, "class_name"); 25 | double confidence = JSONUtils.readRealProperty(json, "confidence"); 26 | 27 | return new Classification(className, confidence); 28 | } 29 | 30 | /** 31 | * @return the classification 32 | */ 33 | public String getClassification() { 34 | return classification; 35 | } 36 | 37 | /** 38 | * @return the confidence 39 | */ 40 | public double getConfidence() { 41 | return confidence; 42 | } 43 | 44 | 45 | @Override 46 | public String toString() { 47 | return "Classification{" + 48 | "classification='" + classification + '\'' + 49 | ", confidence=" + confidence + 50 | '}'; 51 | } 52 | } -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/HttpImpl.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | import java.io.DataOutputStream; 4 | import java.io.IOException; 5 | import java.net.HttpURLConnection; 6 | import java.net.URL; 7 | 8 | public class HttpImpl implements HttpStrategy { 9 | 10 | private static final String ML4K_USER_AGENT = "MIT App Inventor (ML4K extension)"; 11 | 12 | @Override 13 | public APIResponse getJSON(URL url) throws IOException { 14 | HttpURLConnection conn = (HttpURLConnection) url.openConnection(); 15 | conn.setRequestMethod("GET"); 16 | conn.setRequestProperty("Content-Type", "application/json"); 17 | conn.setRequestProperty("User-Agent", ML4K_USER_AGENT); 18 | 19 | // Get response 20 | APIResponse res = APIResponse.fromConnection(conn); 21 | conn.disconnect(); 22 | return res; 23 | } 24 | 25 | @Override 26 | public APIResponse postJSON(URL url, String data) throws IOException { 27 | HttpURLConnection conn = (HttpURLConnection) url.openConnection(); 28 | boolean hasBody = data != null && data.length() != 0; 29 | if (hasBody) { 30 | conn.setFixedLengthStreamingMode(data.length()); 31 | } 32 | conn.setRequestMethod("POST"); 33 | conn.setRequestProperty("Content-Type", "application/json"); 34 | conn.setRequestProperty("User-Agent", ML4K_USER_AGENT); 35 | 36 | // Write the post data 37 | if (hasBody) { 38 | conn.setDoOutput(true); 39 | DataOutputStream os = new DataOutputStream(conn.getOutputStream()); 40 | os.writeBytes(data); 41 | os.flush(); 42 | os.close(); 43 | } 44 | 45 | // Get response 46 | APIResponse res = APIResponse.fromConnection(conn); 47 | conn.disconnect(); 48 | return res; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/HttpStrategy.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | import java.io.IOException; 4 | import java.net.URL; 5 | 6 | public interface HttpStrategy { 7 | 8 | /** 9 | * Get JSON from the URL 10 | * @param url the URL 11 | * @return the JSON response 12 | */ 13 | APIResponse getJSON(URL url) throws IOException; 14 | 15 | /** 16 | * Post JSON to the URL 17 | * @param url the URL 18 | * @param data the POST data 19 | * @return the JSON response 20 | */ 21 | APIResponse postJSON(URL url, String data) throws IOException; 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/ImageEncoder.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.File; 5 | import java.io.FileInputStream; 6 | import java.io.FileNotFoundException; 7 | import java.io.IOException; 8 | import java.io.InputStream; 9 | 10 | class ImageEncoder { 11 | 12 | /** 13 | * Encode an image in base64 format 14 | * @param image The image 15 | * @return the encode image (base64) 16 | * @throws FileNotFoundException if the file does not exist 17 | * @throws IOException if there is an IO error 18 | */ 19 | public static String encode(File image) throws FileNotFoundException, IOException { 20 | byte[] byteArray = readAllBytes(new FileInputStream(image)); 21 | return Base64Encoder.encode(byteArray); 22 | } 23 | 24 | /** 25 | * Reads all bytes from a file. 26 | * @param is The input stream. 27 | * @return The file contents as bytes. 28 | * @throws IOException upon error reading the input file. 29 | */ 30 | private static byte[] readAllBytes(InputStream is) throws IOException { 31 | ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 32 | byte[] buff = new byte[4096]; 33 | int read; 34 | while ((read = is.read(buff)) != -1) { 35 | byteArrayOutputStream.write(buff, 0, read); 36 | } 37 | 38 | byte[] out = byteArrayOutputStream.toByteArray(); 39 | byteArrayOutputStream.close(); 40 | is.close(); 41 | 42 | return out; 43 | } 44 | 45 | } -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/ImageResizer.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.File; 5 | import java.io.FileOutputStream; 6 | import java.io.IOException; 7 | 8 | import android.graphics.Bitmap; 9 | import android.graphics.BitmapFactory; 10 | 11 | import com.google.appinventor.components.runtime.util.FileUtil; 12 | 13 | class ImageResizer { 14 | 15 | private ImageResizer(){} 16 | 17 | public static File resize(File image, int width, int height) throws ML4KException { 18 | try { 19 | Bitmap bitmap = BitmapFactory.decodeFile(image.getAbsolutePath()); 20 | Bitmap resizedBitmap = Bitmap.createScaledBitmap(bitmap, width, height, false); 21 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 22 | resizedBitmap.compress(Bitmap.CompressFormat.JPEG, 70, out); 23 | byte[] bytes = out.toByteArray(); 24 | out.close(); 25 | File file = File.createTempFile("AI_Media_", null); 26 | file.deleteOnExit(); 27 | FileUtil.writeFile(bytes, file.getAbsolutePath()); 28 | return file; 29 | } catch (IOException e){ 30 | throw new ML4KException("Unable to resize image"); 31 | } 32 | } 33 | 34 | } -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/JSONUtils.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | import java.util.regex.Matcher; 4 | import java.util.regex.Pattern; 5 | 6 | public class JSONUtils { 7 | 8 | private JSONUtils(){ 9 | } 10 | 11 | /** 12 | * Reads a string property of a one dimensional JSON object 13 | * @param json the JSON 14 | * @param propertyName the name of the property to read 15 | * @return the property value, or null if it does not exist 16 | */ 17 | public static String readStringProperty(String json, String propertyName){ 18 | String pattern = "\"" + propertyName + "\"\\s*:\\s*\"([^\\\"]*)\""; 19 | Pattern compiled = Pattern.compile(pattern); 20 | 21 | Matcher matches = compiled.matcher(json); 22 | if (matches.find()){ 23 | return matches.group(1); 24 | } 25 | 26 | return null; 27 | } 28 | 29 | /** 30 | * Reads an int property of a one dimensional JSON object 31 | * @param json the JSON 32 | * @param propertyName the name of the property to read 33 | * @return the property value, or -1 if it does not exist (not good indicator) 34 | */ 35 | public static int readIntProperty(String json, String propertyName){ 36 | String pattern = "\"" + propertyName + "\"\\s*:\\s*(\\d+)"; 37 | Pattern compiled = Pattern.compile(pattern); 38 | 39 | Matcher matches = compiled.matcher(json); 40 | if (matches.find()){ 41 | return Integer.parseInt(matches.group(1)); 42 | } 43 | 44 | return -1; 45 | } 46 | 47 | /** 48 | * Reads a real/double property of a one dimensional JSON object 49 | * @param json the JSON 50 | * @param propertyName the name of the property to read 51 | * @return the property value, or NaN if it does not exist 52 | */ 53 | public static double readRealProperty(String json, String propertyName){ 54 | String pattern = "\"" + propertyName + "\"\\s*:\\s*([\\d.]+)"; 55 | Pattern compiled = Pattern.compile(pattern); 56 | 57 | Matcher matches = compiled.matcher(json); 58 | if (matches.find()){ 59 | return Double.parseDouble(matches.group(1)); 60 | } 61 | 62 | return Double.NaN; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/ML4K.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | import java.io.File; 4 | import java.io.FileNotFoundException; 5 | import java.io.IOException; 6 | import java.io.UnsupportedEncodingException; 7 | import java.net.MalformedURLException; 8 | import java.net.URL; 9 | import java.net.URLEncoder; 10 | import java.util.List; 11 | import java.util.regex.Pattern; 12 | import android.text.TextUtils; 13 | 14 | /** 15 | * The ML4K API 16 | */ 17 | public class ML4K { 18 | 19 | private static final String BASE_URL = "https://machinelearningforkids.co.uk/api/scratch/%s"; 20 | private static final String CLASSIFY_ENDPOINT = "/classify"; 21 | private static final String MODELS_ENDPOINT = "/models"; 22 | private static final String TRAIN_ENDPOINT = "/train"; 23 | private static final String STATUS_ENDPOINT = "/status"; 24 | 25 | private HttpStrategy http; 26 | 27 | private final Pattern apiKeyPattern = Pattern.compile( 28 | "[0-9a-f]{8}-[0-9a-f]{4}-[1][0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}" + 29 | "[0-9a-f]{8}-[0-9a-f]{4}-[4][0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}" 30 | ); 31 | 32 | private String apiKey; 33 | 34 | /** 35 | * Default constructor 36 | * @param apiKey the API key 37 | * @throws ML4KException if the API key is invalid 38 | */ 39 | public ML4K(String apiKey) throws ML4KException { 40 | setAPIKey(apiKey); 41 | http = new HttpImpl(); 42 | } 43 | 44 | /** 45 | * @param apiKey the ML4K API key 46 | * @throws ML4KException if the API key is invalid 47 | */ 48 | public void setAPIKey(String apiKey) throws ML4KException { 49 | if (TextUtils.isEmpty(apiKey)) { 50 | throw new ML4KException("API key not set"); 51 | } 52 | if (!apiKeyPattern.matcher(apiKey).matches()) { 53 | throw new ML4KException("API key isn't a Machine Learning for Kids key"); 54 | } 55 | 56 | this.apiKey = apiKey; 57 | } 58 | 59 | 60 | /** 61 | * Classify an image 62 | * @param image the image 63 | * @return the classification of the image 64 | * @throws ML4KException when an error occurs 65 | */ 66 | public Classification classify(File image) throws ML4KException { 67 | try { 68 | String dataStr = "{\"data\": " + "\"" + ImageEncoder.encode(image) + "\"}"; 69 | URL url = new URL(getBaseURL() + CLASSIFY_ENDPOINT); 70 | APIResponse res = http.postJSON(url, dataStr); 71 | 72 | if (res.isOK()) { 73 | return Classification.fromJson(res.getBody()); 74 | } else { 75 | APIErrorResponse response = APIErrorResponse.fromJson(res.getBody()); 76 | throw new ML4KException(response == null ? "Bad response from server: " + res.getResponseCode() : response.getError()); 77 | } 78 | } catch (MalformedURLException e) { 79 | throw new ML4KException("Could not generate URL"); 80 | } catch (FileNotFoundException e) { 81 | throw new ML4KException("Could not load image file"); 82 | } catch (IOException e) { 83 | throw new ML4KException("No Internet connection."); // TODO: standardize error messages 84 | } finally { 85 | try { 86 | image.delete(); 87 | } 88 | catch (Exception exc) { 89 | // okay to ignore - temp files are cleaned up eventually by OS 90 | } 91 | } 92 | } 93 | 94 | /** 95 | * Classify some text 96 | * @param text the text 97 | * @return the classification of the text 98 | * @throws ML4KException when an error occurs 99 | */ 100 | public Classification classify(String text) throws ML4KException { 101 | try { 102 | String urlStr = getBaseURL() + CLASSIFY_ENDPOINT + "?data=" + URLEncoder.encode(text, "UTF-8"); 103 | URL url = new URL(urlStr); 104 | APIResponse res = http.getJSON(url); 105 | 106 | if (res.isOK()) { 107 | return Classification.fromJson(res.getBody()); 108 | } else { 109 | APIErrorResponse response = APIErrorResponse.fromJson(res.getBody()); 110 | throw new ML4KException(response == null ? "Bad response from server: " + res.getResponseCode() : response.getError()); 111 | } 112 | } catch (UnsupportedEncodingException e) { 113 | throw new ML4KException("Could not encode text"); 114 | } catch (MalformedURLException e) { 115 | throw new ML4KException("Could not generate URL"); 116 | } catch (IOException e) { 117 | throw new ML4KException("No Internet connection."); 118 | } 119 | } 120 | 121 | /** 122 | * Classify some numbers 123 | * @param numbers the numbers 124 | * @return the classification of the numbers 125 | * @throws ML4KException when an error occurs 126 | */ 127 | public Classification classify(List numbers) throws ML4KException { 128 | try { 129 | String urlStr = getBaseURL() + CLASSIFY_ENDPOINT + "?" + urlEncodeList("data", numbers); 130 | URL url = new URL(urlStr); 131 | APIResponse res = http.getJSON(url); 132 | 133 | if (res.isOK()) { 134 | return Classification.fromJson(res.getBody()); 135 | } else { 136 | APIErrorResponse response = APIErrorResponse.fromJson(res.getBody()); 137 | throw new ML4KException(response == null ? "Bad response from server: " + res.getResponseCode() : response.getError()); 138 | } 139 | 140 | } catch (UnsupportedEncodingException e) { 141 | throw new ML4KException("Could not encode numbers"); 142 | } catch (MalformedURLException e) { 143 | throw new ML4KException("Could not generate URL"); 144 | } catch (IOException e) { 145 | throw new ML4KException("No Internet connection."); 146 | } 147 | } 148 | 149 | /** 150 | * Adds text training data 151 | * @param label the label of the data 152 | * @param text the data 153 | * @throws ML4KException if there is an error 154 | */ 155 | public void addTrainingData(String label, String text) throws ML4KException { 156 | addTrainingDataHelper(label, "\"" + text + "\""); 157 | } 158 | 159 | /** 160 | * Adds image training data 161 | * @param label the label of the data 162 | * @param image the data 163 | * @throws ML4KException if there is an error 164 | */ 165 | public void addTrainingData(String label, File image) throws ML4KException { 166 | try { 167 | addTrainingDataHelper(label, "\"" + ImageEncoder.encode(image) + "\""); 168 | } catch (IOException e) { 169 | throw new ML4KException("Could not encode image"); 170 | } finally { 171 | try { 172 | image.delete(); 173 | } 174 | catch (Exception exc) { 175 | // okay to ignore - temp files are cleaned up eventually by OS 176 | } 177 | } 178 | } 179 | 180 | /** 181 | * Adds numbers training data 182 | * @param label the label of the data 183 | * @param numbers the data 184 | * @throws ML4KException if there is an error 185 | */ 186 | public void addTrainingData(String label, List numbers) throws ML4KException { 187 | addTrainingDataHelper(label, numbers.toString()); 188 | } 189 | 190 | /** 191 | * @return the model's status 192 | * @throws ML4KException when any error occurs or if the status is undefined 193 | */ 194 | public ModelStatus getModelStatus() throws ML4KException { 195 | try{ 196 | URL url = new URL(getBaseURL() + STATUS_ENDPOINT); 197 | APIResponse res = http.getJSON(url); 198 | 199 | if (res.isOK()) { 200 | return ModelStatus.fromJson(res.getBody()); 201 | } else if (res.isNotFound()) { 202 | throw new ML4KException("Scratch key not found"); 203 | } 204 | else { 205 | APIErrorResponse response = APIErrorResponse.fromJson(res.getBody()); 206 | throw new ML4KException(response == null ? "Bad response from server: " + res.getResponseCode() : response.getError()); 207 | } 208 | } catch (MalformedURLException e) { 209 | throw new ML4KException("Could not generate URL"); 210 | } catch (IOException e) { 211 | throw new ML4KException("Unable to connect to ML4K servers"); 212 | } 213 | } 214 | 215 | 216 | /** 217 | * Train the model 218 | * @throws ML4KException when an error occurs 219 | */ 220 | public void train() throws ML4KException { 221 | try { 222 | URL url = new URL(getBaseURL() + MODELS_ENDPOINT); 223 | APIResponse res = http.postJSON(url, "{}"); 224 | 225 | if (res.isOK()) { 226 | // Do nothing 227 | } else { 228 | APIErrorResponse response = APIErrorResponse.fromJson(res.getBody()); 229 | throw new ML4KException(response == null ? "Bad response from server: " + res.getResponseCode() : response.getError()); 230 | } 231 | 232 | } catch (MalformedURLException e) { 233 | throw new ML4KException("Could not generate URL"); 234 | } catch (IOException e) { 235 | throw new ML4KException("No Internet connection."); 236 | } 237 | } 238 | 239 | 240 | /** 241 | * @return the base URL of the ML4K API 242 | */ 243 | private String getBaseURL(){ 244 | return String.format(BASE_URL, apiKey); 245 | } 246 | 247 | /** 248 | * Encode a list for a URL get request. 249 | * @param paramName The name of the parameter. 250 | * @param list The list to encode. 251 | * @return The encoded list. 252 | */ 253 | private String urlEncodeList(String paramName, List list) { 254 | StringBuilder sb = new StringBuilder(); 255 | if (list == null || list.size() == 0){ 256 | return ""; 257 | } 258 | 259 | for (int i = 0; i < list.size(); i++) { 260 | sb.append(paramName); 261 | sb.append('='); 262 | sb.append(list.get(i)); 263 | if (i != list.size() - 1){ 264 | sb.append('&'); 265 | } 266 | } 267 | 268 | return sb.toString(); 269 | } 270 | 271 | private void addTrainingDataHelper(String label, String data) throws ML4KException { 272 | try { 273 | URL url = new URL(getBaseURL() + TRAIN_ENDPOINT); 274 | APIResponse res = http.postJSON(url, "{ \"data\": " + data + ", \"label\": \"" + label + "\" }"); 275 | 276 | if (res.isOK()) { 277 | // Do nothing 278 | } else { 279 | APIErrorResponse response = APIErrorResponse.fromJson(res.getBody()); 280 | throw new ML4KException(response == null ? "Bad response from server: " + res.getResponseCode() : response.getError()); 281 | } 282 | } catch (UnsupportedEncodingException e) { 283 | throw new ML4KException("Could not encode data"); 284 | } catch (MalformedURLException e) { 285 | throw new ML4KException("Could not generate URL"); 286 | } catch (IOException e) { 287 | throw new ML4KException("No Internet connection."); 288 | } 289 | } 290 | 291 | 292 | // For testing 293 | 294 | 295 | void setHttpStrategy(HttpStrategy http){ 296 | this.http = http; 297 | } 298 | 299 | } 300 | -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/ML4KComponent.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | import com.google.appinventor.components.common.ComponentCategory; 4 | import com.google.appinventor.components.common.PropertyTypeConstants; 5 | import com.google.appinventor.components.common.YaVersion; 6 | import com.google.appinventor.components.annotations.DesignerComponent; 7 | import com.google.appinventor.components.annotations.DesignerProperty; 8 | import com.google.appinventor.components.annotations.SimpleFunction; 9 | import com.google.appinventor.components.annotations.SimpleObject; 10 | import com.google.appinventor.components.annotations.SimpleEvent; 11 | import com.google.appinventor.components.annotations.SimpleProperty; 12 | import com.google.appinventor.components.annotations.UsesPermissions; 13 | import com.google.appinventor.components.annotations.UsesAssets; 14 | import com.google.appinventor.components.runtime.AndroidNonvisibleComponent; 15 | import com.google.appinventor.components.runtime.Component; 16 | import com.google.appinventor.components.runtime.ComponentContainer; 17 | import com.google.appinventor.components.runtime.EventDispatcher; 18 | import com.google.appinventor.components.runtime.util.YailList; 19 | import com.google.appinventor.components.runtime.util.AsynchUtil; 20 | import com.google.appinventor.components.runtime.util.MediaUtil; 21 | import com.google.appinventor.components.runtime.WebViewer; 22 | 23 | import android.app.Activity; 24 | import android.app.AlertDialog; 25 | import android.content.DialogInterface; 26 | import android.os.Build; 27 | import android.util.Log; 28 | import android.webkit.JavascriptInterface; 29 | import android.webkit.WebResourceResponse; 30 | import android.webkit.WebSettings; 31 | import android.webkit.WebView; 32 | import android.webkit.WebViewClient; 33 | 34 | import java.io.InputStream; 35 | import java.io.IOException; 36 | import java.util.ArrayList; 37 | import java.util.HashMap; 38 | import java.util.List; 39 | import java.util.Map; 40 | import java.util.Scanner; 41 | 42 | @DesignerComponent(version = YaVersion.LABEL_COMPONENT_VERSION, 43 | description = "This provides an interface for the Machine Learning for Kids website.", 44 | category = ComponentCategory.EXTENSION, 45 | nonVisible = true, 46 | iconName = "aiwebres/ml4k.png") 47 | @UsesAssets(fileNames = "api.txt, ml4k.html, ml4k.js, promisepool.js, tf.min.js, model.json, group1-shard1of1, group10-shard1of1, group11-shard1of1, group12-shard1of1, group13-shard1of1, group14-shard1of1, group15-shard1of1, group16-shard1of1, group17-shard1of1, group18-shard1of1, group19-shard1of1, group2-shard1of1, group20-shard1of1, group21-shard1of1, group22-shard1of1, group23-shard1of1, group24-shard1of1, group25-shard1of1, group26-shard1of1, group27-shard1of1, group28-shard1of1, group29-shard1of1, group3-shard1of1, group30-shard1of1, group31-shard1of1, group32-shard1of1, group33-shard1of1, group34-shard1of1, group35-shard1of1, group36-shard1of1, group37-shard1of1, group38-shard1of1, group39-shard1of1, group4-shard1of1, group40-shard1of1, group41-shard1of1, group42-shard1of1, group43-shard1of1, group44-shard1of1, group45-shard1of1, group46-shard1of1, group47-shard1of1, group48-shard1of1, group49-shard1of1, group5-shard1of1, group50-shard1of1, group51-shard1of1, group52-shard1of1, group53-shard1of1, group54-shard1of1, group55-shard1of1, group6-shard1of1, group7-shard1of1, group8-shard1of1, group9-shard1of1") 48 | @SimpleObject(external = true) 49 | @UsesPermissions(permissionNames = "android.permission.INTERNET, android.permission.WRITE_EXTERNAL_STORAGE, android.permission.READ_EXTERNAL_STORAGE") 50 | public final class ML4KComponent extends AndroidNonvisibleComponent { 51 | 52 | private static final String LOGPREFIX = "ML4KComponent"; 53 | 54 | private final Activity activity; 55 | 56 | private String key = getKeyFromFile(); 57 | 58 | private WebView browser = null; 59 | private ML4KWebPage webPageObj = null; 60 | 61 | private final String ML4KURL = "https://machinelearningforkids.co.uk/appinventor-assets/"; 62 | 63 | private final Map scratchKeyTypes; 64 | 65 | // -------------------------------------------------------------------- 66 | // MESSAGES DISPLAYED TO USERS IN DIALOG POP-UPS IN THE EVENT OF ERRORS 67 | // -------------------------------------------------------------------- 68 | private final static String EXPLAIN_API_KEY_NEEDED = "A Machine Learning for Kids 'API key' is needed to use the machine learning blocks. This is a secret code you can copy from the App Inventor page on the Machine Learning for Kids website."; 69 | private final static String EXPLAIN_API_KEY_EXPIRED = "The Machine Learning for Kids website did not recognise your project API key. The most likely reason is that your ML project has been deleted."; 70 | private final static String EXPLAIN_API_KEY_INVALID = "Your API key does not look like a machine learning key. It is a secret code you have to copy from the App Inventor page on the Machine Learning for Kids website."; 71 | private final static String EXPLAIN_API_KEY_TYPE_TEXT = "You created a machine learning project to recognise text, so you can only use the text ML blocks in this project."; 72 | private final static String EXPLAIN_API_KEY_TYPE_NUMBERS = "You created a machine learning project to recognise numbers, so you can only use the numbers ML blocks in this project."; 73 | private final static String EXPLAIN_API_KEY_TYPE_IMAGES = "You created a machine learning project to recognise images, so you can only use the images ML blocks in this project."; 74 | private final static String EXPLAIN_API_KEY_TYPE_SOUNDS = "You created a machine learning project to recognise sounds, so you cannot use that block"; 75 | private final static String EXPLAIN_ML_MODEL_NEEDED = "Please train a machine learning model before you try to do this. You must do this in your App Inventor project, using the 'TrainNewModel' block."; 76 | private final static String EXPLAIN_ML_MODEL_NOTREADY = "Your machine learning model is still training. Please wait for this to complete."; 77 | private final static String EXPLAIN_ML_MODEL_FAILED = "Sorry! Your machine learning model failed to train."; 78 | 79 | 80 | 81 | public ML4KComponent(ComponentContainer container) { 82 | super(container.$form()); 83 | activity = container.$context(); 84 | 85 | scratchKeyTypes = new HashMap(); 86 | 87 | if (!isApiKeyMissing()) { 88 | checkProjectType(new NextAction(NextStep.Init)); 89 | } 90 | } 91 | 92 | 93 | // -------------------------------------------------------------------- 94 | // DISPLAY USER FEEDBACK 95 | // -------------------------------------------------------------------- 96 | 97 | public void displayErrorMessage(final String errormessage) { 98 | activity.runOnUiThread(new Runnable() { 99 | @Override 100 | public void run() { 101 | AlertDialog.Builder builder = new AlertDialog.Builder(activity); 102 | builder 103 | .setTitle("Machine Learning for Kids") 104 | .setMessage(errormessage) 105 | .setPositiveButton("OK", new DialogInterface.OnClickListener() { 106 | public void onClick(DialogInterface dialog, int id) { 107 | dialog.dismiss(); 108 | } 109 | }) 110 | .create() 111 | .show(); 112 | } 113 | }); 114 | } 115 | 116 | 117 | 118 | @DesignerProperty(editorType = PropertyTypeConstants.PROPERTY_TYPE_STRING) 119 | @SimpleProperty(description = "The API key for the ML4K app.") 120 | public void Key(String key) { 121 | Log.d(LOGPREFIX, "Updating API key to " + key); 122 | this.key = key.trim(); 123 | 124 | if (!isApiKeyMissing()) { 125 | checkProjectType(new NextAction(NextStep.Init)); 126 | } 127 | } 128 | 129 | @SimpleProperty 130 | public String Key() { 131 | return key; 132 | } 133 | 134 | // -------------------------------------------------------------------- 135 | // ACTIONS REPRESENTED AS BLOCKS IN APP INVENTOR 136 | // -------------------------------------------------------------------- 137 | 138 | @SimpleFunction(description = "Get the classification for the image.") 139 | public void ClassifyImage(final String path) { 140 | checkProjectType(new ClassifyImageAction(path)); 141 | } 142 | 143 | @SimpleFunction(description = "Get the classification for the text.") 144 | public void ClassifyText(final String data) { 145 | checkProjectType(new ClassifyTextAction(data)); 146 | } 147 | 148 | @SimpleFunction(description = "Get the classification for the numbers.") 149 | public void ClassifyNumbers(final YailList numbers) { 150 | checkProjectType(new ClassifyNumbersAction(numbers)); 151 | } 152 | 153 | 154 | @SimpleFunction(description = "Train new machine learning model") 155 | public void TrainNewModel() { 156 | checkProjectType(new NextAction(NextStep.TrainModel)); 157 | } 158 | 159 | 160 | @SimpleFunction(description = "Adds an image training data to the model") 161 | public void AddImageTrainingData(final String label, final String path) { 162 | checkProjectType(new ImageTrainingAction(label, path)); 163 | } 164 | 165 | @SimpleFunction(description = "Adds a text training data to the model") 166 | public void AddTextTrainingData(final String label, final String text) { 167 | checkProjectType(new TextTrainingAction(label, text)); 168 | } 169 | 170 | @SimpleFunction(description = "Adds numbers training data to the model") 171 | public void AddNumbersTrainingData(final String label, final YailList numbers){ 172 | checkProjectType(new NumbersTrainingAction(label, numbers)); 173 | } 174 | 175 | 176 | @SimpleFunction(description = "Gets the status of the model") 177 | public void GetModelStatus() { 178 | checkProjectType(new NextAction(NextStep.CheckModelStatus)); 179 | } 180 | 181 | 182 | 183 | // -------------------------------------------------------------------- 184 | // SUBMIT DATA TO ML4K SERVER FOR CLASSIFICATION 185 | // 186 | // network access needs to happen on background threads 187 | // -------------------------------------------------------------------- 188 | 189 | private void classifyTestDataOnServer(final NextAction classifyAction) { 190 | runInBackground(new Runnable() { 191 | @Override 192 | public void run() { 193 | String data = null; 194 | try { 195 | ML4K ml4k = new ML4K(key); 196 | 197 | Classification classification = null; 198 | switch (classifyAction.step) { 199 | case ClassifyText: 200 | data = ((ClassifyTextAction)classifyAction).text; 201 | classification = ml4k.classify(data); 202 | break; 203 | case ClassifyNumbers: 204 | YailList numbers = ((ClassifyNumbersAction)classifyAction).numbers; 205 | data = numbers.toString(); 206 | classification = ml4k.classify(convertYailListToDouble(numbers)); 207 | break; 208 | default: 209 | Log.d(LOGPREFIX, "Unexpected classify action"); 210 | return; 211 | } 212 | 213 | GotClassification(data, classification.getClassification(), classification.getConfidence()); 214 | 215 | } catch (Exception e) { 216 | GotError(data, e.getMessage()); 217 | } 218 | } 219 | }); 220 | } 221 | 222 | 223 | 224 | // -------------------------------------------------------------------- 225 | // ADD TRAINING DATA TO ML4K SERVER 226 | // 227 | // network access needs to happen on background threads 228 | // -------------------------------------------------------------------- 229 | 230 | private void addTrainingDataToProject(final NextAction addTrainingAction) { 231 | runInBackground(new Runnable() { 232 | @Override 233 | public void run() { 234 | try { 235 | ML4K ml4k = new ML4K(key); 236 | 237 | switch (addTrainingAction.step) { 238 | case AddTextTraining: 239 | ml4k.addTrainingData( 240 | ((TextTrainingAction)addTrainingAction).label, 241 | ((TextTrainingAction)addTrainingAction).text); 242 | break; 243 | case AddImageTraining: 244 | java.io.File image = loadImageFile(((ImageTrainingAction)addTrainingAction).path); 245 | if (image == null) { 246 | throw new ML4KException("Could not load image"); 247 | } 248 | ml4k.addTrainingData( 249 | ((ImageTrainingAction)addTrainingAction).label, 250 | image); 251 | break; 252 | case AddNumbersTraining: 253 | ml4k.addTrainingData( 254 | ((NumbersTrainingAction)addTrainingAction).label, 255 | convertYailListToDouble(((NumbersTrainingAction)addTrainingAction).numbers)); 256 | break; 257 | default: 258 | Log.d(LOGPREFIX, "Unexpected training action"); 259 | return; 260 | } 261 | } catch (Exception e) { 262 | GotError("train", e.getMessage()); 263 | } 264 | } 265 | }); 266 | } 267 | 268 | 269 | 270 | // -------------------------------------------------------------------- 271 | // ML MODEL ACTIONS FOR MODELS HOSTED ON ML4K SERVER 272 | // -------------------------------------------------------------------- 273 | 274 | private void trainModelWithWatson() { 275 | runInBackground(new Runnable() { 276 | @Override 277 | public void run() { 278 | try { 279 | ML4K ml4k = new ML4K(key); 280 | ml4k.train(); 281 | } catch (Exception e) { 282 | GotError("train", e.getMessage()); 283 | } 284 | } 285 | }); 286 | } 287 | 288 | 289 | private void fetchModelStatusFromServer() { 290 | runInBackground(new Runnable() { 291 | @Override 292 | public void run() { 293 | try { 294 | ML4K ml4k = new ML4K(key); 295 | ModelStatus status = ml4k.getModelStatus(); 296 | GotStatus(status.getStatusCode(), status.getMessage()); 297 | } catch (Exception e) { 298 | GotError("status", e.getMessage()); 299 | } 300 | } 301 | }); 302 | } 303 | 304 | 305 | 306 | // -------------------------------------------------------------------- 307 | // EVENTS FIRED BY THE EXTENSION 308 | // -------------------------------------------------------------------- 309 | 310 | /** 311 | * Event fired when the status check completes. 312 | * 313 | * @param statusCode The status code of the model (2 = ready, 1 = training, 0 = error) 314 | * @param message The status message of the model 315 | */ 316 | @SimpleEvent 317 | public void GotStatus(final int statusCode, final String message) { 318 | broadcastEvent("GotStatus", statusCode, message); 319 | } 320 | 321 | /** 322 | * Event indicating that a classification got an error. 323 | * 324 | * @param data The data 325 | * @param error The error 326 | */ 327 | @SimpleEvent 328 | public void GotError(final String data, final String error) { 329 | broadcastEvent("GotError", data, error); 330 | } 331 | 332 | /** 333 | * Event indicating that a classification has finished. 334 | * 335 | * @param data The data 336 | * @param classification The classification 337 | * @param confidence The confidence of the classification 338 | */ 339 | @SimpleEvent 340 | public void GotClassification(final String data, final String classification, final double confidence) { 341 | broadcastEvent("GotClassification", data, classification, confidence); 342 | } 343 | 344 | 345 | // -------------------------------------------------------------------- 346 | // Helpers 347 | // -------------------------------------------------------------------- 348 | 349 | /** 350 | * Broadcasts an event on the UI thread 351 | * @param eventName the name of the event 352 | * @param data the data of the event 353 | */ 354 | private void broadcastEvent(final String eventName, final Object...data){ 355 | final Component component = this; 356 | activity.runOnUiThread(new Runnable() { 357 | @Override 358 | public void run() { 359 | EventDispatcher.dispatchEvent(component, eventName, data); 360 | } 361 | }); 362 | } 363 | 364 | /** 365 | * Converts a Yail List to a double list 366 | * @param list the list 367 | * @return the double list 368 | */ 369 | private List convertYailListToDouble(YailList list){ 370 | List numbersList = new ArrayList(list.size()); 371 | for (int i = 0; i < list.size(); i++){ 372 | String s = list.getString(i); 373 | numbersList.add(Double.parseDouble(s)); 374 | } 375 | return numbersList; 376 | } 377 | 378 | /** 379 | * Turn the data of an image into base 64. 380 | * 381 | * @param path The path to the image. 382 | * @return The data of the image as a base 64 string. 383 | */ 384 | private java.io.File loadImageFile(String path) { 385 | try { 386 | java.io.File image = MediaUtil.copyMediaToTempFile(form, path); 387 | return ImageResizer.resize(image, 224, 224); 388 | } catch (Exception e) { 389 | GotError(path, e.getMessage()); 390 | } 391 | return null; 392 | } 393 | 394 | /** 395 | * Loads the key from api.txt if exists 396 | * @return the API key 397 | */ 398 | private String getKeyFromFile(){ 399 | try { 400 | InputStream inputStream = form.openAssetForExtension(ML4KComponent.this, "api.txt"); 401 | Scanner scanner = new Scanner(inputStream); 402 | if (scanner.hasNext()){ 403 | return scanner.next(); 404 | } else { 405 | return ""; 406 | } 407 | } catch (IOException e) { 408 | e.printStackTrace(); 409 | return ""; 410 | } 411 | } 412 | 413 | /** 414 | * Runs a function in the background 415 | * @param runnable the runnable object 416 | */ 417 | private void runInBackground(Runnable runnable) { 418 | AsynchUtil.runAsynchronously(runnable); 419 | } 420 | 421 | private boolean isApiKeyMissing() { 422 | return key == null || key.trim().isEmpty(); 423 | } 424 | 425 | 426 | 427 | 428 | 429 | // --------------------------------------------------------------------- 430 | // SUPPORT FOR TENSORFLOW.JS MODELS ON-DEVICE IN HIDDEN EMBEDDED BROWSER 431 | // --------------------------------------------------------------------- 432 | 433 | private void initialiseTensorFlow() { 434 | Log.d(LOGPREFIX, "initializing tensorflowjs"); 435 | if (webPageObj == null) { 436 | activity.runOnUiThread(new Runnable() { 437 | @Override 438 | public void run() { 439 | browser = prepareBrowser(); 440 | loadWebPage(); 441 | } 442 | }); 443 | } 444 | } 445 | 446 | 447 | private void trainModelWithTensorflow() { 448 | initialiseTensorFlow(); 449 | 450 | if (webPageObj != null && webPageObj.isReady()) { 451 | webPageObj.trainNewModel(key); 452 | } 453 | else { 454 | // very unlikely to happen, but there is maybe a tiny race condition where it is possible? 455 | displayErrorMessage("Not ready yet. Please try again in a moment"); 456 | } 457 | } 458 | 459 | private void checkTensorFlowJsStatus() { 460 | if (webPageObj == null) { 461 | GotStatus(0, "Not trained"); 462 | } 463 | else { 464 | switch (webPageObj.getModelStatus()) { 465 | case "Not trained": 466 | case "Failed": 467 | GotStatus(0, webPageObj.getModelStatus()); 468 | break; 469 | case "Available": 470 | GotStatus(2, webPageObj.getModelStatus()); 471 | break; 472 | case "Training": 473 | GotStatus(1, webPageObj.getModelProgress() + "%"); 474 | break; 475 | } 476 | } 477 | } 478 | 479 | 480 | private void classifyImageWithTensorflow(final String path) { 481 | initialiseTensorFlow(); 482 | 483 | switch (webPageObj.getModelStatus()) { 484 | case "Not trained": 485 | displayErrorMessage(EXPLAIN_ML_MODEL_NEEDED); 486 | return; 487 | case "Failed": 488 | displayErrorMessage(EXPLAIN_ML_MODEL_FAILED); 489 | return; 490 | case "Training": 491 | displayErrorMessage(EXPLAIN_ML_MODEL_NOTREADY); 492 | return; 493 | case "Available": 494 | // yay, we're all good! 495 | break; 496 | } 497 | 498 | runInBackground(new Runnable() { 499 | @Override 500 | public void run() { 501 | try { 502 | Log.d(LOGPREFIX, "getting and resizing image"); 503 | final java.io.File image = loadImageFile(path); 504 | 505 | Log.d(LOGPREFIX, "encoding image"); 506 | final String imagedata = ImageEncoder.encode(image); 507 | 508 | activity.runOnUiThread(new Runnable() { 509 | @Override 510 | public void run() { 511 | webPageObj.submitClassificationRequest(imagedata); 512 | } 513 | }); 514 | 515 | image.delete(); 516 | } catch (Exception e) { 517 | e.printStackTrace(); 518 | GotError(path, e.getMessage()); 519 | } 520 | } 521 | }); 522 | } 523 | 524 | 525 | private WebResourceResponse prepareAssetForBrowser(String filename) throws IOException { 526 | Log.d(LOGPREFIX, "loading from assets " + filename); 527 | InputStream fileStream = form.openAssetForExtension(ML4KComponent.this, filename); 528 | String mime = "text/plain"; 529 | if (filename.endsWith(".json")) { 530 | mime = "application/json"; 531 | } 532 | else if (filename.endsWith(".js")) { 533 | mime = "text/javascript"; 534 | } 535 | else if (filename.endsWith(".html")) { 536 | mime = "text/html"; 537 | } 538 | else if (filename.endsWith("-shard1of1")) { 539 | mime = "application/octet-stream"; 540 | } 541 | else { 542 | Log.d(LOGPREFIX, "not available in assets " + filename); 543 | throw new IOException("File not included in assets"); 544 | } 545 | 546 | Map responseHeaders = new HashMap<>(); 547 | responseHeaders.put("Access-Control-Allow-Origin", "*"); 548 | return new WebResourceResponse(mime, "UTF-8", 200, "OK", responseHeaders, fileStream); 549 | } 550 | 551 | 552 | private WebView prepareBrowser() { 553 | Log.d(LOGPREFIX, "Creating browser to use for TensorFlow.js"); 554 | WebView webView = new WebView(activity); 555 | 556 | WebSettings webViewSettings = webView.getSettings(); 557 | webViewSettings.setJavaScriptEnabled(true); 558 | webViewSettings.setMediaPlaybackRequiresUserGesture(false); 559 | webViewSettings.setDomStorageEnabled(true); 560 | 561 | webView.setWebViewClient(new WebViewClient() { 562 | @Override 563 | public WebResourceResponse shouldInterceptRequest(WebView v, String url) { 564 | Log.d(LOGPREFIX, "shouldInterceptRequest " + url); 565 | 566 | if (url.startsWith(ML4KURL)) { 567 | try { 568 | String localUrl = url.substring(ML4KURL.length()); 569 | return prepareAssetForBrowser(localUrl); 570 | } 571 | catch (Exception exc) { 572 | exc.printStackTrace(); 573 | } 574 | } 575 | 576 | return super.shouldInterceptRequest(v, url); 577 | } 578 | }); 579 | 580 | return webView; 581 | } 582 | 583 | 584 | private void loadWebPage() { 585 | try { 586 | Log.d(LOGPREFIX, "binding to Java object"); 587 | webPageObj = new ML4KWebPage(browser, key.trim(), this); 588 | browser.addJavascriptInterface(webPageObj, "ML4KJavaInterface"); 589 | 590 | Log.d(LOGPREFIX, "loading tfjs web page"); 591 | browser.loadUrl(ML4KURL + "ml4k.html"); 592 | } 593 | catch (Exception e) { 594 | e.printStackTrace(); 595 | } 596 | } 597 | 598 | 599 | 600 | // --------------------------------------------------------------------- 601 | // COORDINATE TAKING ACTIONS THAT ARE IMPACTED BY THE PROJECT TYPE 602 | // --------------------------------------------------------------------- 603 | 604 | /** 605 | * Check the type of project before taking the requested action. 606 | * 607 | * The project type is read from an in-memory cache if it's already 608 | * known, otherwise an API call to ML4K servers is made first before 609 | * taking the requested action. 610 | * 611 | * @param nextAction Action that should be taken once project type has been verified 612 | */ 613 | private void checkProjectType(final NextAction nextAction) { 614 | if (isApiKeyMissing()) { 615 | processNextAction("unknown", nextAction); 616 | } 617 | else if (scratchKeyTypes.containsKey(key)) { 618 | processNextAction(scratchKeyTypes.get(key), nextAction); 619 | } 620 | else { 621 | runInBackground(new Runnable() { 622 | @Override 623 | public void run() { 624 | try { 625 | ML4K ml4k = new ML4K(key); 626 | ModelStatus status = ml4k.getModelStatus(); 627 | String type = status.getProjectType(); 628 | scratchKeyTypes.put(key, type); 629 | 630 | processNextAction(type, nextAction); 631 | } 632 | catch (ML4KException exc) { 633 | if (exc.getMessage().equals("Scratch key not found")) 634 | { 635 | scratchKeyTypes.put(key, "expired"); 636 | processNextAction("expired", nextAction); 637 | } 638 | else if (exc.getMessage().equals("API key isn't a Machine Learning for Kids key")) 639 | { 640 | scratchKeyTypes.put(key, "invalid"); 641 | processNextAction("invalid", nextAction); 642 | } 643 | else { 644 | processNextAction("unknown", nextAction); 645 | } 646 | } 647 | } 648 | }); 649 | } 650 | } 651 | 652 | // take the requested action 653 | private void processNextAction(final String projectType, final NextAction action) { 654 | Log.d(LOGPREFIX, "processNextAction " + action.step.name() + " for " + projectType); 655 | 656 | // startup actions 657 | if (action.step == NextStep.Init) { 658 | if (projectType.equals("imgtfjs")) { 659 | initialiseTensorFlow(); 660 | } 661 | return; 662 | } 663 | 664 | // if there is a problem with the API key, it 665 | // doesn't matter what the next step is, the 666 | // response is the same: display a message 667 | if ("invalid".equals(projectType)) { 668 | displayErrorMessage(EXPLAIN_API_KEY_INVALID); 669 | return; 670 | } 671 | if ("expired".equals(projectType)) { 672 | displayErrorMessage(EXPLAIN_API_KEY_EXPIRED); 673 | return; 674 | } 675 | if ("unknown".equals(projectType)) { 676 | displayErrorMessage(EXPLAIN_API_KEY_NEEDED); 677 | return; 678 | } 679 | 680 | // otherwise... 681 | switch (action.step) { 682 | // training data actions 683 | case AddTextTraining: 684 | switch (projectType) { 685 | case "text": 686 | addTrainingDataToProject(action); 687 | break; 688 | case "imgtfjs": 689 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_IMAGES); 690 | break; 691 | case "numbers": 692 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_NUMBERS); 693 | break; 694 | case "sounds": 695 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_SOUNDS); 696 | break; 697 | } 698 | break; 699 | case AddImageTraining: 700 | switch (projectType) { 701 | case "imgtfjs": 702 | addTrainingDataToProject(action); 703 | break; 704 | case "text": 705 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_TEXT); 706 | break; 707 | case "numbers": 708 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_NUMBERS); 709 | break; 710 | case "sounds": 711 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_SOUNDS); 712 | break; 713 | } 714 | break; 715 | case AddNumbersTraining: 716 | switch (projectType) { 717 | case "numbers": 718 | addTrainingDataToProject(action); 719 | break; 720 | case "imgtfjs": 721 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_IMAGES); 722 | break; 723 | case "text": 724 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_TEXT); 725 | break; 726 | case "sounds": 727 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_SOUNDS); 728 | break; 729 | } 730 | break; 731 | // classification actions 732 | case ClassifyText: 733 | switch (projectType) { 734 | case "text": 735 | classifyTestDataOnServer(action); 736 | break; 737 | case "imgtfjs": 738 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_IMAGES); 739 | break; 740 | case "numbers": 741 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_NUMBERS); 742 | break; 743 | case "sounds": 744 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_SOUNDS); 745 | break; 746 | } 747 | break; 748 | case ClassifyImage: 749 | switch (projectType) { 750 | case "imgtfjs": 751 | classifyImageWithTensorflow(((ClassifyImageAction)action).imagePath); 752 | break; 753 | case "text": 754 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_TEXT); 755 | break; 756 | case "numbers": 757 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_NUMBERS); 758 | break; 759 | case "sounds": 760 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_SOUNDS); 761 | break; 762 | } 763 | break; 764 | case ClassifyNumbers: 765 | switch (projectType) { 766 | case "numbers": 767 | classifyTestDataOnServer(action); 768 | break; 769 | case "imgtfjs": 770 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_IMAGES); 771 | break; 772 | case "text": 773 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_TEXT); 774 | break; 775 | case "sounds": 776 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_SOUNDS); 777 | break; 778 | } 779 | break; 780 | // ML model actions 781 | case TrainModel: 782 | switch (projectType) { 783 | case "text": 784 | case "numbers": 785 | trainModelWithWatson(); 786 | break; 787 | case "imgtfjs": 788 | trainModelWithTensorflow(); 789 | break; 790 | default: 791 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_SOUNDS); 792 | break; 793 | } 794 | break; 795 | case CheckModelStatus: 796 | switch (projectType) { 797 | case "text": 798 | case "numbers": 799 | fetchModelStatusFromServer(); 800 | break; 801 | case "imgtfjs": 802 | checkTensorFlowJsStatus(); 803 | break; 804 | default: 805 | displayErrorMessage(EXPLAIN_API_KEY_TYPE_SOUNDS); 806 | break; 807 | } 808 | break; 809 | } 810 | } 811 | 812 | 813 | 814 | 815 | 816 | 817 | // -------------------------------------------------------------------- 818 | // DEFINES ACTIONS THAT ARE CARRIED OUT DIFFERENTLY DEPENDING ON THE 819 | // TYPE OF ML PROJECT (e.g. text, images, etc.) 820 | // 821 | // This means these actions can only be carried out after fetching the 822 | // type of project. 823 | // -------------------------------------------------------------------- 824 | 825 | private enum NextStep { 826 | // init local ML model - only relevant for project types 827 | // where models are hosted on-device 828 | Init, 829 | // add training data to the project 830 | AddTextTraining, 831 | AddImageTraining, 832 | AddNumbersTraining, 833 | // classify test data using a model 834 | ClassifyText, 835 | ClassifyImage, 836 | ClassifyNumbers, 837 | // train a new ML model 838 | TrainModel, 839 | // check the current status of the ML model 840 | CheckModelStatus; 841 | } 842 | private class NextAction { 843 | private NextStep step; 844 | private NextAction(NextStep step) { 845 | this.step = step; 846 | } 847 | } 848 | private class ClassifyImageAction extends NextAction { 849 | private String imagePath; 850 | private ClassifyImageAction(String path) { 851 | super(NextStep.ClassifyImage); 852 | this.imagePath = path; 853 | } 854 | } 855 | private class ClassifyNumbersAction extends NextAction { 856 | private YailList numbers; 857 | private ClassifyNumbersAction(YailList nums) { 858 | super(NextStep.ClassifyNumbers); 859 | this.numbers = nums; 860 | } 861 | } 862 | private class ClassifyTextAction extends NextAction { 863 | private String text; 864 | private ClassifyTextAction(String textData) { 865 | super(NextStep.ClassifyText); 866 | this.text = textData; 867 | } 868 | } 869 | private class TextTrainingAction extends NextAction { 870 | private String text; 871 | private String label; 872 | private TextTrainingAction(String label, String textData) { 873 | super(NextStep.AddTextTraining); 874 | this.text = textData; 875 | this.label = label; 876 | } 877 | } 878 | private class ImageTrainingAction extends NextAction { 879 | private String path; 880 | private String label; 881 | private ImageTrainingAction(String label, String imagePath) { 882 | super(NextStep.AddImageTraining); 883 | this.path = imagePath; 884 | this.label = label; 885 | } 886 | } 887 | private class NumbersTrainingAction extends NextAction { 888 | private YailList numbers; 889 | private String label; 890 | private NumbersTrainingAction(String label, YailList nums) { 891 | super(NextStep.AddNumbersTraining); 892 | this.label = label; 893 | this.numbers = nums; 894 | } 895 | } 896 | } 897 | -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/ML4KException.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | class ML4KException extends Exception { 4 | 5 | private static final long serialVersionUID = 1L; 6 | 7 | public ML4KException(String msg) { 8 | super(msg); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/ML4KWebPage.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | import java.util.Date; 4 | import android.app.Activity; 5 | import android.util.Log; 6 | import android.webkit.JavascriptInterface; 7 | import android.webkit.ValueCallback; 8 | import android.webkit.WebView; 9 | 10 | public class ML4KWebPage { 11 | 12 | private static final String LOGPREFIX = "ML4KWebPage"; 13 | private boolean pageReady = false; 14 | 15 | private String currentScratchKey; 16 | 17 | private String modelStatus = "Not trained"; 18 | private int modelProgress = 0; 19 | 20 | private WebView browser; 21 | 22 | private ML4KComponent callback; 23 | 24 | ML4KWebPage(WebView browserView, String scratchkey, ML4KComponent parent) { 25 | Log.d(LOGPREFIX, "Creating JS/Java interface"); 26 | browser = browserView; 27 | pageReady = false; 28 | currentScratchKey = scratchkey; 29 | callback = parent; 30 | } 31 | 32 | @JavascriptInterface 33 | public void setReady(boolean ready) { 34 | pageReady = ready; 35 | } 36 | public boolean isReady() { 37 | return pageReady; 38 | } 39 | 40 | 41 | @JavascriptInterface 42 | public String getInitialScratchKey() { 43 | return currentScratchKey; 44 | } 45 | 46 | 47 | @JavascriptInterface 48 | public void setModelStatus(String status, int progress) { 49 | modelStatus = status; 50 | modelProgress = progress; 51 | } 52 | 53 | public int getModelProgress() { 54 | return modelProgress; 55 | } 56 | public String getModelStatus() { 57 | return modelStatus; 58 | } 59 | 60 | @JavascriptInterface 61 | public void classifyResponse(String label, double confidence){ 62 | Log.d(LOGPREFIX, "Received classify response"); 63 | callback.GotClassification(label, label, confidence); 64 | } 65 | 66 | @JavascriptInterface 67 | public void returnErrorMessage(String errormessage) { 68 | Log.d(LOGPREFIX, "Returning error " + errormessage); 69 | callback.displayErrorMessage(errormessage); 70 | } 71 | 72 | public void trainNewModel(String scratchkey) { 73 | Log.d(LOGPREFIX, "Training new TensorflowJS model"); 74 | runWebpageFunction("ml4kTrainNewModel('" + scratchkey + "')"); 75 | } 76 | 77 | public void submitClassificationRequest(String base64imagedata) { 78 | Log.d(LOGPREFIX, "Classifying image"); 79 | runWebpageFunction("ml4kClassifyImage('" + base64imagedata + "')"); 80 | } 81 | 82 | private void runWebpageFunction(String function) { 83 | browser.evaluateJavascript("(function() { " + function + "; })();", null); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/ModelStatus.java: -------------------------------------------------------------------------------- 1 | package com.kylecorry.ml4k; 2 | 3 | class ModelStatus { 4 | private int statusCode; 5 | private String message; 6 | private String projectType; 7 | 8 | /** 9 | * Default constructor 10 | * @param statusCode the status code 11 | * @param message the message 12 | */ 13 | private ModelStatus(int statusCode, String message, String projectType){ 14 | this.statusCode = statusCode; 15 | this.message = message; 16 | this.projectType = projectType; 17 | } 18 | 19 | /** 20 | * Loads a model status from JSON 21 | * @param json the JSON 22 | * @return the model status 23 | * @throws ML4KException if the JSON does not represent a model status 24 | */ 25 | public static ModelStatus fromJson(String json) throws ML4KException { 26 | int code = JSONUtils.readIntProperty(json, "status"); 27 | String message = JSONUtils.readStringProperty(json, "msg"); 28 | String type = JSONUtils.readStringProperty(json, "type"); 29 | if (message == null || type == null){ 30 | throw new ML4KException("JSON is not valid: " + json); 31 | } 32 | if (type.equals("text") || 33 | type.equals("imgtfjs") || 34 | type.equals("numbers") || 35 | type.equals("sounds")) 36 | { 37 | return new ModelStatus(code, message, type); 38 | } 39 | else { 40 | throw new ML4KException("Unexpected project type: " + type); 41 | } 42 | } 43 | 44 | /** 45 | * @return the status code (0 = error, 1 = training, 2 = ready) 46 | */ 47 | public int getStatusCode(){ 48 | return statusCode; 49 | } 50 | 51 | /** 52 | * @return the message 53 | */ 54 | public String getMessage(){ 55 | return message; 56 | } 57 | 58 | public String getProjectType(){ 59 | return projectType; 60 | } 61 | 62 | @Override 63 | public String toString() { 64 | return "ModelStatus{" + 65 | "statusCode=" + statusCode + 66 | ", message='" + message + '\'' + 67 | ", type='" + projectType + '\'' + 68 | '}'; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/aiwebres/ml4k.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/src/com/kylecorry/ml4k/aiwebres/ml4k.png -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/assets/api.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylecorry31/ML4K-AI-Extension/da02a75532e9e1536ee4c3bd7bdcbb8e42eaa18d/src/com/kylecorry/ml4k/assets/api.txt -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/assets/ml4k.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /src/com/kylecorry/ml4k/assets/ml4k.js: -------------------------------------------------------------------------------- 1 | // --------------------------------------------------------------------- 2 | 3 | var _ML4K_IMG_WIDTH = 224; 4 | var _ML4K_IMG_HEIGHT = 224; 5 | 6 | var _ML4K_MODEL_TYPE = 'images'; 7 | 8 | // --------------------------------------------------------------------- 9 | 10 | var _ml4kBaseModel; 11 | var _ml4kTransferModel; 12 | 13 | var _ml4kModelClasses; 14 | 15 | var _ml4kUsingRestoredModel = false; 16 | 17 | // --------------------------------------------------------------------- 18 | 19 | function _ml4kFetchJson(urlString) { 20 | return fetch(urlString) 21 | .then(function (resp) { 22 | if (resp.ok) { 23 | return resp.json(); 24 | } 25 | else { 26 | console.log(resp); 27 | throw new Error('Error in communicating with Machine Learning for Kids server'); 28 | } 29 | }); 30 | } 31 | 32 | function _ml4kFetchData(urlString) { 33 | return fetch(urlString) 34 | .then(function (resp) { 35 | if (resp.ok) { 36 | return resp.arrayBuffer(); 37 | } 38 | else { 39 | console.log(resp); 40 | throw new Error('Error in downloading image data'); 41 | } 42 | }); 43 | } 44 | 45 | // --------------------------------------------------------------------- 46 | 47 | function _ml4kGetModelDbLocation(modeltype, scratchkey) { 48 | return 'indexeddb://ml4k-models-' + 49 | modeltype + '-' + 50 | scratchkey.replace(/-/g, ''); 51 | } 52 | 53 | function _ml4kLoadModel(modeltype, scratchkey) { 54 | console.log('loading model'); 55 | var savelocation = _ml4kGetModelDbLocation(modeltype, scratchkey); 56 | return tf.loadLayersModel(savelocation) 57 | .then(function (resp) { 58 | console.log('loaded model'); 59 | 60 | if (window.localStorage) { 61 | var modelMetadataStr = window.localStorage.getItem('ml4k-modelinfo-' + modeltype + '-' + scratchkey); 62 | _ml4kModelClasses = JSON.parse(modelMetadataStr).classes; 63 | } 64 | else { 65 | console.log('Unable to access local storage'); 66 | } 67 | 68 | ML4KJavaInterface.setModelStatus('Available', 100); 69 | _ml4kUsingRestoredModel = true; 70 | return resp; 71 | }) 72 | .catch(function (err) { 73 | console.log('failed to load model'); 74 | console.log(err); 75 | }); 76 | } 77 | 78 | function _ml4kSaveModel(modeltype, scratchkey, modelClasses, transferModel) { 79 | console.log('saving model'); 80 | var savelocation = _ml4kGetModelDbLocation(modeltype, scratchkey); 81 | return transferModel.save(savelocation) 82 | .then(function () { 83 | if (window.localStorage) { 84 | window.localStorage.setItem('ml4k-modelinfo-' + modeltype + '-' + scratchkey, 85 | JSON.stringify({ classes : modelClasses })); 86 | } 87 | else { 88 | console.log('unable to save model metadata'); 89 | } 90 | }) 91 | .catch(function (err) { 92 | console.log('failed to save model'); 93 | console.log(err); 94 | }); 95 | } 96 | 97 | // --------------------------------------------------------------------- 98 | 99 | 100 | function _ml4kPrepareMobilenet() { 101 | return tf.loadLayersModel('https://machinelearningforkids.co.uk/appinventor-assets/model.json') 102 | .then(function (pretrainedModel) { 103 | var activationLayer = pretrainedModel.getLayer('conv_pw_13_relu'); 104 | return tf.model({ 105 | inputs : pretrainedModel.inputs, 106 | outputs: activationLayer.output 107 | }); 108 | }) 109 | .catch(function (err) { 110 | console.log('failed to prepare mobilenet'); 111 | console.log(err); 112 | throw err; 113 | }); 114 | } 115 | 116 | function _ml4kPrepareTransferLearningModel(modifiedMobilenet, numClasses) { 117 | var model = tf.sequential({ 118 | layers : [ 119 | tf.layers.flatten({ 120 | inputShape : modifiedMobilenet.outputs[0].shape.slice(1) 121 | }), 122 | tf.layers.dense({ 123 | units : 100, 124 | activation : 'relu', 125 | kernelInitializer : 'varianceScaling', 126 | useBias : true 127 | }), 128 | tf.layers.dense({ 129 | units : numClasses, 130 | activation : 'softmax', 131 | kernelInitializer : 'varianceScaling', 132 | useBias : false 133 | }) 134 | ] 135 | }); 136 | 137 | model.compile({ 138 | optimizer : tf.train.adam(0.0001), 139 | loss : 'categoricalCrossentropy' 140 | }); 141 | 142 | return model; 143 | } 144 | 145 | function _ml4kTrainModel(baseModel, transferModel, scratchkey, trainingdata) { 146 | ML4KJavaInterface.setModelStatus('Training', 0); 147 | 148 | _ml4kModelClasses = trainingdata.labels; 149 | 150 | var xs; 151 | var ys; 152 | 153 | for (var i=0; i < trainingdata.imagedata.length; i++) { 154 | var trainingdataitem = trainingdata.imagedata[i]; 155 | 156 | var labelIdx = _ml4kModelClasses.indexOf(trainingdataitem.label); 157 | 158 | var xval = baseModel.predict(trainingdataitem.tensor); 159 | var yval = tf.tidy(function () { 160 | return tf.oneHot(tf.tensor1d([ labelIdx ]).toInt(), _ml4kModelClasses.length); 161 | }); 162 | 163 | if (i === 0) { 164 | xs = xval; 165 | ys = yval; 166 | } 167 | else { 168 | var oldxs = xs; 169 | var oldys = ys; 170 | xs = oldxs.concat(xval, 0); 171 | ys = oldys.concat(yval, 0); 172 | 173 | oldxs.dispose(); 174 | oldys.dispose(); 175 | } 176 | } 177 | 178 | var epochs = 10; 179 | if (trainingdata.imagedata.length > 55) { 180 | epochs = 15; 181 | } 182 | 183 | transferModel.fit(xs, ys, { 184 | batchSize : 10, 185 | epochs : epochs, 186 | callbacks : { 187 | onEpochEnd : function (epoch, logs) { 188 | console.log('epoch ' + epoch + ' loss ' + logs.loss); 189 | if (epochs === 15) { 190 | ML4KJavaInterface.setModelStatus('Training', (epoch + 1) * 7); 191 | } 192 | else { 193 | ML4KJavaInterface.setModelStatus('Training', (epoch + 1) * 10); 194 | } 195 | }, 196 | onTrainEnd : function () { 197 | return _ml4kSaveModel(_ML4K_MODEL_TYPE, scratchkey, _ml4kModelClasses, transferModel) 198 | .then(function () { 199 | console.log('training complete'); 200 | ML4KJavaInterface.setModelStatus('Available', 100); 201 | 202 | _ml4kUsingRestoredModel = false; 203 | }); 204 | } 205 | } 206 | }); 207 | } 208 | 209 | // --------------------------------------------------------------------- 210 | 211 | function _ml4kCreateTensorForImageData(imageid, imagedata, imagelabel) { 212 | return new Promise(function (resolve, reject) { 213 | var imageDataBlob = URL.createObjectURL(new Blob([ imagedata ])); 214 | 215 | var hiddenImg = document.createElement('img'); 216 | hiddenImg.id = '_ml4k_' + imageid; 217 | hiddenImg.width = _ML4K_IMG_WIDTH; 218 | hiddenImg.height = _ML4K_IMG_HEIGHT; 219 | hiddenImg.onerror = function (err) { 220 | console.log('Failed to load image', imageid); 221 | console.log(err); 222 | return reject(err); 223 | }; 224 | hiddenImg.onload = function () { 225 | var tensorData = tf.tidy(function () { 226 | return tf.browser.fromPixels(hiddenImg) 227 | .expandDims(0) 228 | .toFloat() 229 | .div(127) 230 | .sub(1); 231 | }); 232 | 233 | resolve({ id : imageid, label : imagelabel, tensor : tensorData }); 234 | 235 | URL.revokeObjectURL(imageDataBlob); 236 | }; 237 | 238 | hiddenImg.src = imageDataBlob; 239 | }); 240 | } 241 | 242 | function _ml4kCreateTensorForTestImage(imagedata) { 243 | return new Promise(function (resolve, reject) { 244 | var hiddenImg = document.createElement('img'); 245 | hiddenImg.id = '_ml4k_test_' + Date.now(); 246 | hiddenImg.width = _ML4K_IMG_WIDTH; 247 | hiddenImg.height = _ML4K_IMG_HEIGHT; 248 | hiddenImg.onerror = function (err) { 249 | console.log('Failed to load image'); 250 | console.log(imagedata); 251 | return reject(err); 252 | }; 253 | hiddenImg.onload = function () { 254 | var tensorData = tf.tidy(function () { 255 | return tf.browser.fromPixels(hiddenImg) 256 | .expandDims(0) 257 | .toFloat() 258 | .div(127) 259 | .sub(1); 260 | }); 261 | 262 | resolve(tensorData); 263 | }; 264 | hiddenImg.src = 'data:image/jpg;base64,' + imagedata; 265 | }); 266 | } 267 | 268 | function _ml4kGetImageData(imageid, imageurl, imagelabel) { 269 | console.log('getImageData : ' + imageid + ' : ' + imageurl); 270 | return _ml4kFetchData(imageurl) 271 | .then(function (imagedata) { 272 | return _ml4kCreateTensorForImageData(imageid, imagedata, imagelabel); 273 | }) 274 | .catch(function (err) { 275 | console.log(err); 276 | throw new Error('Unable to process training image at ' + imageurl); 277 | }); 278 | } 279 | 280 | function _ml4kGetTrainingImages(scratchkey) { 281 | console.log('getting training images'); 282 | var labels = new Set(); 283 | return _ml4kFetchJson('https://machinelearningforkids.co.uk/api/scratch/' + scratchkey + '/train?proxy=true') 284 | .then(function (imagesList) { 285 | return pool({ 286 | collection: imagesList, 287 | maxConcurrency: 10, 288 | task: function (imageInfo) { 289 | labels.add(imageInfo.label); 290 | return _ml4kGetImageData(imageInfo.id, imageInfo.imageurl, imageInfo.label); 291 | } 292 | }); 293 | }) 294 | .then(function (trainingimages) { 295 | return { imagedata : trainingimages, labels : Array.from(labels) }; 296 | }); 297 | } 298 | 299 | // --------------------------------------------------------------------- 300 | 301 | function _ml4kSortByConfidence(a, b) { 302 | if (a.confidence < b.confidence) { 303 | return 1; 304 | } 305 | else if (a.confidence > b.confidence) { 306 | return -1; 307 | } 308 | else { 309 | return 0; 310 | } 311 | } 312 | 313 | 314 | function _ml4kTestImageDataTensor(testTensor) { 315 | var baseModelOutput = _ml4kBaseModel.predict(testTensor); 316 | var transferModelOutput = _ml4kTransferModel.predict(baseModelOutput); 317 | 318 | return transferModelOutput.data() 319 | .then(function (output) { 320 | console.log(JSON.stringify(_ml4kModelClasses)); 321 | console.log(JSON.stringify(output)); 322 | 323 | if (output.length !== _ml4kModelClasses.length) { 324 | console.log('unexpected output from model', output); 325 | return []; 326 | } 327 | 328 | var scores = _ml4kModelClasses.map(function (label, idx) { 329 | return { 330 | class_name : label, 331 | confidence : 100 * output[idx] 332 | }; 333 | }).sort(_ml4kSortByConfidence); 334 | return scores; 335 | }) 336 | .catch(function (err) { 337 | console.log('failed to test image'); 338 | console.log(err); 339 | return []; 340 | }); 341 | } 342 | 343 | function ml4kClassifyImage(imagedata) { 344 | return _ml4kCreateTensorForTestImage(imagedata) 345 | .then(function (tensor) { 346 | return _ml4kTestImageDataTensor(tensor); 347 | }) 348 | .then(function (modelOutput) { 349 | if (modelOutput.length > 0) { 350 | ML4KJavaInterface.classifyResponse(modelOutput[0].class_name, modelOutput[0].confidence); 351 | } 352 | else { 353 | ML4KJavaInterface.classifyResponse("Unknown", 0.1); 354 | } 355 | }) 356 | .catch(function (err) { 357 | console.log('failed to classify image'); 358 | console.log(err); 359 | ML4KJavaInterface.classifyResponse("Unknown", 0.1); 360 | }); 361 | } 362 | 363 | // --------------------------------------------------------------------- 364 | 365 | function ml4kTrainNewModel(scratchkey) { 366 | return _ml4kGetTrainingImages(scratchkey) 367 | .then(function (trainingdata) { 368 | if (_ml4kUsingRestoredModel || !_ml4kTransferModel) { 369 | _ml4kTransferModel = _ml4kPrepareTransferLearningModel(_ml4kBaseModel, trainingdata.labels.length); 370 | } 371 | return _ml4kTrainModel(_ml4kBaseModel, _ml4kTransferModel, scratchkey, trainingdata); 372 | }) 373 | .catch(function (err) { 374 | console.log('model training failure'); 375 | console.log(err); 376 | ML4KJavaInterface.setModelStatus('Failed', 0); 377 | ML4KJavaInterface.returnErrorMessage(err.message); 378 | }); 379 | } 380 | 381 | 382 | 383 | 384 | function ml4kOnStart(scratchkey) { 385 | if (tf && tf.enableProdMode) { 386 | tf.enableProdMode(); 387 | } 388 | console.log(tf.version); 389 | 390 | return _ml4kPrepareMobilenet() 391 | .then(function (preparedBaseModel) { 392 | _ml4kBaseModel = preparedBaseModel; 393 | 394 | if (scratchkey) { 395 | return _ml4kLoadModel(_ML4K_MODEL_TYPE, scratchkey); 396 | } 397 | }) 398 | .then(function (loadedmodel) { 399 | if (loadedmodel) { 400 | _ml4kTransferModel = loadedmodel; 401 | } 402 | 403 | ML4KJavaInterface.setReady(true); 404 | }); 405 | } 406 | --------------------------------------------------------------------------------