├── .expo-shared └── assets.json ├── .gitignore ├── App.tsx ├── README.md ├── app.json ├── assets ├── fonts │ └── SpaceMono-Regular.ttf ├── images │ ├── favicon.png │ ├── icon.png │ └── splash.png └── model_tfjs │ └── .gitkeep ├── babel.config.js ├── components ├── ModelService.tsx ├── StyledText.tsx ├── Themed.tsx └── __tests__ │ └── StyledText-test.js ├── config.tsx ├── constants ├── Colors.ts └── Layout.ts ├── demo └── app_in_action.gif ├── hooks ├── useCachedResources.ts ├── useColorScheme.ts └── useColorScheme.web.ts ├── metro.config.js ├── navigation ├── BottomTabNavigator.tsx ├── LinkingConfiguration.ts └── index.tsx ├── notes.md ├── package.json ├── screens ├── AboutScreen.tsx ├── DebugScreen.tsx ├── HomeScreen.tsx └── NotFoundScreen.tsx ├── test.md ├── tsconfig.json ├── types.tsx ├── yarn-error.log └── yarn.lock /.expo-shared/assets.json: -------------------------------------------------------------------------------- 1 | { 2 | "e997a5256149a4b76e6bfd6cbf519c5e5a0f1d278a3d8fa1253022b03c90473b": true, 3 | "af683c96e0ffd2cf81287651c9433fa44debc1220ca7cb431fe482747f34a505": true, 4 | "12bb71342c6255bbf50437ec8f4441c083f47cdb74bd89160c15e4f43e52a1cb": true, 5 | "40b842e832070c58deac6aa9e08fa459302ee3f9da492c7e77d93d2fbf4a56fd": true 6 | } 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/**/* 2 | .expo/* 3 | npm-debug.* 4 | *.jks 5 | *.p8 6 | *.p12 7 | *.key 8 | *.mobileprovision 9 | *.orig.* 10 | web-build/ 11 | 12 | # macOS 13 | .DS_Store 14 | .vscode 15 | assets/model_tfjs/* 16 | 17 | !assets/model_tfjs/.gitkeep -------------------------------------------------------------------------------- /App.tsx: -------------------------------------------------------------------------------- 1 | import { StatusBar } from 'expo-status-bar'; 2 | import React from 'react'; 3 | import { SafeAreaProvider } from 'react-native-safe-area-context'; 4 | 5 | import useCachedResources from './hooks/useCachedResources'; 6 | import useColorScheme from './hooks/useColorScheme'; 7 | import Navigation from './navigation'; 8 | import { Button, ThemeProvider } from 'react-native-elements'; 9 | 10 | 11 | export default function App() { 12 | const isLoadingComplete = useCachedResources(); 13 | const colorScheme = useColorScheme(); 14 | 15 | if (!isLoadingComplete) { 16 | return null; 17 | } else { 18 | return ( 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | ); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # deploying-mobile-app 2 | 3 | 4 | ## About 5 | 6 | This is a sample template repo showing how to use tensorflow.js with react native. 7 | 8 | Here is the app in action. 9 | 10 | ![In Action](demo/app_in_action.gif) 11 | 12 | 13 | 14 | ## Assumptions 15 | 16 | In order to use this repo, you need to have a tfjs model. You can get a smaple artifact from this repo's release. 17 | 18 | Please download it from this repos's release and place the contents in `assets/model_tfjs` 19 | 20 | ``` 21 | (base) deploying-mobile-app % tree assets/model_tfjs 22 | assets/model_tfjs 23 | ├── classes.json 24 | ├── classes.txt 25 | ├── group1-shard1of1.bin 26 | └── model.json 27 | ``` 28 | 29 | If you want to bring in your own model, please use a command like below to convert a keras h5 artifact. 30 | 31 | The below command converts `artifacts/model_tf_keras.h5` to tfjs and saves it to `artifacts/model_tfjs`. 32 | 33 | The model is also 16 bit quanitzed. 34 | The model is converted to `tfjs_graph_model` which is an optimized version of the graph. 35 | The model is broken into 100MB shards. 36 | 37 | ``` 38 | pip install tensorflowjs==2.3.0 39 | 40 | tensorflowjs_converter \ 41 | --input_format=keras \ 42 | --output_format=tfjs_graph_model \ 43 | --split_weights_by_layer \ 44 | --weight_shard_size_bytes=99999999 \ 45 | --quantize_float16=* \ 46 | artifacts/model_tf_keras.h5 artifacts/model_tfjs 47 | ``` 48 | 49 | 50 | 51 | ## Running 52 | 53 | 54 | ``` 55 | yarn global add expo-cli 56 | 57 | # you can open iOS, Android, or web from here, or run them directly with the commands below. 58 | yarn start 59 | 60 | # you can open in android simulator or device driectly 61 | yarn android 62 | 63 | # you can open in ios simulator or device directly 64 | yarn ios 65 | 66 | 67 | ``` 68 | 69 | 70 | 71 | 72 | ## Future 73 | 74 | In the future, it is possible that packages may be out of date. 75 | 76 | The below upgrade will upgrade expo and its dependencies 77 | ``` 78 | expo upgrade 79 | ``` 80 | 81 | The below command command can be used to interactively update dependencies. 82 | 83 | ``` 84 | yarn upgrade-interactive 85 | ``` 86 | 87 | ## Other 88 | 89 | If you are interested in a guided exploration of this content, consider Manning's Live project [Deploying a Deep Learning Model on Web and Mobile Applications 90 | ](https://www.manning.com/liveproject/deploying-a-deep-learning-model-on-web-and-mobile-applications) 91 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "expo": { 3 | "name": "mobile-app", 4 | "slug": "mobile-app", 5 | "version": "1.0.0", 6 | "orientation": "portrait", 7 | "icon": "./assets/images/icon.png", 8 | "scheme": "myapp", 9 | "userInterfaceStyle": "automatic", 10 | "splash": { 11 | "image": "./assets/images/splash.png", 12 | "resizeMode": "contain", 13 | "backgroundColor": "#ffffff" 14 | }, 15 | "updates": { 16 | "fallbackToCacheTimeout": 0 17 | }, 18 | "assetBundlePatterns": [ 19 | "**/*" 20 | ], 21 | "ios": { 22 | "supportsTablet": true 23 | }, 24 | "web": { 25 | "favicon": "./assets/images/favicon.png" 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /assets/fonts/SpaceMono-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/assets/fonts/SpaceMono-Regular.ttf -------------------------------------------------------------------------------- /assets/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/assets/images/favicon.png -------------------------------------------------------------------------------- /assets/images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/assets/images/icon.png -------------------------------------------------------------------------------- /assets/images/splash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/assets/images/splash.png -------------------------------------------------------------------------------- /assets/model_tfjs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/assets/model_tfjs/.gitkeep -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = function(api) { 2 | api.cache(true); 3 | return { 4 | presets: ['babel-preset-expo'], 5 | env: { 6 | production: { 7 | plugins: ['react-native-paper/babel'], 8 | }, 9 | } 10 | }; 11 | }; 12 | -------------------------------------------------------------------------------- /components/ModelService.tsx: -------------------------------------------------------------------------------- 1 | import * as tf from '@tensorflow/tfjs'; 2 | import * as FileSystem from 'expo-file-system' 3 | import { fetch ,asyncStorageIO,bundleResourceIO,decodeJpeg} from '@tensorflow/tfjs-react-native' 4 | import {Image} from 'react-native'; 5 | import * as ImageManipulator from 'expo-image-manipulator'; 6 | import {AppConfig} from "../config" 7 | 8 | export interface ModelPrediction { 9 | className:string; 10 | probability:number; 11 | } 12 | 13 | export interface IModelPredictionTiming { 14 | totalTime:number; 15 | imageLoadingTime:number; 16 | imagePreprocessing:number; 17 | imagePrediction:number; 18 | imageDecodePrediction:number; 19 | } 20 | 21 | export interface IModelPredictionResponse { 22 | predictions?:ModelPrediction[] | null 23 | timing?:IModelPredictionTiming | null 24 | error?:string | null 25 | } 26 | 27 | const imageToTensor = (rawImageData:Uint8Array)=> { 28 | return decodeJpeg(rawImageData); 29 | } 30 | 31 | 32 | const fetchImage = async (image:ImageManipulator.ImageResult) => { 33 | let imgB64:string; 34 | if(image.base64){ 35 | imgB64=image.base64 36 | }else{ 37 | const imageAssetPath = Image.resolveAssetSource(image) 38 | console.log(imageAssetPath.uri); 39 | 40 | imgB64 = await FileSystem.readAsStringAsync(imageAssetPath.uri, { 41 | encoding: FileSystem.EncodingType.Base64, 42 | }); 43 | } 44 | 45 | const imgBuffer = tf.util.encodeString(imgB64, 'base64').buffer; 46 | const rawImageData = new Uint8Array(imgBuffer) 47 | 48 | return rawImageData; 49 | } 50 | const preprocessImage = (img:tf.Tensor3D,imageSize:number) =>{ 51 | // https://github.com/keras-team/keras-applications/blob/master/keras_applications/imagenet_utils.py#L43 52 | 53 | let imageTensor = img.resizeBilinear([imageSize, imageSize]).toFloat(); 54 | 55 | const offset = tf.scalar(127.5); 56 | const normalized = imageTensor.sub(offset).div(offset); 57 | const preProcessedImage = normalized.reshape([1, imageSize, imageSize, 3]); 58 | return preProcessedImage; 59 | 60 | } 61 | 62 | const decodePredictions = (predictions:tf.Tensor, classes:String[],topK=3) =>{ 63 | const {values, indices} = predictions.topk(topK); 64 | const topKValues = values.dataSync(); 65 | const topKIndices = indices.dataSync(); 66 | 67 | const topClassesAndProbs:ModelPrediction[] = []; 68 | for (let i = 0; i < topKIndices.length; i++) { 69 | topClassesAndProbs.push({ 70 | className: classes[topKIndices[i]], 71 | probability: topKValues[i] 72 | } as ModelPrediction); 73 | } 74 | return topClassesAndProbs; 75 | } 76 | 77 | 78 | export class ModelService { 79 | 80 | private model:tf.GraphModel; 81 | private model_classes: String[]; 82 | private imageSize:number; 83 | private static instance: ModelService; 84 | 85 | constructor(imageSize:number,model:tf.GraphModel, model_classes: String[] ){ 86 | this.imageSize=imageSize; 87 | this.model = model; 88 | this.model_classes=model_classes; 89 | } 90 | 91 | 92 | static async create(imageSize:number) { 93 | if (!ModelService.instance){ 94 | await tf.ready(); 95 | const modelJSON = require('../assets/model_tfjs/model.json'); 96 | const modelWeights = require('../assets/model_tfjs/group1-shard1of1.bin'); 97 | const model_classes = require("../assets/model_tfjs/classes.json") 98 | 99 | const model = await tf.loadGraphModel(bundleResourceIO(modelJSON, modelWeights)); 100 | model.predict(tf.zeros([1, imageSize, imageSize, 3])); 101 | 102 | ModelService.instance = new ModelService(imageSize,model,model_classes); 103 | } 104 | 105 | return ModelService.instance; 106 | 107 | } 108 | 109 | async classifyImage(image:ImageManipulator.ImageResult):Promise{ 110 | const predictionResponse = {timing:null,predictions:null,error:null} as IModelPredictionResponse; 111 | try { 112 | console.log(`Classifying Image: Start `) 113 | 114 | let imgBuffer:Uint8Array = await fetchImage(image); 115 | const timeStart = new Date().getTime() 116 | console.log(`Backend: ${tf.getBackend()} `) 117 | tf.tidy(()=>{ 118 | console.log(`Fetching Image: Start `) 119 | 120 | const imageTensor:tf.Tensor3D = imageToTensor(imgBuffer); 121 | 122 | 123 | console.log(`Fetching Image: Done `) 124 | const timeLoadDone = new Date().getTime() 125 | 126 | console.log("Preprocessing image: Start") 127 | 128 | const preProcessedImage = preprocessImage(imageTensor,this.imageSize); 129 | 130 | console.log("Preprocessing image: Done") 131 | const timePrepocessDone = new Date().getTime() 132 | 133 | console.log("Prediction: Start") 134 | const predictionsTensor:tf.Tensor = this.model.predict(preProcessedImage) as tf.Tensor; 135 | 136 | console.log("Prediction: Done") 137 | const timePredictionDone = new Date().getTime() 138 | 139 | console.log("Post Processing: Start") 140 | 141 | // post processing 142 | predictionResponse.predictions = decodePredictions(predictionsTensor,this.model_classes,AppConfig.topK); 143 | 144 | 145 | //tf.dispose(imageTensor); 146 | //tf.dispose(preProcessedImage); 147 | //tf.dispose(predictions); 148 | 149 | console.log("Post Processing: Done") 150 | 151 | const timeEnd = new Date().getTime() 152 | 153 | const timing:IModelPredictionTiming = { 154 | totalTime: timeEnd-timeStart, 155 | imageLoadingTime : timeLoadDone-timeStart, 156 | imagePreprocessing : timePrepocessDone-timeLoadDone, 157 | imagePrediction : timePredictionDone-timePrepocessDone , 158 | imageDecodePrediction : timeEnd-timePredictionDone 159 | } as IModelPredictionTiming; 160 | predictionResponse.timing = timing; 161 | 162 | }); 163 | 164 | 165 | console.log(`Classifying Image: End `); 166 | 167 | console.log(`Response: ${JSON.stringify(predictionResponse ,null, 2 ) } `); 168 | return predictionResponse as IModelPredictionResponse 169 | 170 | } catch (error) { 171 | console.log('Exception Error: ', error) 172 | return {error} 173 | } 174 | } 175 | } 176 | 177 | -------------------------------------------------------------------------------- /components/StyledText.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | 3 | import { Text, TextProps } from './Themed'; 4 | 5 | export function MonoText(props: TextProps) { 6 | return ; 7 | } 8 | -------------------------------------------------------------------------------- /components/Themed.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { View as DefaultView ,ScrollView as DefaultScrollView, ActivityIndicator as DefaultActivityIndicator } from 'react-native'; 3 | import {Text as DefaultText, ListItem as DefaultListItem } from 'react-native-elements'; 4 | 5 | import Colors from '../constants/Colors'; 6 | import useColorScheme from '../hooks/useColorScheme'; 7 | 8 | export function useThemeColor( 9 | props: { light?: string; dark?: string }, 10 | colorName: keyof typeof Colors.light & keyof typeof Colors.dark 11 | ) { 12 | const theme = useColorScheme(); 13 | const colorFromProps = props[theme]; 14 | 15 | if (colorFromProps) { 16 | return colorFromProps; 17 | } else { 18 | return Colors[theme][colorName]; 19 | } 20 | } 21 | 22 | type ThemeProps = { 23 | lightColor?: string; 24 | darkColor?: string; 25 | }; 26 | 27 | export type TextProps = ThemeProps & DefaultText['props']; 28 | export type ViewProps = ThemeProps & DefaultView['props']; 29 | 30 | export function Text(props: TextProps) { 31 | const { style, lightColor, darkColor, ...otherProps } = props; 32 | const color = useThemeColor({ light: lightColor, dark: darkColor }, 'text'); 33 | 34 | return ; 35 | } 36 | 37 | export function View(props: ViewProps) { 38 | const { style, lightColor, darkColor, ...otherProps } = props; 39 | const backgroundColor = useThemeColor({ light: lightColor, dark: darkColor }, 'background'); 40 | 41 | return ; 42 | } 43 | 44 | export function ScrollView(props: ViewProps) { 45 | const { style, lightColor, darkColor, ...otherProps } = props; 46 | const backgroundColor = useThemeColor({ light: lightColor, dark: darkColor }, 'background'); 47 | 48 | return ; 49 | } 50 | 51 | 52 | export function getColor(property:keyof typeof Colors.light & keyof typeof Colors.dark){ 53 | 54 | //const backgroundColor = useThemeColor({ light: Colors.lightColor, dark: darkColor }, property); 55 | const theme = useColorScheme(); 56 | 57 | 58 | const color = Colors[theme][property]; 59 | return color as string; 60 | } 61 | 62 | export function ActivityIndicator(props: ViewProps) { 63 | const { style, lightColor, darkColor, ...otherProps } = props; 64 | const textColor = useThemeColor({ light: lightColor, dark: darkColor }, 'text'); 65 | 66 | return 67 | } 68 | 69 | 70 | export function ListItem(props: any) { 71 | const { style, lightColor, darkColor, ...otherProps } = props; 72 | const backgroundColor = useThemeColor({ light: lightColor, dark: darkColor }, 'background'); 73 | const color = useThemeColor({ light: lightColor, dark: darkColor }, 'text'); 74 | 75 | 76 | return ; 77 | } 78 | -------------------------------------------------------------------------------- /components/__tests__/StyledText-test.js: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import renderer from 'react-test-renderer'; 3 | 4 | import { MonoText } from '../StyledText'; 5 | 6 | it(`renders correctly`, () => { 7 | const tree = renderer.create(Snapshot test!).toJSON(); 8 | 9 | expect(tree).toMatchSnapshot(); 10 | }); 11 | -------------------------------------------------------------------------------- /config.tsx: -------------------------------------------------------------------------------- 1 | const AppConfig = 2 | 3 | { 4 | title : "Food Classifier" 5 | ,description: ` 6 | 7 | This mobile app was developed by 8 | - [Nidhin Pattaniyil](https://npatta01.github.io) 9 | - [Reshama Shaikh](https://reshamas.github.io/) 10 | 11 | This app lets you submit a photo of food and returns the predicted food category. 🍕 12 | 13 | The model was developed using the [Food-101 dataset](https://www.vision.ee.ethz.ch/datasets_extra/food-101/) and TensorFlow 2.3.0. 14 | 15 | This is a Manning [liveProject](https://liveproject.manning.com). 16 | 17 | Notes: 18 | - Time to inference will vary by user as it is dependent on the hardware. 19 | ` 20 | ,imageSize :224 21 | ,topK:3 22 | ,precision:2 23 | 24 | }; 25 | 26 | 27 | export {AppConfig} -------------------------------------------------------------------------------- /constants/Colors.ts: -------------------------------------------------------------------------------- 1 | const tintColorLight = '#2f95dc'; 2 | const tintColorDark = '#fff'; 3 | 4 | export default { 5 | light: { 6 | text: '#000', 7 | background: '#fff', 8 | tint: tintColorLight, 9 | tabIconDefault: '#ccc', 10 | tabIconSelected: tintColorLight, 11 | }, 12 | dark: { 13 | text: '#fff', 14 | background: '#000', 15 | tint: tintColorDark, 16 | tabIconDefault: '#ccc', 17 | tabIconSelected: tintColorDark, 18 | }, 19 | }; 20 | -------------------------------------------------------------------------------- /constants/Layout.ts: -------------------------------------------------------------------------------- 1 | import { Dimensions } from 'react-native'; 2 | 3 | const width = Dimensions.get('window').width; 4 | const height = Dimensions.get('window').height; 5 | 6 | export default { 7 | window: { 8 | width, 9 | height, 10 | }, 11 | isSmallDevice: width < 375, 12 | }; 13 | -------------------------------------------------------------------------------- /demo/app_in_action.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/demo/app_in_action.gif -------------------------------------------------------------------------------- /hooks/useCachedResources.ts: -------------------------------------------------------------------------------- 1 | import { Ionicons } from '@expo/vector-icons'; 2 | import * as Font from 'expo-font'; 3 | import * as SplashScreen from 'expo-splash-screen'; 4 | import * as React from 'react'; 5 | 6 | export default function useCachedResources() { 7 | const [isLoadingComplete, setLoadingComplete] = React.useState(false); 8 | 9 | // Load any resources or data that we need prior to rendering the app 10 | React.useEffect(() => { 11 | async function loadResourcesAndDataAsync() { 12 | try { 13 | SplashScreen.preventAutoHideAsync(); 14 | 15 | // Load fonts 16 | await Font.loadAsync({ 17 | ...Ionicons.font, 18 | 'space-mono': require('../assets/fonts/SpaceMono-Regular.ttf'), 19 | }); 20 | } catch (e) { 21 | // We might want to provide this error information to an error reporting service 22 | console.warn(e); 23 | } finally { 24 | setLoadingComplete(true); 25 | SplashScreen.hideAsync(); 26 | } 27 | } 28 | 29 | loadResourcesAndDataAsync(); 30 | }, []); 31 | 32 | return isLoadingComplete; 33 | } 34 | -------------------------------------------------------------------------------- /hooks/useColorScheme.ts: -------------------------------------------------------------------------------- 1 | import { ColorSchemeName, useColorScheme as _useColorScheme } from 'react-native'; 2 | 3 | // The useColorScheme value is always either light or dark, but the built-in 4 | // type suggests that it can be null. This will not happen in practice, so this 5 | // makes it a bit easier to work with. 6 | export default function useColorScheme(): NonNullable { 7 | return _useColorScheme() as NonNullable; 8 | } 9 | -------------------------------------------------------------------------------- /hooks/useColorScheme.web.ts: -------------------------------------------------------------------------------- 1 | // useColorScheme from react-native does not support web currently. You can replace 2 | // this with react-native-appearance if you would like theme support on web. 3 | export default function useColorScheme() { 4 | return 'light'; 5 | } -------------------------------------------------------------------------------- /metro.config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @license 3 | * Copyright 2020 Google LLC. All Rights Reserved. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * ============================================================================= 16 | */ 17 | 18 | 19 | const { getDefaultConfig } = require('metro-config'); 20 | module.exports = (async () => { 21 | const defaultConfig = await getDefaultConfig(); 22 | const { assetExts } = defaultConfig.resolver; 23 | return { 24 | resolver: { 25 | assetExts: [...assetExts, 'bin'], 26 | } 27 | }; 28 | })(); -------------------------------------------------------------------------------- /navigation/BottomTabNavigator.tsx: -------------------------------------------------------------------------------- 1 | import { Ionicons } from '@expo/vector-icons'; 2 | import { createBottomTabNavigator } from '@react-navigation/bottom-tabs'; 3 | import { createStackNavigator } from '@react-navigation/stack'; 4 | import * as React from 'react'; 5 | import { Platform } from 'react-native'; 6 | 7 | import Colors from '../constants/Colors'; 8 | import useColorScheme from '../hooks/useColorScheme'; 9 | import HomeScreen from '../screens/HomeScreen'; 10 | import AboutScreen from '../screens/AboutScreen'; 11 | import DebugScreen from '../screens/DebugScreen'; 12 | import { BottomTabParamList, HomeParamList, AboutParamList, DebugParamList } from '../types'; 13 | 14 | const BottomTab = createBottomTabNavigator(); 15 | 16 | export default function BottomTabNavigator() { 17 | const colorScheme = useColorScheme(); 18 | 19 | return ( 20 | 23 | , 28 | }} 29 | /> 30 | , 35 | }} 36 | /> 37 | , 42 | }} 43 | /> 44 | 45 | ); 46 | } 47 | 48 | // You can explore the built-in icon families and icons on the web at: 49 | // https://icons.expo.fyi/ 50 | function TabBarIcon(props: { name: string; color: string }) { 51 | return ; 52 | } 53 | 54 | // Each tab has its own navigation stack, you can read more about this pattern here: 55 | // https://reactnavigation.org/docs/tab-based-navigation#a-stack-navigator-for-each-tab 56 | const HomeStack = createStackNavigator(); 57 | 58 | function HomeNavigator() { 59 | return ( 60 | 61 | 66 | 67 | ); 68 | } 69 | 70 | const AboutTabStack = createStackNavigator(); 71 | 72 | function TabAboutNavigator() { 73 | return ( 74 | 75 | 80 | 81 | ); 82 | } 83 | 84 | const DebugTabStack = createStackNavigator(); 85 | 86 | function TabDebugNavigator() { 87 | return ( 88 | 89 | 94 | 95 | ); 96 | } -------------------------------------------------------------------------------- /navigation/LinkingConfiguration.ts: -------------------------------------------------------------------------------- 1 | import * as Linking from 'expo-linking'; 2 | 3 | export default { 4 | prefixes: [Linking.makeUrl('/')], 5 | config: { 6 | screens: { 7 | Root: { 8 | screens: { 9 | TabHome: { 10 | screens: { 11 | HomeScreen: 'one', 12 | }, 13 | }, 14 | TabAbout: { 15 | screens: { 16 | AboutScreen: 'two', 17 | }, 18 | }, 19 | TabDebug: { 20 | screens: { 21 | DebugScreen: 'three', 22 | }, 23 | }, 24 | }, 25 | }, 26 | NotFound: '*', 27 | }, 28 | }, 29 | }; 30 | -------------------------------------------------------------------------------- /navigation/index.tsx: -------------------------------------------------------------------------------- 1 | import { NavigationContainer, DefaultTheme, DarkTheme } from '@react-navigation/native'; 2 | import { createStackNavigator } from '@react-navigation/stack'; 3 | import * as React from 'react'; 4 | import { ColorSchemeName } from 'react-native'; 5 | 6 | import NotFoundScreen from '../screens/NotFoundScreen'; 7 | import { RootStackParamList } from '../types'; 8 | import BottomTabNavigator from './BottomTabNavigator'; 9 | import LinkingConfiguration from './LinkingConfiguration'; 10 | 11 | // If you are not familiar with React Navigation, we recommend going through the 12 | // "Fundamentals" guide: https://reactnavigation.org/docs/getting-started 13 | export default function Navigation({ colorScheme }: { colorScheme: ColorSchemeName }) { 14 | return ( 15 | 18 | 19 | 20 | ); 21 | } 22 | 23 | // A root stack navigator is often used for displaying modals on top of all other content 24 | // Read more here: https://reactnavigation.org/docs/modal 25 | const Stack = createStackNavigator(); 26 | 27 | function RootNavigator() { 28 | return ( 29 | 30 | 31 | 32 | 33 | ); 34 | } 35 | -------------------------------------------------------------------------------- /notes.md: -------------------------------------------------------------------------------- 1 | https://github.com/yzzhang/machine-learning/blob/master/mobile_apps/mobile_bert/components/TabBarIcon.js 2 | 3 | https://towardsdatascience.com/deep-learning-for-detecting-objects-in-an-image-on-mobile-devices-7d5b2e5621f9 4 | 5 | https://github.com/tensorflow/tfjs/issues/3070 -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "main": "node_modules/expo/AppEntry.js", 3 | "scripts": { 4 | "start": "expo start", 5 | "android": "expo start --android", 6 | "ios": "expo start --ios", 7 | "web": "expo start --web", 8 | "eject": "expo eject", 9 | "test": "jest --watchAll" 10 | }, 11 | "jest": { 12 | "preset": "jest-expo" 13 | }, 14 | "dependencies": { 15 | "@expo/vector-icons": "^12.0.0", 16 | "@react-native-community/async-storage": "~1.12.0", 17 | "@react-native-community/masked-view": "0.1.10", 18 | "@react-navigation/bottom-tabs": "^5.6.1", 19 | "@react-navigation/native": "^5.6.1", 20 | "@react-navigation/stack": "^5.6.2", 21 | "@tensorflow/tfjs": "^2.3.0", 22 | "@tensorflow/tfjs-react-native": "^0.5.0", 23 | "expo": "^40.0.0", 24 | "expo-asset": "~8.2.1", 25 | "expo-camera": "~9.1.0", 26 | "expo-constants": "~9.3.3", 27 | "expo-font": "~8.4.0", 28 | "expo-gl": "~9.2.0", 29 | "expo-image-manipulator": "~8.4.0", 30 | "expo-image-picker": "~9.2.0", 31 | "expo-linking": "~2.0.1", 32 | "expo-permissions": "~10.0.0", 33 | "expo-splash-screen": "~0.8.1", 34 | "expo-status-bar": "~1.0.3", 35 | "expo-web-browser": "~8.6.0", 36 | "jpeg-js": "^0.3.7", 37 | "react": "16.13.1", 38 | "react-dom": "16.13.1", 39 | "react-native": "0.63.4", 40 | "react-native-elements": "^2.3.1", 41 | "react-native-fs": "^2.16.6", 42 | "react-native-gesture-handler": "~1.8.0", 43 | "react-native-markdown-display": "^6.1.6", 44 | "react-native-paper": "^4.0.1", 45 | "react-native-safe-area-context": "3.1.9", 46 | "react-native-screens": "~2.15.2", 47 | "react-native-unimodules": "~0.12.0", 48 | "react-native-web": "~0.13.12" 49 | }, 50 | "devDependencies": { 51 | "@babel/core": "~7.9.0", 52 | "@types/react": "~16.9.35", 53 | "@types/react-native": "~0.63.2", 54 | "babel-preset-expo": "8.3.0", 55 | "jest-expo": "^40.0.0", 56 | "typescript": "~4.0.0" 57 | }, 58 | "private": true 59 | } 60 | -------------------------------------------------------------------------------- /screens/AboutScreen.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { StyleSheet } from 'react-native'; 3 | 4 | import { View } from '../components/Themed'; 5 | import Markdown from 'react-native-markdown-display'; 6 | import {AppConfig} from "../config" 7 | import {getColor} from "../components/Themed" 8 | export default function AboutScreen() { 9 | const color = getColor('text') 10 | return ( 11 | 12 | 13 | 17 | {AppConfig.description} 18 | 19 | 20 | ); 21 | } 22 | 23 | const styles = StyleSheet.create({ 24 | container: { 25 | flex: 1, 26 | alignItems: 'center', 27 | justifyContent: 'space-between', 28 | paddingHorizontal: 10 29 | } 30 | 31 | 32 | }); 33 | -------------------------------------------------------------------------------- /screens/DebugScreen.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { StyleSheet } from 'react-native'; 3 | 4 | import { View, Text} from '../components/Themed'; 5 | import Markdown from 'react-native-markdown-display'; 6 | import {AppConfig} from "../config" 7 | import {getColor} from "../components/Themed" 8 | export default function DebugScreen() { 9 | const color = getColor('text') 10 | const modelClasses = require("../assets/model_tfjs/classes.json") 11 | 12 | return ( 13 | 14 | 15 | Classes 16 | 17 | 18 | 19 | {modelClasses.map(p => { 20 | return ( 21 | 22 | {p} 23 | 24 | ); 25 | })} 26 | 27 | 28 | ); 29 | } 30 | 31 | const styles = StyleSheet.create({ 32 | container: { 33 | flex: 1, 34 | paddingHorizontal: 10 35 | } 36 | 37 | 38 | }); 39 | -------------------------------------------------------------------------------- /screens/HomeScreen.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | 3 | //import { Text, View } from '../components/Themed'; 4 | 5 | import * as Permissions from 'expo-permissions' 6 | import * as tf from '@tensorflow/tfjs'; 7 | import * as ImageManipulator from 'expo-image-manipulator'; 8 | 9 | 10 | import { 11 | Image, 12 | StyleSheet, 13 | } from 'react-native'; 14 | import {AppConfig} from "../config" 15 | 16 | import {Text ,View,getColor,ActivityIndicator,ScrollView} from '../components/Themed' 17 | 18 | import {Icon, ListItem} from 'react-native-elements'; 19 | 20 | 21 | import * as ImagePicker from 'expo-image-picker'; 22 | import { ModelService, IModelPredictionResponse,IModelPredictionTiming,ModelPrediction } from '../components/ModelService'; 23 | 24 | 25 | type State = { 26 | image: ImageManipulator.ImageResult; 27 | loading:boolean; 28 | isTfReady: boolean; 29 | isModelReady: boolean; 30 | predictions: ModelPrediction[]|null; 31 | error:string|null; 32 | timing:IModelPredictionTiming|null; 33 | }; 34 | 35 | export default class HomeScreen extends React.Component<{},State> { 36 | static navigationOptions = { 37 | header: null, 38 | }; 39 | 40 | state:State = { 41 | image: {}, 42 | loading: false, 43 | isTfReady: false, 44 | isModelReady: false, 45 | predictions: null, 46 | error:null, 47 | timing:null 48 | } 49 | 50 | modelService!:ModelService; 51 | 52 | async componentDidMount() { 53 | this.setState({ loading: true }); 54 | this.modelService = await ModelService.create(AppConfig.imageSize); 55 | this.setState({ isTfReady: true,isModelReady: true,loading: false }); 56 | } 57 | 58 | render() { 59 | 60 | const modelLoadingStatus = this.state.isModelReady ? "✅" : "❓"; 61 | return ( 62 | 63 | 64 | 65 | 66 | {AppConfig.title} 67 | 68 | 69 | 70 | 71 | Model Status: {modelLoadingStatus} 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | {this.renderPredictions()} 88 | 89 | 90 | 91 | 92 | ); 93 | } 94 | 95 | 96 | renderPredictions() { 97 | if (this.state.loading) { 98 | return 99 | } 100 | let predictions= this.state.predictions || []; 101 | 102 | if (predictions.length > 0) { 103 | return ( 104 | 105 | Predictions 106 | 107 | { 108 | predictions.map((item, index) => ( 109 | 110 | 111 | 112 | {item.className} 113 | {`prob: ${item.probability.toFixed(AppConfig.precision)}`} 114 | 115 | 116 | 117 | )) 118 | } 119 | 120 | 121 | 122 | Timing (ms) 123 | 124 | total time: {this.state.timing?.totalTime} 125 | loading time: {this.state.timing?.imageLoadingTime} 126 | preprocessing time: {this.state.timing?.imagePreprocessing} 127 | prediction time: {this.state.timing?.imagePrediction} 128 | decode time: {this.state.timing?.imageDecodePrediction} 129 | 130 | 131 | 132 | 133 | ) 134 | } else { 135 | return null 136 | } 137 | } 138 | 139 | 140 | _verifyPermissions = async () => { 141 | console.log("Verifying Permissions"); 142 | const { status, expires, permissions } = await Permissions.getAsync(Permissions.CAMERA, Permissions.CAMERA_ROLL); 143 | 144 | if (status !== 'granted') { 145 | const { status, permissions } = await Permissions.askAsync(Permissions.CAMERA, Permissions.CAMERA_ROLL) 146 | 147 | if (status === 'granted') { 148 | console.log("Permissions granted"); 149 | return true 150 | } else { 151 | alert('Hey! You have not enabled selected permissions'); 152 | return false 153 | } 154 | 155 | }else{ 156 | return true; 157 | } 158 | }; 159 | 160 | _pickImageFromLibrary = async () => { 161 | const status = await this._verifyPermissions(); 162 | 163 | try { 164 | let response = await ImagePicker.launchImageLibraryAsync({ 165 | mediaTypes: ImagePicker.MediaTypeOptions.Images, 166 | allowsEditing: true, 167 | aspect: [4, 3] 168 | }) 169 | 170 | if (!response.cancelled) { 171 | //const source = { uri: response.uri } 172 | 173 | //this.setState({ image: source }) 174 | this._classifyImage(response.uri) 175 | } 176 | } catch (error) { 177 | console.log(error) 178 | } 179 | 180 | }; 181 | 182 | _pickImageFromCamera = async () => { 183 | const status = await this._verifyPermissions(); 184 | 185 | try { 186 | 187 | let response = await ImagePicker.launchCameraAsync({ 188 | mediaTypes: ImagePicker.MediaTypeOptions.Images, 189 | allowsEditing: true, 190 | aspect: [4, 3] 191 | }); 192 | 193 | if (!response.cancelled) { 194 | //const source = { uri: response.uri } 195 | 196 | this._classifyImage(response.uri) 197 | } 198 | } catch (error) { 199 | console.log(error) 200 | } 201 | 202 | }; 203 | 204 | _classifyImage = async (imageUri:string) => { 205 | try { 206 | const res:ImageManipulator.ImageResult = await ImageManipulator.manipulateAsync(imageUri, 207 | [{ resize: { width:AppConfig.imageSize, height:AppConfig.imageSize }}], 208 | { compress: 1, format: ImageManipulator.SaveFormat.JPEG,base64:true } 209 | ); 210 | 211 | this.setState({ image: res}) 212 | console.log('numTensors (before prediction): ' + tf.memory().numTensors); 213 | this.setState({ predictions: [] ,error:null , loading:true }) 214 | 215 | const predictionResponse = await this.modelService.classifyImage(res); 216 | 217 | 218 | if (predictionResponse.error){ 219 | this.setState({ error: predictionResponse.error , loading:false}) 220 | }else{ 221 | const predictions = predictionResponse.predictions || null; 222 | this.setState({ predictions: predictions, timing:predictionResponse.timing, loading:false}) 223 | } 224 | 225 | 226 | //tf.dispose(predictions); 227 | console.log('numTensors (after prediction): ' + tf.memory().numTensors); 228 | 229 | } catch (error) { 230 | console.log('Exception Error: ', error) 231 | } 232 | } 233 | 234 | } 235 | 236 | 237 | const styles = StyleSheet.create({ 238 | container: { 239 | paddingTop: 5, 240 | flex: 1, 241 | }, 242 | 243 | contentContainer: { 244 | alignItems: 'center', 245 | justifyContent: 'center', 246 | 247 | }, 248 | titleContainer: { 249 | alignItems: 'center', 250 | marginTop: 10, 251 | //flex: 2, 252 | justifyContent: 'center', 253 | }, 254 | actionsContainer: { 255 | alignItems: 'center', 256 | marginTop: 5, 257 | marginBottom: 5, 258 | //flex: 1, 259 | }, 260 | imageContainer: { 261 | alignItems: 'center', 262 | }, 263 | callToActionContainer: { 264 | flexDirection: "row" 265 | }, 266 | 267 | feedBackActionsContainer: { 268 | flexDirection: "row" 269 | }, 270 | 271 | predictionsContainer: { 272 | padding: 10, 273 | justifyContent: 'center', 274 | }, 275 | 276 | predictionsContentContainer: { 277 | padding: 10, 278 | }, 279 | predictionRow: { 280 | flexDirection: "row", 281 | }, 282 | predictionRowCategory: { 283 | justifyContent: "space-between" 284 | }, 285 | predictionRowLabel: { 286 | justifyContent: "space-between" 287 | } 288 | }); 289 | -------------------------------------------------------------------------------- /screens/NotFoundScreen.tsx: -------------------------------------------------------------------------------- 1 | import { StackScreenProps } from '@react-navigation/stack'; 2 | import * as React from 'react'; 3 | import { StyleSheet, Text, TouchableOpacity, View } from 'react-native'; 4 | 5 | import { RootStackParamList } from '../types'; 6 | 7 | export default function NotFoundScreen({ 8 | navigation, 9 | }: StackScreenProps) { 10 | return ( 11 | 12 | This screen doesn't exist. 13 | navigation.replace('Root')} style={styles.link}> 14 | Go to home screen! 15 | 16 | 17 | ); 18 | } 19 | 20 | const styles = StyleSheet.create({ 21 | container: { 22 | flex: 1, 23 | backgroundColor: '#fff', 24 | alignItems: 'center', 25 | justifyContent: 'center', 26 | padding: 20, 27 | }, 28 | title: { 29 | fontSize: 20, 30 | fontWeight: 'bold', 31 | }, 32 | link: { 33 | marginTop: 15, 34 | paddingVertical: 15, 35 | }, 36 | linkText: { 37 | fontSize: 14, 38 | color: '#2e78b7', 39 | }, 40 | }); 41 | -------------------------------------------------------------------------------- /test.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowSyntheticDefaultImports": true, 4 | "jsx": "react-native", 5 | "lib": ["dom", "esnext"], 6 | "moduleResolution": "node", 7 | "noEmit": true, 8 | "skipLibCheck": true, 9 | "resolveJsonModule": true, 10 | "strict": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /types.tsx: -------------------------------------------------------------------------------- 1 | export type RootStackParamList = { 2 | Root: undefined; 3 | NotFound: undefined; 4 | }; 5 | 6 | export type BottomTabParamList = { 7 | Home: undefined; 8 | About: undefined; 9 | Debug: undefined; 10 | }; 11 | 12 | export type HomeParamList = { 13 | HomeScreen: undefined; 14 | }; 15 | 16 | export type AboutParamList = { 17 | AboutScreen: undefined; 18 | }; 19 | 20 | export type DebugParamList = { 21 | DebugScreen: undefined; 22 | }; 23 | --------------------------------------------------------------------------------