├── .expo-shared
└── assets.json
├── .gitignore
├── App.tsx
├── README.md
├── app.json
├── assets
├── fonts
│ └── SpaceMono-Regular.ttf
├── images
│ ├── favicon.png
│ ├── icon.png
│ └── splash.png
└── model_tfjs
│ └── .gitkeep
├── babel.config.js
├── components
├── ModelService.tsx
├── StyledText.tsx
├── Themed.tsx
└── __tests__
│ └── StyledText-test.js
├── config.tsx
├── constants
├── Colors.ts
└── Layout.ts
├── demo
└── app_in_action.gif
├── hooks
├── useCachedResources.ts
├── useColorScheme.ts
└── useColorScheme.web.ts
├── metro.config.js
├── navigation
├── BottomTabNavigator.tsx
├── LinkingConfiguration.ts
└── index.tsx
├── notes.md
├── package.json
├── screens
├── AboutScreen.tsx
├── DebugScreen.tsx
├── HomeScreen.tsx
└── NotFoundScreen.tsx
├── test.md
├── tsconfig.json
├── types.tsx
├── yarn-error.log
└── yarn.lock
/.expo-shared/assets.json:
--------------------------------------------------------------------------------
1 | {
2 | "e997a5256149a4b76e6bfd6cbf519c5e5a0f1d278a3d8fa1253022b03c90473b": true,
3 | "af683c96e0ffd2cf81287651c9433fa44debc1220ca7cb431fe482747f34a505": true,
4 | "12bb71342c6255bbf50437ec8f4441c083f47cdb74bd89160c15e4f43e52a1cb": true,
5 | "40b842e832070c58deac6aa9e08fa459302ee3f9da492c7e77d93d2fbf4a56fd": true
6 | }
7 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/**/*
2 | .expo/*
3 | npm-debug.*
4 | *.jks
5 | *.p8
6 | *.p12
7 | *.key
8 | *.mobileprovision
9 | *.orig.*
10 | web-build/
11 |
12 | # macOS
13 | .DS_Store
14 | .vscode
15 | assets/model_tfjs/*
16 |
17 | !assets/model_tfjs/.gitkeep
--------------------------------------------------------------------------------
/App.tsx:
--------------------------------------------------------------------------------
1 | import { StatusBar } from 'expo-status-bar';
2 | import React from 'react';
3 | import { SafeAreaProvider } from 'react-native-safe-area-context';
4 |
5 | import useCachedResources from './hooks/useCachedResources';
6 | import useColorScheme from './hooks/useColorScheme';
7 | import Navigation from './navigation';
8 | import { Button, ThemeProvider } from 'react-native-elements';
9 |
10 |
11 | export default function App() {
12 | const isLoadingComplete = useCachedResources();
13 | const colorScheme = useColorScheme();
14 |
15 | if (!isLoadingComplete) {
16 | return null;
17 | } else {
18 | return (
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 | );
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # deploying-mobile-app
2 |
3 |
4 | ## About
5 |
6 | This is a sample template repo showing how to use tensorflow.js with react native.
7 |
8 | Here is the app in action.
9 |
10 | 
11 |
12 |
13 |
14 | ## Assumptions
15 |
16 | In order to use this repo, you need to have a tfjs model. You can get a smaple artifact from this repo's release.
17 |
18 | Please download it from this repos's release and place the contents in `assets/model_tfjs`
19 |
20 | ```
21 | (base) deploying-mobile-app % tree assets/model_tfjs
22 | assets/model_tfjs
23 | ├── classes.json
24 | ├── classes.txt
25 | ├── group1-shard1of1.bin
26 | └── model.json
27 | ```
28 |
29 | If you want to bring in your own model, please use a command like below to convert a keras h5 artifact.
30 |
31 | The below command converts `artifacts/model_tf_keras.h5` to tfjs and saves it to `artifacts/model_tfjs`.
32 |
33 | The model is also 16 bit quanitzed.
34 | The model is converted to `tfjs_graph_model` which is an optimized version of the graph.
35 | The model is broken into 100MB shards.
36 |
37 | ```
38 | pip install tensorflowjs==2.3.0
39 |
40 | tensorflowjs_converter \
41 | --input_format=keras \
42 | --output_format=tfjs_graph_model \
43 | --split_weights_by_layer \
44 | --weight_shard_size_bytes=99999999 \
45 | --quantize_float16=* \
46 | artifacts/model_tf_keras.h5 artifacts/model_tfjs
47 | ```
48 |
49 |
50 |
51 | ## Running
52 |
53 |
54 | ```
55 | yarn global add expo-cli
56 |
57 | # you can open iOS, Android, or web from here, or run them directly with the commands below.
58 | yarn start
59 |
60 | # you can open in android simulator or device driectly
61 | yarn android
62 |
63 | # you can open in ios simulator or device directly
64 | yarn ios
65 |
66 |
67 | ```
68 |
69 |
70 |
71 |
72 | ## Future
73 |
74 | In the future, it is possible that packages may be out of date.
75 |
76 | The below upgrade will upgrade expo and its dependencies
77 | ```
78 | expo upgrade
79 | ```
80 |
81 | The below command command can be used to interactively update dependencies.
82 |
83 | ```
84 | yarn upgrade-interactive
85 | ```
86 |
87 | ## Other
88 |
89 | If you are interested in a guided exploration of this content, consider Manning's Live project [Deploying a Deep Learning Model on Web and Mobile Applications
90 | ](https://www.manning.com/liveproject/deploying-a-deep-learning-model-on-web-and-mobile-applications)
91 |
--------------------------------------------------------------------------------
/app.json:
--------------------------------------------------------------------------------
1 | {
2 | "expo": {
3 | "name": "mobile-app",
4 | "slug": "mobile-app",
5 | "version": "1.0.0",
6 | "orientation": "portrait",
7 | "icon": "./assets/images/icon.png",
8 | "scheme": "myapp",
9 | "userInterfaceStyle": "automatic",
10 | "splash": {
11 | "image": "./assets/images/splash.png",
12 | "resizeMode": "contain",
13 | "backgroundColor": "#ffffff"
14 | },
15 | "updates": {
16 | "fallbackToCacheTimeout": 0
17 | },
18 | "assetBundlePatterns": [
19 | "**/*"
20 | ],
21 | "ios": {
22 | "supportsTablet": true
23 | },
24 | "web": {
25 | "favicon": "./assets/images/favicon.png"
26 | }
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/assets/fonts/SpaceMono-Regular.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/assets/fonts/SpaceMono-Regular.ttf
--------------------------------------------------------------------------------
/assets/images/favicon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/assets/images/favicon.png
--------------------------------------------------------------------------------
/assets/images/icon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/assets/images/icon.png
--------------------------------------------------------------------------------
/assets/images/splash.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/assets/images/splash.png
--------------------------------------------------------------------------------
/assets/model_tfjs/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/assets/model_tfjs/.gitkeep
--------------------------------------------------------------------------------
/babel.config.js:
--------------------------------------------------------------------------------
1 | module.exports = function(api) {
2 | api.cache(true);
3 | return {
4 | presets: ['babel-preset-expo'],
5 | env: {
6 | production: {
7 | plugins: ['react-native-paper/babel'],
8 | },
9 | }
10 | };
11 | };
12 |
--------------------------------------------------------------------------------
/components/ModelService.tsx:
--------------------------------------------------------------------------------
1 | import * as tf from '@tensorflow/tfjs';
2 | import * as FileSystem from 'expo-file-system'
3 | import { fetch ,asyncStorageIO,bundleResourceIO,decodeJpeg} from '@tensorflow/tfjs-react-native'
4 | import {Image} from 'react-native';
5 | import * as ImageManipulator from 'expo-image-manipulator';
6 | import {AppConfig} from "../config"
7 |
8 | export interface ModelPrediction {
9 | className:string;
10 | probability:number;
11 | }
12 |
13 | export interface IModelPredictionTiming {
14 | totalTime:number;
15 | imageLoadingTime:number;
16 | imagePreprocessing:number;
17 | imagePrediction:number;
18 | imageDecodePrediction:number;
19 | }
20 |
21 | export interface IModelPredictionResponse {
22 | predictions?:ModelPrediction[] | null
23 | timing?:IModelPredictionTiming | null
24 | error?:string | null
25 | }
26 |
27 | const imageToTensor = (rawImageData:Uint8Array)=> {
28 | return decodeJpeg(rawImageData);
29 | }
30 |
31 |
32 | const fetchImage = async (image:ImageManipulator.ImageResult) => {
33 | let imgB64:string;
34 | if(image.base64){
35 | imgB64=image.base64
36 | }else{
37 | const imageAssetPath = Image.resolveAssetSource(image)
38 | console.log(imageAssetPath.uri);
39 |
40 | imgB64 = await FileSystem.readAsStringAsync(imageAssetPath.uri, {
41 | encoding: FileSystem.EncodingType.Base64,
42 | });
43 | }
44 |
45 | const imgBuffer = tf.util.encodeString(imgB64, 'base64').buffer;
46 | const rawImageData = new Uint8Array(imgBuffer)
47 |
48 | return rawImageData;
49 | }
50 | const preprocessImage = (img:tf.Tensor3D,imageSize:number) =>{
51 | // https://github.com/keras-team/keras-applications/blob/master/keras_applications/imagenet_utils.py#L43
52 |
53 | let imageTensor = img.resizeBilinear([imageSize, imageSize]).toFloat();
54 |
55 | const offset = tf.scalar(127.5);
56 | const normalized = imageTensor.sub(offset).div(offset);
57 | const preProcessedImage = normalized.reshape([1, imageSize, imageSize, 3]);
58 | return preProcessedImage;
59 |
60 | }
61 |
62 | const decodePredictions = (predictions:tf.Tensor, classes:String[],topK=3) =>{
63 | const {values, indices} = predictions.topk(topK);
64 | const topKValues = values.dataSync();
65 | const topKIndices = indices.dataSync();
66 |
67 | const topClassesAndProbs:ModelPrediction[] = [];
68 | for (let i = 0; i < topKIndices.length; i++) {
69 | topClassesAndProbs.push({
70 | className: classes[topKIndices[i]],
71 | probability: topKValues[i]
72 | } as ModelPrediction);
73 | }
74 | return topClassesAndProbs;
75 | }
76 |
77 |
78 | export class ModelService {
79 |
80 | private model:tf.GraphModel;
81 | private model_classes: String[];
82 | private imageSize:number;
83 | private static instance: ModelService;
84 |
85 | constructor(imageSize:number,model:tf.GraphModel, model_classes: String[] ){
86 | this.imageSize=imageSize;
87 | this.model = model;
88 | this.model_classes=model_classes;
89 | }
90 |
91 |
92 | static async create(imageSize:number) {
93 | if (!ModelService.instance){
94 | await tf.ready();
95 | const modelJSON = require('../assets/model_tfjs/model.json');
96 | const modelWeights = require('../assets/model_tfjs/group1-shard1of1.bin');
97 | const model_classes = require("../assets/model_tfjs/classes.json")
98 |
99 | const model = await tf.loadGraphModel(bundleResourceIO(modelJSON, modelWeights));
100 | model.predict(tf.zeros([1, imageSize, imageSize, 3]));
101 |
102 | ModelService.instance = new ModelService(imageSize,model,model_classes);
103 | }
104 |
105 | return ModelService.instance;
106 |
107 | }
108 |
109 | async classifyImage(image:ImageManipulator.ImageResult):Promise{
110 | const predictionResponse = {timing:null,predictions:null,error:null} as IModelPredictionResponse;
111 | try {
112 | console.log(`Classifying Image: Start `)
113 |
114 | let imgBuffer:Uint8Array = await fetchImage(image);
115 | const timeStart = new Date().getTime()
116 | console.log(`Backend: ${tf.getBackend()} `)
117 | tf.tidy(()=>{
118 | console.log(`Fetching Image: Start `)
119 |
120 | const imageTensor:tf.Tensor3D = imageToTensor(imgBuffer);
121 |
122 |
123 | console.log(`Fetching Image: Done `)
124 | const timeLoadDone = new Date().getTime()
125 |
126 | console.log("Preprocessing image: Start")
127 |
128 | const preProcessedImage = preprocessImage(imageTensor,this.imageSize);
129 |
130 | console.log("Preprocessing image: Done")
131 | const timePrepocessDone = new Date().getTime()
132 |
133 | console.log("Prediction: Start")
134 | const predictionsTensor:tf.Tensor = this.model.predict(preProcessedImage) as tf.Tensor;
135 |
136 | console.log("Prediction: Done")
137 | const timePredictionDone = new Date().getTime()
138 |
139 | console.log("Post Processing: Start")
140 |
141 | // post processing
142 | predictionResponse.predictions = decodePredictions(predictionsTensor,this.model_classes,AppConfig.topK);
143 |
144 |
145 | //tf.dispose(imageTensor);
146 | //tf.dispose(preProcessedImage);
147 | //tf.dispose(predictions);
148 |
149 | console.log("Post Processing: Done")
150 |
151 | const timeEnd = new Date().getTime()
152 |
153 | const timing:IModelPredictionTiming = {
154 | totalTime: timeEnd-timeStart,
155 | imageLoadingTime : timeLoadDone-timeStart,
156 | imagePreprocessing : timePrepocessDone-timeLoadDone,
157 | imagePrediction : timePredictionDone-timePrepocessDone ,
158 | imageDecodePrediction : timeEnd-timePredictionDone
159 | } as IModelPredictionTiming;
160 | predictionResponse.timing = timing;
161 |
162 | });
163 |
164 |
165 | console.log(`Classifying Image: End `);
166 |
167 | console.log(`Response: ${JSON.stringify(predictionResponse ,null, 2 ) } `);
168 | return predictionResponse as IModelPredictionResponse
169 |
170 | } catch (error) {
171 | console.log('Exception Error: ', error)
172 | return {error}
173 | }
174 | }
175 | }
176 |
177 |
--------------------------------------------------------------------------------
/components/StyledText.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 |
3 | import { Text, TextProps } from './Themed';
4 |
5 | export function MonoText(props: TextProps) {
6 | return ;
7 | }
8 |
--------------------------------------------------------------------------------
/components/Themed.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import { View as DefaultView ,ScrollView as DefaultScrollView, ActivityIndicator as DefaultActivityIndicator } from 'react-native';
3 | import {Text as DefaultText, ListItem as DefaultListItem } from 'react-native-elements';
4 |
5 | import Colors from '../constants/Colors';
6 | import useColorScheme from '../hooks/useColorScheme';
7 |
8 | export function useThemeColor(
9 | props: { light?: string; dark?: string },
10 | colorName: keyof typeof Colors.light & keyof typeof Colors.dark
11 | ) {
12 | const theme = useColorScheme();
13 | const colorFromProps = props[theme];
14 |
15 | if (colorFromProps) {
16 | return colorFromProps;
17 | } else {
18 | return Colors[theme][colorName];
19 | }
20 | }
21 |
22 | type ThemeProps = {
23 | lightColor?: string;
24 | darkColor?: string;
25 | };
26 |
27 | export type TextProps = ThemeProps & DefaultText['props'];
28 | export type ViewProps = ThemeProps & DefaultView['props'];
29 |
30 | export function Text(props: TextProps) {
31 | const { style, lightColor, darkColor, ...otherProps } = props;
32 | const color = useThemeColor({ light: lightColor, dark: darkColor }, 'text');
33 |
34 | return ;
35 | }
36 |
37 | export function View(props: ViewProps) {
38 | const { style, lightColor, darkColor, ...otherProps } = props;
39 | const backgroundColor = useThemeColor({ light: lightColor, dark: darkColor }, 'background');
40 |
41 | return ;
42 | }
43 |
44 | export function ScrollView(props: ViewProps) {
45 | const { style, lightColor, darkColor, ...otherProps } = props;
46 | const backgroundColor = useThemeColor({ light: lightColor, dark: darkColor }, 'background');
47 |
48 | return ;
49 | }
50 |
51 |
52 | export function getColor(property:keyof typeof Colors.light & keyof typeof Colors.dark){
53 |
54 | //const backgroundColor = useThemeColor({ light: Colors.lightColor, dark: darkColor }, property);
55 | const theme = useColorScheme();
56 |
57 |
58 | const color = Colors[theme][property];
59 | return color as string;
60 | }
61 |
62 | export function ActivityIndicator(props: ViewProps) {
63 | const { style, lightColor, darkColor, ...otherProps } = props;
64 | const textColor = useThemeColor({ light: lightColor, dark: darkColor }, 'text');
65 |
66 | return
67 | }
68 |
69 |
70 | export function ListItem(props: any) {
71 | const { style, lightColor, darkColor, ...otherProps } = props;
72 | const backgroundColor = useThemeColor({ light: lightColor, dark: darkColor }, 'background');
73 | const color = useThemeColor({ light: lightColor, dark: darkColor }, 'text');
74 |
75 |
76 | return ;
77 | }
78 |
--------------------------------------------------------------------------------
/components/__tests__/StyledText-test.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import renderer from 'react-test-renderer';
3 |
4 | import { MonoText } from '../StyledText';
5 |
6 | it(`renders correctly`, () => {
7 | const tree = renderer.create(Snapshot test!).toJSON();
8 |
9 | expect(tree).toMatchSnapshot();
10 | });
11 |
--------------------------------------------------------------------------------
/config.tsx:
--------------------------------------------------------------------------------
1 | const AppConfig =
2 |
3 | {
4 | title : "Food Classifier"
5 | ,description: `
6 |
7 | This mobile app was developed by
8 | - [Nidhin Pattaniyil](https://npatta01.github.io)
9 | - [Reshama Shaikh](https://reshamas.github.io/)
10 |
11 | This app lets you submit a photo of food and returns the predicted food category. 🍕
12 |
13 | The model was developed using the [Food-101 dataset](https://www.vision.ee.ethz.ch/datasets_extra/food-101/) and TensorFlow 2.3.0.
14 |
15 | This is a Manning [liveProject](https://liveproject.manning.com).
16 |
17 | Notes:
18 | - Time to inference will vary by user as it is dependent on the hardware.
19 | `
20 | ,imageSize :224
21 | ,topK:3
22 | ,precision:2
23 |
24 | };
25 |
26 |
27 | export {AppConfig}
--------------------------------------------------------------------------------
/constants/Colors.ts:
--------------------------------------------------------------------------------
1 | const tintColorLight = '#2f95dc';
2 | const tintColorDark = '#fff';
3 |
4 | export default {
5 | light: {
6 | text: '#000',
7 | background: '#fff',
8 | tint: tintColorLight,
9 | tabIconDefault: '#ccc',
10 | tabIconSelected: tintColorLight,
11 | },
12 | dark: {
13 | text: '#fff',
14 | background: '#000',
15 | tint: tintColorDark,
16 | tabIconDefault: '#ccc',
17 | tabIconSelected: tintColorDark,
18 | },
19 | };
20 |
--------------------------------------------------------------------------------
/constants/Layout.ts:
--------------------------------------------------------------------------------
1 | import { Dimensions } from 'react-native';
2 |
3 | const width = Dimensions.get('window').width;
4 | const height = Dimensions.get('window').height;
5 |
6 | export default {
7 | window: {
8 | width,
9 | height,
10 | },
11 | isSmallDevice: width < 375,
12 | };
13 |
--------------------------------------------------------------------------------
/demo/app_in_action.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/reshamas/deploying-mobile-app/82aca213e2e76d6a9f08d50efb66fa381abbf846/demo/app_in_action.gif
--------------------------------------------------------------------------------
/hooks/useCachedResources.ts:
--------------------------------------------------------------------------------
1 | import { Ionicons } from '@expo/vector-icons';
2 | import * as Font from 'expo-font';
3 | import * as SplashScreen from 'expo-splash-screen';
4 | import * as React from 'react';
5 |
6 | export default function useCachedResources() {
7 | const [isLoadingComplete, setLoadingComplete] = React.useState(false);
8 |
9 | // Load any resources or data that we need prior to rendering the app
10 | React.useEffect(() => {
11 | async function loadResourcesAndDataAsync() {
12 | try {
13 | SplashScreen.preventAutoHideAsync();
14 |
15 | // Load fonts
16 | await Font.loadAsync({
17 | ...Ionicons.font,
18 | 'space-mono': require('../assets/fonts/SpaceMono-Regular.ttf'),
19 | });
20 | } catch (e) {
21 | // We might want to provide this error information to an error reporting service
22 | console.warn(e);
23 | } finally {
24 | setLoadingComplete(true);
25 | SplashScreen.hideAsync();
26 | }
27 | }
28 |
29 | loadResourcesAndDataAsync();
30 | }, []);
31 |
32 | return isLoadingComplete;
33 | }
34 |
--------------------------------------------------------------------------------
/hooks/useColorScheme.ts:
--------------------------------------------------------------------------------
1 | import { ColorSchemeName, useColorScheme as _useColorScheme } from 'react-native';
2 |
3 | // The useColorScheme value is always either light or dark, but the built-in
4 | // type suggests that it can be null. This will not happen in practice, so this
5 | // makes it a bit easier to work with.
6 | export default function useColorScheme(): NonNullable {
7 | return _useColorScheme() as NonNullable;
8 | }
9 |
--------------------------------------------------------------------------------
/hooks/useColorScheme.web.ts:
--------------------------------------------------------------------------------
1 | // useColorScheme from react-native does not support web currently. You can replace
2 | // this with react-native-appearance if you would like theme support on web.
3 | export default function useColorScheme() {
4 | return 'light';
5 | }
--------------------------------------------------------------------------------
/metro.config.js:
--------------------------------------------------------------------------------
1 | /**
2 | * @license
3 | * Copyright 2020 Google LLC. All Rights Reserved.
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | * =============================================================================
16 | */
17 |
18 |
19 | const { getDefaultConfig } = require('metro-config');
20 | module.exports = (async () => {
21 | const defaultConfig = await getDefaultConfig();
22 | const { assetExts } = defaultConfig.resolver;
23 | return {
24 | resolver: {
25 | assetExts: [...assetExts, 'bin'],
26 | }
27 | };
28 | })();
--------------------------------------------------------------------------------
/navigation/BottomTabNavigator.tsx:
--------------------------------------------------------------------------------
1 | import { Ionicons } from '@expo/vector-icons';
2 | import { createBottomTabNavigator } from '@react-navigation/bottom-tabs';
3 | import { createStackNavigator } from '@react-navigation/stack';
4 | import * as React from 'react';
5 | import { Platform } from 'react-native';
6 |
7 | import Colors from '../constants/Colors';
8 | import useColorScheme from '../hooks/useColorScheme';
9 | import HomeScreen from '../screens/HomeScreen';
10 | import AboutScreen from '../screens/AboutScreen';
11 | import DebugScreen from '../screens/DebugScreen';
12 | import { BottomTabParamList, HomeParamList, AboutParamList, DebugParamList } from '../types';
13 |
14 | const BottomTab = createBottomTabNavigator();
15 |
16 | export default function BottomTabNavigator() {
17 | const colorScheme = useColorScheme();
18 |
19 | return (
20 |
23 | ,
28 | }}
29 | />
30 | ,
35 | }}
36 | />
37 | ,
42 | }}
43 | />
44 |
45 | );
46 | }
47 |
48 | // You can explore the built-in icon families and icons on the web at:
49 | // https://icons.expo.fyi/
50 | function TabBarIcon(props: { name: string; color: string }) {
51 | return ;
52 | }
53 |
54 | // Each tab has its own navigation stack, you can read more about this pattern here:
55 | // https://reactnavigation.org/docs/tab-based-navigation#a-stack-navigator-for-each-tab
56 | const HomeStack = createStackNavigator();
57 |
58 | function HomeNavigator() {
59 | return (
60 |
61 |
66 |
67 | );
68 | }
69 |
70 | const AboutTabStack = createStackNavigator();
71 |
72 | function TabAboutNavigator() {
73 | return (
74 |
75 |
80 |
81 | );
82 | }
83 |
84 | const DebugTabStack = createStackNavigator();
85 |
86 | function TabDebugNavigator() {
87 | return (
88 |
89 |
94 |
95 | );
96 | }
--------------------------------------------------------------------------------
/navigation/LinkingConfiguration.ts:
--------------------------------------------------------------------------------
1 | import * as Linking from 'expo-linking';
2 |
3 | export default {
4 | prefixes: [Linking.makeUrl('/')],
5 | config: {
6 | screens: {
7 | Root: {
8 | screens: {
9 | TabHome: {
10 | screens: {
11 | HomeScreen: 'one',
12 | },
13 | },
14 | TabAbout: {
15 | screens: {
16 | AboutScreen: 'two',
17 | },
18 | },
19 | TabDebug: {
20 | screens: {
21 | DebugScreen: 'three',
22 | },
23 | },
24 | },
25 | },
26 | NotFound: '*',
27 | },
28 | },
29 | };
30 |
--------------------------------------------------------------------------------
/navigation/index.tsx:
--------------------------------------------------------------------------------
1 | import { NavigationContainer, DefaultTheme, DarkTheme } from '@react-navigation/native';
2 | import { createStackNavigator } from '@react-navigation/stack';
3 | import * as React from 'react';
4 | import { ColorSchemeName } from 'react-native';
5 |
6 | import NotFoundScreen from '../screens/NotFoundScreen';
7 | import { RootStackParamList } from '../types';
8 | import BottomTabNavigator from './BottomTabNavigator';
9 | import LinkingConfiguration from './LinkingConfiguration';
10 |
11 | // If you are not familiar with React Navigation, we recommend going through the
12 | // "Fundamentals" guide: https://reactnavigation.org/docs/getting-started
13 | export default function Navigation({ colorScheme }: { colorScheme: ColorSchemeName }) {
14 | return (
15 |
18 |
19 |
20 | );
21 | }
22 |
23 | // A root stack navigator is often used for displaying modals on top of all other content
24 | // Read more here: https://reactnavigation.org/docs/modal
25 | const Stack = createStackNavigator();
26 |
27 | function RootNavigator() {
28 | return (
29 |
30 |
31 |
32 |
33 | );
34 | }
35 |
--------------------------------------------------------------------------------
/notes.md:
--------------------------------------------------------------------------------
1 | https://github.com/yzzhang/machine-learning/blob/master/mobile_apps/mobile_bert/components/TabBarIcon.js
2 |
3 | https://towardsdatascience.com/deep-learning-for-detecting-objects-in-an-image-on-mobile-devices-7d5b2e5621f9
4 |
5 | https://github.com/tensorflow/tfjs/issues/3070
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "main": "node_modules/expo/AppEntry.js",
3 | "scripts": {
4 | "start": "expo start",
5 | "android": "expo start --android",
6 | "ios": "expo start --ios",
7 | "web": "expo start --web",
8 | "eject": "expo eject",
9 | "test": "jest --watchAll"
10 | },
11 | "jest": {
12 | "preset": "jest-expo"
13 | },
14 | "dependencies": {
15 | "@expo/vector-icons": "^12.0.0",
16 | "@react-native-community/async-storage": "~1.12.0",
17 | "@react-native-community/masked-view": "0.1.10",
18 | "@react-navigation/bottom-tabs": "^5.6.1",
19 | "@react-navigation/native": "^5.6.1",
20 | "@react-navigation/stack": "^5.6.2",
21 | "@tensorflow/tfjs": "^2.3.0",
22 | "@tensorflow/tfjs-react-native": "^0.5.0",
23 | "expo": "^40.0.0",
24 | "expo-asset": "~8.2.1",
25 | "expo-camera": "~9.1.0",
26 | "expo-constants": "~9.3.3",
27 | "expo-font": "~8.4.0",
28 | "expo-gl": "~9.2.0",
29 | "expo-image-manipulator": "~8.4.0",
30 | "expo-image-picker": "~9.2.0",
31 | "expo-linking": "~2.0.1",
32 | "expo-permissions": "~10.0.0",
33 | "expo-splash-screen": "~0.8.1",
34 | "expo-status-bar": "~1.0.3",
35 | "expo-web-browser": "~8.6.0",
36 | "jpeg-js": "^0.3.7",
37 | "react": "16.13.1",
38 | "react-dom": "16.13.1",
39 | "react-native": "0.63.4",
40 | "react-native-elements": "^2.3.1",
41 | "react-native-fs": "^2.16.6",
42 | "react-native-gesture-handler": "~1.8.0",
43 | "react-native-markdown-display": "^6.1.6",
44 | "react-native-paper": "^4.0.1",
45 | "react-native-safe-area-context": "3.1.9",
46 | "react-native-screens": "~2.15.2",
47 | "react-native-unimodules": "~0.12.0",
48 | "react-native-web": "~0.13.12"
49 | },
50 | "devDependencies": {
51 | "@babel/core": "~7.9.0",
52 | "@types/react": "~16.9.35",
53 | "@types/react-native": "~0.63.2",
54 | "babel-preset-expo": "8.3.0",
55 | "jest-expo": "^40.0.0",
56 | "typescript": "~4.0.0"
57 | },
58 | "private": true
59 | }
60 |
--------------------------------------------------------------------------------
/screens/AboutScreen.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import { StyleSheet } from 'react-native';
3 |
4 | import { View } from '../components/Themed';
5 | import Markdown from 'react-native-markdown-display';
6 | import {AppConfig} from "../config"
7 | import {getColor} from "../components/Themed"
8 | export default function AboutScreen() {
9 | const color = getColor('text')
10 | return (
11 |
12 |
13 |
17 | {AppConfig.description}
18 |
19 |
20 | );
21 | }
22 |
23 | const styles = StyleSheet.create({
24 | container: {
25 | flex: 1,
26 | alignItems: 'center',
27 | justifyContent: 'space-between',
28 | paddingHorizontal: 10
29 | }
30 |
31 |
32 | });
33 |
--------------------------------------------------------------------------------
/screens/DebugScreen.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import { StyleSheet } from 'react-native';
3 |
4 | import { View, Text} from '../components/Themed';
5 | import Markdown from 'react-native-markdown-display';
6 | import {AppConfig} from "../config"
7 | import {getColor} from "../components/Themed"
8 | export default function DebugScreen() {
9 | const color = getColor('text')
10 | const modelClasses = require("../assets/model_tfjs/classes.json")
11 |
12 | return (
13 |
14 |
15 | Classes
16 |
17 |
18 |
19 | {modelClasses.map(p => {
20 | return (
21 |
22 | {p}
23 |
24 | );
25 | })}
26 |
27 |
28 | );
29 | }
30 |
31 | const styles = StyleSheet.create({
32 | container: {
33 | flex: 1,
34 | paddingHorizontal: 10
35 | }
36 |
37 |
38 | });
39 |
--------------------------------------------------------------------------------
/screens/HomeScreen.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 |
3 | //import { Text, View } from '../components/Themed';
4 |
5 | import * as Permissions from 'expo-permissions'
6 | import * as tf from '@tensorflow/tfjs';
7 | import * as ImageManipulator from 'expo-image-manipulator';
8 |
9 |
10 | import {
11 | Image,
12 | StyleSheet,
13 | } from 'react-native';
14 | import {AppConfig} from "../config"
15 |
16 | import {Text ,View,getColor,ActivityIndicator,ScrollView} from '../components/Themed'
17 |
18 | import {Icon, ListItem} from 'react-native-elements';
19 |
20 |
21 | import * as ImagePicker from 'expo-image-picker';
22 | import { ModelService, IModelPredictionResponse,IModelPredictionTiming,ModelPrediction } from '../components/ModelService';
23 |
24 |
25 | type State = {
26 | image: ImageManipulator.ImageResult;
27 | loading:boolean;
28 | isTfReady: boolean;
29 | isModelReady: boolean;
30 | predictions: ModelPrediction[]|null;
31 | error:string|null;
32 | timing:IModelPredictionTiming|null;
33 | };
34 |
35 | export default class HomeScreen extends React.Component<{},State> {
36 | static navigationOptions = {
37 | header: null,
38 | };
39 |
40 | state:State = {
41 | image: {},
42 | loading: false,
43 | isTfReady: false,
44 | isModelReady: false,
45 | predictions: null,
46 | error:null,
47 | timing:null
48 | }
49 |
50 | modelService!:ModelService;
51 |
52 | async componentDidMount() {
53 | this.setState({ loading: true });
54 | this.modelService = await ModelService.create(AppConfig.imageSize);
55 | this.setState({ isTfReady: true,isModelReady: true,loading: false });
56 | }
57 |
58 | render() {
59 |
60 | const modelLoadingStatus = this.state.isModelReady ? "✅" : "❓";
61 | return (
62 |
63 |
64 |
65 |
66 | {AppConfig.title}
67 |
68 |
69 |
70 |
71 | Model Status: {modelLoadingStatus}
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 | {this.renderPredictions()}
88 |
89 |
90 |
91 |
92 | );
93 | }
94 |
95 |
96 | renderPredictions() {
97 | if (this.state.loading) {
98 | return
99 | }
100 | let predictions= this.state.predictions || [];
101 |
102 | if (predictions.length > 0) {
103 | return (
104 |
105 | Predictions
106 |
107 | {
108 | predictions.map((item, index) => (
109 |
110 |
111 |
112 | {item.className}
113 | {`prob: ${item.probability.toFixed(AppConfig.precision)}`}
114 |
115 |
116 |
117 | ))
118 | }
119 |
120 |
121 |
122 | Timing (ms)
123 |
124 | total time: {this.state.timing?.totalTime}
125 | loading time: {this.state.timing?.imageLoadingTime}
126 | preprocessing time: {this.state.timing?.imagePreprocessing}
127 | prediction time: {this.state.timing?.imagePrediction}
128 | decode time: {this.state.timing?.imageDecodePrediction}
129 |
130 |
131 |
132 |
133 | )
134 | } else {
135 | return null
136 | }
137 | }
138 |
139 |
140 | _verifyPermissions = async () => {
141 | console.log("Verifying Permissions");
142 | const { status, expires, permissions } = await Permissions.getAsync(Permissions.CAMERA, Permissions.CAMERA_ROLL);
143 |
144 | if (status !== 'granted') {
145 | const { status, permissions } = await Permissions.askAsync(Permissions.CAMERA, Permissions.CAMERA_ROLL)
146 |
147 | if (status === 'granted') {
148 | console.log("Permissions granted");
149 | return true
150 | } else {
151 | alert('Hey! You have not enabled selected permissions');
152 | return false
153 | }
154 |
155 | }else{
156 | return true;
157 | }
158 | };
159 |
160 | _pickImageFromLibrary = async () => {
161 | const status = await this._verifyPermissions();
162 |
163 | try {
164 | let response = await ImagePicker.launchImageLibraryAsync({
165 | mediaTypes: ImagePicker.MediaTypeOptions.Images,
166 | allowsEditing: true,
167 | aspect: [4, 3]
168 | })
169 |
170 | if (!response.cancelled) {
171 | //const source = { uri: response.uri }
172 |
173 | //this.setState({ image: source })
174 | this._classifyImage(response.uri)
175 | }
176 | } catch (error) {
177 | console.log(error)
178 | }
179 |
180 | };
181 |
182 | _pickImageFromCamera = async () => {
183 | const status = await this._verifyPermissions();
184 |
185 | try {
186 |
187 | let response = await ImagePicker.launchCameraAsync({
188 | mediaTypes: ImagePicker.MediaTypeOptions.Images,
189 | allowsEditing: true,
190 | aspect: [4, 3]
191 | });
192 |
193 | if (!response.cancelled) {
194 | //const source = { uri: response.uri }
195 |
196 | this._classifyImage(response.uri)
197 | }
198 | } catch (error) {
199 | console.log(error)
200 | }
201 |
202 | };
203 |
204 | _classifyImage = async (imageUri:string) => {
205 | try {
206 | const res:ImageManipulator.ImageResult = await ImageManipulator.manipulateAsync(imageUri,
207 | [{ resize: { width:AppConfig.imageSize, height:AppConfig.imageSize }}],
208 | { compress: 1, format: ImageManipulator.SaveFormat.JPEG,base64:true }
209 | );
210 |
211 | this.setState({ image: res})
212 | console.log('numTensors (before prediction): ' + tf.memory().numTensors);
213 | this.setState({ predictions: [] ,error:null , loading:true })
214 |
215 | const predictionResponse = await this.modelService.classifyImage(res);
216 |
217 |
218 | if (predictionResponse.error){
219 | this.setState({ error: predictionResponse.error , loading:false})
220 | }else{
221 | const predictions = predictionResponse.predictions || null;
222 | this.setState({ predictions: predictions, timing:predictionResponse.timing, loading:false})
223 | }
224 |
225 |
226 | //tf.dispose(predictions);
227 | console.log('numTensors (after prediction): ' + tf.memory().numTensors);
228 |
229 | } catch (error) {
230 | console.log('Exception Error: ', error)
231 | }
232 | }
233 |
234 | }
235 |
236 |
237 | const styles = StyleSheet.create({
238 | container: {
239 | paddingTop: 5,
240 | flex: 1,
241 | },
242 |
243 | contentContainer: {
244 | alignItems: 'center',
245 | justifyContent: 'center',
246 |
247 | },
248 | titleContainer: {
249 | alignItems: 'center',
250 | marginTop: 10,
251 | //flex: 2,
252 | justifyContent: 'center',
253 | },
254 | actionsContainer: {
255 | alignItems: 'center',
256 | marginTop: 5,
257 | marginBottom: 5,
258 | //flex: 1,
259 | },
260 | imageContainer: {
261 | alignItems: 'center',
262 | },
263 | callToActionContainer: {
264 | flexDirection: "row"
265 | },
266 |
267 | feedBackActionsContainer: {
268 | flexDirection: "row"
269 | },
270 |
271 | predictionsContainer: {
272 | padding: 10,
273 | justifyContent: 'center',
274 | },
275 |
276 | predictionsContentContainer: {
277 | padding: 10,
278 | },
279 | predictionRow: {
280 | flexDirection: "row",
281 | },
282 | predictionRowCategory: {
283 | justifyContent: "space-between"
284 | },
285 | predictionRowLabel: {
286 | justifyContent: "space-between"
287 | }
288 | });
289 |
--------------------------------------------------------------------------------
/screens/NotFoundScreen.tsx:
--------------------------------------------------------------------------------
1 | import { StackScreenProps } from '@react-navigation/stack';
2 | import * as React from 'react';
3 | import { StyleSheet, Text, TouchableOpacity, View } from 'react-native';
4 |
5 | import { RootStackParamList } from '../types';
6 |
7 | export default function NotFoundScreen({
8 | navigation,
9 | }: StackScreenProps) {
10 | return (
11 |
12 | This screen doesn't exist.
13 | navigation.replace('Root')} style={styles.link}>
14 | Go to home screen!
15 |
16 |
17 | );
18 | }
19 |
20 | const styles = StyleSheet.create({
21 | container: {
22 | flex: 1,
23 | backgroundColor: '#fff',
24 | alignItems: 'center',
25 | justifyContent: 'center',
26 | padding: 20,
27 | },
28 | title: {
29 | fontSize: 20,
30 | fontWeight: 'bold',
31 | },
32 | link: {
33 | marginTop: 15,
34 | paddingVertical: 15,
35 | },
36 | linkText: {
37 | fontSize: 14,
38 | color: '#2e78b7',
39 | },
40 | });
41 |
--------------------------------------------------------------------------------
/test.md:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "allowSyntheticDefaultImports": true,
4 | "jsx": "react-native",
5 | "lib": ["dom", "esnext"],
6 | "moduleResolution": "node",
7 | "noEmit": true,
8 | "skipLibCheck": true,
9 | "resolveJsonModule": true,
10 | "strict": true
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/types.tsx:
--------------------------------------------------------------------------------
1 | export type RootStackParamList = {
2 | Root: undefined;
3 | NotFound: undefined;
4 | };
5 |
6 | export type BottomTabParamList = {
7 | Home: undefined;
8 | About: undefined;
9 | Debug: undefined;
10 | };
11 |
12 | export type HomeParamList = {
13 | HomeScreen: undefined;
14 | };
15 |
16 | export type AboutParamList = {
17 | AboutScreen: undefined;
18 | };
19 |
20 | export type DebugParamList = {
21 | DebugScreen: undefined;
22 | };
23 |
--------------------------------------------------------------------------------