16 |
17 | ## Requirements
18 |
19 | - Android Studio 3.1.x
20 | - Gradle 3.1.x
21 | - Android Support Library, revision 21 or newer
22 |
23 | ## Algorithm
24 |
25 | ### Pose Matching
26 |
27 | The angles of the six edges of the input upper body are extracted and the angle and accuracy of the skeleton to be matched are measured.
28 |
29 | 
30 |
31 | **Angle calculation**
32 |
33 | ```java
34 | private double getAngle(float[][] resultArr, int index) {
35 | ListWARNING: This is an experimental API and subject to change.
24 | */
25 | public class GpuDelegateHelper {
26 | private GpuDelegateHelper() {}
27 |
28 | /** Checks whether {@code GpuDelegate} is available. */
29 | public static boolean isGpuDelegateAvailable() {
30 | try {
31 | //gpu 드라이버 확인
32 | Class.forName("org.tensorflow.lite.experimental.GpuDelegate");
33 | return true;
34 | } catch (Exception e) {
35 | return false;
36 | }
37 | }
38 |
39 | /** Returns an instance of {@code GpuDelegate} if available. */
40 | /** TODO: 세부적인 분석 가능하다면, 분석할 필요가 있음 */
41 | public static Delegate createGpuDelegate() {
42 | try {
43 | return Class.forName("org.tensorflow.lite.experimental.GpuDelegate")
44 | .asSubclass(Delegate.class)
45 | .getDeclaredConstructor()
46 | .newInstance();
47 | } catch (Exception e) {
48 | throw new IllegalStateException(e);
49 | }
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifier.java:
--------------------------------------------------------------------------------
1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2 |
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at
6 |
7 | http://www.apache.org/licenses/LICENSE-2.0
8 |
9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 | ==============================================================================*/
15 |
16 | package com.example.android.tflitecamerademo;
17 |
18 | import android.app.Activity;
19 | import android.content.res.AssetFileDescriptor;
20 | import android.graphics.Bitmap;
21 | import android.os.SystemClock;
22 | import android.text.SpannableString;
23 | import android.text.SpannableStringBuilder;
24 | import android.util.Log;
25 |
26 | import com.example.android.tflitecamerademo.model.BaseAngle;
27 | import com.example.android.tflitecamerademo.view.DrawView;
28 |
29 | import java.io.FileInputStream;
30 | import java.io.IOException;
31 | import java.nio.ByteBuffer;
32 | import java.nio.ByteOrder;
33 | import java.nio.MappedByteBuffer;
34 | import java.nio.channels.FileChannel;
35 | import java.util.ArrayList;
36 | import java.util.List;
37 |
38 | import org.tensorflow.lite.Delegate;
39 | import org.tensorflow.lite.Interpreter;
40 |
41 | /**
42 | * Classifies images with Tensorflow Lite.
43 | */
44 | public abstract class ImageClassifier {
45 |
46 | // Display preferences
47 | /**
48 | * 30% 이상은 강조글로 표현
49 | */
50 | //private static final float GOOD_PROB_THRESHOLD = 0.3f;
51 | private static final int SMALL_COLOR = 0xffddaa88;
52 |
53 | /**
54 | * zk * output shape (heatmap shape)
55 | */
56 |
57 | //model_h
58 | private static final int HEATMAPWIGHT = 96;
59 | private static final int HEATMAPHEIGHT = 96;
60 | private static final int NUMJOINT = 14;
61 | //이게 점들의수
62 |
63 | /**
64 | * Tag for the {@link Log}.
65 | */
66 | private static final String TAG = "TfLiteCameraDemo";
67 |
68 | /** Number of results to show in the UI.
69 | * 몇개 결과창으로 보여 줄지 */
70 | //private static final int RESULTS_TO_SHOW = 3;
71 |
72 | /**
73 | * Dimensions of inputs.
74 | */
75 | private static final int DIM_BATCH_SIZE = 1;
76 |
77 | /**
78 | * 채널 사이즈 입력 값
79 | */
80 | private static final int DIM_PIXEL_SIZE = 3;
81 |
82 | private static final int FULL_DEGREE = 360;
83 | private static final int HALF_DEGREE = 180;
84 | private static final int QUARTER_DEGREE = 90;
85 |
86 | private static List This additional method is necessary, because we don't have a common base for different
514 | * primitive data types.
515 | */
516 | protected abstract void runInference();
517 |
518 | /**
519 | * Get the total number of labels.
520 | *
521 | * @return
522 | */
523 | /*
524 | protected int getNumLabels() {
525 | return labelList.size();
526 | }
527 | */
528 |
529 | /**
530 | * Get the shape of output(heatmap) .
531 | *
532 | * @return
533 | */
534 | protected int getHeatmapWidth() {
535 | return HEATMAPWIGHT;
536 | }
537 |
538 | protected int getHeatmapHeight() {
539 | return HEATMAPHEIGHT;
540 | }
541 |
542 | protected int getNumJoint() {
543 | return NUMJOINT;
544 | }
545 | }
546 |
--------------------------------------------------------------------------------
/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierFloatBodypose.java:
--------------------------------------------------------------------------------
1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2 |
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at
6 |
7 | http://www.apache.org/licenses/LICENSE-2.0
8 |
9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 | ==============================================================================*/
15 |
16 | package com.example.android.tflitecamerademo;
17 |
18 | import android.app.Activity;
19 | import android.util.Log;
20 |
21 | import java.io.IOException;
22 |
23 | /**
24 | * This classifier works with the Inception-v3 slim model.
25 | * It applies floating point inference rather than using a quantized model.
26 | */
27 | public class ImageClassifierFloatBodypose extends ImageClassifier {
28 |
29 | /**
30 | * The inception net requires additional normalization of the used input.
31 | */
32 | //private static final int IMAGE_MEAN = 128;
33 | //private static final float IMAGE_STD = 128.0f;
34 |
35 | /**
36 | * An array to hold inference results, to be feed into Tensorflow Lite as outputs.
37 | * This isn't part of the super class, because we need a primitive array here.
38 | */
39 | //input shape float32[1,353,257,3]
40 | //outpu shape float32[1,23,17,17]
41 | private float[][][][] jointProbArray = null;
42 |
43 | /**
44 | * Initializes an {@code ImageClassifier}.
45 | *
46 | * @param activity
47 | */
48 | ImageClassifierFloatBodypose(Activity activity) throws IOException {
49 | super(activity);
50 | jointProbArray = new float[1][getHeatmapWidth()][getHeatmapHeight()][getNumJoint()];
51 | }
52 |
53 | @Override
54 | protected String getModelPath() {
55 | // you can download this file from
56 | // https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip
57 | //return "multi_person_mobilenet_v1_075_float.tflite";
58 | //return "model_cpm.tflite";
59 | return "model_h.tflite";
60 | }
61 |
62 | /*
63 | @Override
64 | protected String getLabelPath() {
65 | return "labels_imagenet_slim.txt";
66 | }
67 | */
68 |
69 | @Override
70 | protected int getImageSizeX() {
71 | return 192;
72 | //return 353;
73 |
74 | }
75 |
76 |
77 | @Override
78 | protected int getImageSizeY() {
79 | return 192;
80 | //return 257;
81 | }
82 |
83 | @Override
84 | protected int getNumBytesPerChannel() {
85 | // a 32bit float value requires 4 bytes
86 | return 4;
87 | }
88 |
89 | @Override
90 | protected void addPixelValue(int pixelValue) {
91 |
92 | /*
93 | imgData.putFloat((pixelValue & 0xFF) / 255.f);
94 | imgData.putFloat(((pixelValue >> 8) & 0xFF) / 255.f);
95 | imgData.putFloat(((pixelValue >> 16) & 0xFF) / 255.f);
96 | */
97 |
98 | imgData.putFloat(Float.valueOf(pixelValue & 0xFF));
99 | imgData.putFloat(Float.valueOf(pixelValue >> 8 & 0xFF));
100 | imgData.putFloat(Float.valueOf(pixelValue >> 16 & 0xFF));
101 |
102 | }
103 |
104 | @Override
105 | protected float getProbability(int index, int width, int height, int joint) {
106 | return jointProbArray[index][width][height][joint];
107 | }
108 |
109 |
110 | /*
111 | @Override
112 | protected void setProbability(int labelIndex, Number value) {
113 | labelProbArray[0][labelIndex] = value.floatValue();
114 | }
115 | */
116 |
117 | /*
118 | @Override
119 | protected float getNormalizedProbability(int labelIndex) {
120 | // TODO the following value isn't in [0,1] yet, but may be greater. Why?
121 | return getProbability(labelIndex);
122 | }
123 | */
124 | @Override
125 | protected void runInference() {
126 | tflite.run(imgData, jointProbArray);
127 | /*
128 | for (int i=0; i<96; i++){
129 | for (int j=0; j<96; j++) {
130 | for(int k=0; k<14; k++) {
131 | Log.d("yangace", "value[" + i + "][" + j + "][" + k + "] = " + jointProbArray[0][i][j][1]);
132 | }
133 | }
134 | }
135 | */
136 |
137 | }
138 | }
139 |
--------------------------------------------------------------------------------
/demo/app/src/main/java/com/example/android/tflitecamerademo/activity/CameraActivity.java:
--------------------------------------------------------------------------------
1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2 |
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at
6 |
7 | http://www.apache.org/licenses/LICENSE-2.0
8 |
9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 | ==============================================================================*/
15 |
16 | package com.example.android.tflitecamerademo.activity;
17 |
18 | import android.app.Activity;
19 | import android.os.Bundle;
20 |
21 | import com.example.android.tflitecamerademo.Camera2BasicFragment;
22 | import com.example.android.tflitecamerademo.R;
23 |
24 | /**
25 | * Main {@code Activity} class for the Camera app.
26 | */
27 | public class CameraActivity extends Activity {
28 |
29 | @Override
30 | protected void onCreate(Bundle savedInstanceState) {
31 | super.onCreate(savedInstanceState);
32 | setContentView(R.layout.activity_camera);
33 |
34 | if (null == savedInstanceState) {
35 | getFragmentManager()
36 | .beginTransaction()
37 | .replace(R.id.container, Camera2BasicFragment.newInstance())
38 | .commit();
39 | }
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/demo/app/src/main/java/com/example/android/tflitecamerademo/activity/GetImageActivity.java:
--------------------------------------------------------------------------------
1 | package com.example.android.tflitecamerademo.activity;
2 |
3 | import android.app.Activity;
4 | import android.content.Intent;
5 | import android.database.Cursor;
6 | import android.graphics.Bitmap;
7 | import android.graphics.BitmapFactory;
8 | import android.net.Uri;
9 | import android.os.Bundle;
10 | import android.os.Environment;
11 | import android.provider.MediaStore;
12 | import android.support.annotation.Nullable;
13 | import android.util.Log;
14 | import android.view.View;
15 | import android.widget.Button;
16 | import android.widget.ImageView;
17 | import android.widget.Toast;
18 |
19 | import com.example.android.tflitecamerademo.R;
20 |
21 | import java.io.File;
22 | import java.io.FileOutputStream;
23 | import java.io.IOException;
24 | import java.text.SimpleDateFormat;
25 | import java.util.Date;
26 |
27 | public class GetImageActivity extends Activity {
28 |
29 | private static final String TAG = "1번";
30 |
31 | private Boolean isPermission = true;
32 |
33 | private static final int PICK_FROM_ALBUM = 1;
34 | private static final int PICK_FROM_CAMERA = 2;
35 |
36 | private File tempFile;
37 |
38 | @Override
39 | protected void onCreate(@Nullable Bundle savedInstanceState) {
40 | super.onCreate(savedInstanceState);
41 | setContentView(R.layout.activity_get_image);
42 |
43 |
44 | findViewById(R.id.btnGallery).setOnClickListener(new View.OnClickListener() {
45 | @Override
46 | public void onClick(View view) {
47 | // 권한 허용에 동의하지 않았을 경우 토스트를 띄웁니다.
48 | if (isPermission) goToAlbum();
49 | }
50 | });
51 |
52 | findViewById(R.id.btnCamera).setOnClickListener(new View.OnClickListener() {
53 | @Override
54 | public void onClick(View view) {
55 | // 권한 허용에 동의하지 않았을 경우 토스트를 띄웁니다.
56 | if (isPermission) takePhoto();
57 |
58 | }
59 | });
60 | Button poseMatchingButton = findViewById(R.id.poseMatchingButton);
61 |
62 | poseMatchingButton.setOnClickListener(new View.OnClickListener() {
63 | @Override
64 | public void onClick(View v) {
65 | Intent intent = new Intent(GetImageActivity.this, CameraActivity.class);
66 | startActivity(intent);
67 | }
68 | });
69 | }
70 |
71 | @Override
72 | protected void onActivityResult(int requestCode, int resultCode, Intent data) {
73 | if (resultCode != Activity.RESULT_OK) {
74 | Toast.makeText(this, "취소 되었습니다.", Toast.LENGTH_SHORT).show();
75 |
76 | if (tempFile != null) {
77 | if (tempFile.exists()) {
78 | if (tempFile.delete()) {
79 | Log.e(TAG, tempFile.getAbsolutePath() + " 삭제 성공");
80 | tempFile = null;
81 | }
82 | }
83 | }
84 |
85 | return;
86 | }
87 |
88 | if (requestCode == PICK_FROM_ALBUM) {
89 | Uri photoUri = data.getData();
90 | Log.d(TAG, "PICK_FROM_ALBUM photoUri : " + photoUri);
91 | Cursor cursor = null;
92 | try {
93 | /*
94 | * Uri 스키마를
95 | * content:/// 에서 file:/// 로 변경한다.
96 | */
97 | String[] proj = {MediaStore.Images.Media.DATA};
98 | assert photoUri != null;
99 | cursor = getContentResolver().query(photoUri, proj, null, null, null);
100 | assert cursor != null;
101 | int column_index = cursor.getColumnIndexOrThrow(MediaStore.Images.Media.DATA);
102 | cursor.moveToFirst();
103 | tempFile = new File(cursor.getString(column_index));
104 | Log.d(TAG, "tempFile Uri : " + Uri.fromFile(tempFile));
105 | } finally {
106 | if (cursor != null) {
107 | cursor.close();
108 | }
109 | }
110 | setImage();
111 | } else if (requestCode == PICK_FROM_CAMERA) {
112 | setImage();
113 | }
114 | }
115 |
116 | /**
117 | * 앨범에서 이미지 가져오기
118 | */
119 | private void goToAlbum() {
120 | Intent intent = new Intent(Intent.ACTION_PICK);
121 | intent.setType(MediaStore.Images.Media.CONTENT_TYPE);
122 | startActivityForResult(intent, PICK_FROM_ALBUM);
123 | }
124 |
125 |
126 | /**
127 | * 카메라에서 이미지 가져오기
128 | */
129 | private void takePhoto() {
130 | Intent intent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE);
131 | try {
132 | tempFile = createImageFile();
133 | } catch (IOException e) {
134 | Toast.makeText(this, "이미지 처리 오류! 다시 시도해주세요.", Toast.LENGTH_SHORT).show();
135 | finish();
136 | e.printStackTrace();
137 | }
138 | if (tempFile != null) {
139 | Uri photoUri = Uri.fromFile(tempFile);
140 | intent.putExtra(MediaStore.EXTRA_OUTPUT, photoUri);
141 | startActivityForResult(intent, PICK_FROM_CAMERA);
142 | }
143 | }
144 |
145 | /**
146 | * 폴더 및 파일 만들기
147 | */
148 | private File createImageFile() throws IOException {
149 |
150 | // 이미지 파일 이름 ( rehap_{시간}_ )
151 | String timeStamp = new SimpleDateFormat("HHmmss").format(new Date());
152 | String imageFileName = "rehap_" + timeStamp + "_";
153 |
154 | // 이미지가 저장될 폴더 이름 ( rehap )
155 | File storageDir = new File(Environment.getExternalStorageDirectory() + "/rehap/");
156 | if (!storageDir.exists()) {
157 | storageDir.mkdirs();
158 | }
159 |
160 | // 파일 생성
161 | File image = File.createTempFile(imageFileName, ".jpg", storageDir);
162 | Log.d(TAG, "createImageFile : " + image.getAbsolutePath());
163 |
164 | return image;
165 | }
166 |
167 | /**
168 | * tempFile 을 bitmap 으로 변환 후 ImageView 에 설정한다.
169 | */
170 | private void setImage() {
171 |
172 | ImageView imageView = (ImageView) findViewById(R.id.imageView);
173 |
174 | BitmapFactory.Options options = new BitmapFactory.Options();
175 | Bitmap originalBm = BitmapFactory.decodeFile(tempFile.getAbsolutePath(), options);
176 | Log.d(TAG, "setImage : " + tempFile.getAbsolutePath());
177 |
178 | imageView.setImageBitmap(originalBm);
179 | saveImage(originalBm);
180 | /**
181 | * tempFile 사용 후 null 처리를 해줘야 합니다.
182 | * (resultCode != RESULT_OK) 일 때 tempFile 을 삭제하기 때문에
183 | * 기존에 데이터가 남아 있게 되면 원치 않은 삭제가 이뤄집니다.
184 | */
185 | tempFile = null;
186 |
187 | }
188 |
189 | private void saveImage(Bitmap finalBitmap) {
190 |
191 | String timeStamp = new SimpleDateFormat("HHmmss").format(new Date());
192 | String imageFileName = "rehap_" + timeStamp +".jpg";
193 | String root = Environment.getExternalStorageDirectory()+"/rehap/";
194 | File myDir = new File(root);
195 | if(!myDir.exists()){
196 | myDir.mkdirs();
197 | }
198 | File file = new File(myDir, imageFileName);
199 | if (file.exists()) {
200 | file.delete();
201 | }
202 | Log.i("LOAD", root + imageFileName);
203 | try {
204 | FileOutputStream out = new FileOutputStream(file);
205 | finalBitmap.compress(Bitmap.CompressFormat.JPEG, 90, out);
206 | out.flush();
207 | out.close();
208 | } catch (Exception e) {
209 | e.printStackTrace();
210 | }
211 | }
212 |
213 | }
214 |
--------------------------------------------------------------------------------
/demo/app/src/main/java/com/example/android/tflitecamerademo/activity/PoseListActivity.java:
--------------------------------------------------------------------------------
1 | package com.example.android.tflitecamerademo.activity;
2 |
3 | import android.app.Activity;
4 | import android.content.Intent;
5 | import android.os.Bundle;
6 | import android.support.v7.widget.LinearLayoutManager;
7 | import android.support.v7.widget.RecyclerView;
8 | import android.view.View;
9 | import android.widget.Button;
10 |
11 | import com.example.android.tflitecamerademo.R;
12 | import com.example.android.tflitecamerademo.model.PoseList;
13 | import com.example.android.tflitecamerademo.model.PoseRecyclerAdapter;
14 |
15 | import java.util.List;
16 |
17 | public class PoseListActivity extends Activity {
18 | private PoseRecyclerAdapter adapter;
19 | @Override
20 | protected void onCreate(Bundle savedInstanceState) {
21 | super.onCreate(savedInstanceState);
22 | setContentView(R.layout.activity_pose_list);
23 |
24 | init();
25 |
26 | setData();
27 | }
28 |
29 | private void init() {
30 | RecyclerView recyclerView = (RecyclerView) findViewById(R.id.recyclerView);
31 | LinearLayoutManager linearLayoutManager = new LinearLayoutManager(this);
32 | recyclerView.setLayoutManager(linearLayoutManager);
33 | Button poseRegisterButton = findViewById(R.id.poseRegisterButton);
34 | adapter = new PoseRecyclerAdapter();
35 | recyclerView.setAdapter(adapter);
36 |
37 | poseRegisterButton.setOnClickListener(new View.OnClickListener() {
38 | @Override
39 | public void onClick(View v) {
40 | Intent intent = new Intent(getApplicationContext(), GetImageActivity.class);
41 | startActivity(intent);
42 | }
43 | });
44 | }
45 | private void setData() {
46 | PoseList pose = new PoseList();
47 | adapter.addItem(pose);
48 | adapter.addItem(pose);
49 | adapter.addItem(pose);
50 | adapter.notifyDataSetChanged();
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/demo/app/src/main/java/com/example/android/tflitecamerademo/model/BaseAngle.java:
--------------------------------------------------------------------------------
1 | package com.example.android.tflitecamerademo.model;
2 |
3 | import java.util.Arrays;
4 | import java.util.List;
5 |
6 | public class BaseAngle {
7 | private int first;
8 | private int second;
9 | private int thrid;
10 |
11 | public BaseAngle(int first, int second, int thrid) {
12 | this.first = first;
13 | this.second = second;
14 | this.thrid = thrid;
15 | }
16 |
17 | public List