├── .DS_Store ├── .firebaserc ├── .gitignore ├── README.md ├── custom_model ├── Train Classifier.ipynb ├── activate.sh ├── create_tf_record.py ├── download_model.sh ├── download_trained_model.sh ├── eval_cloudml.sh ├── export_images.py ├── gcs_deploy.sh ├── model │ └── graph.pbtxt ├── object_detection_tutorial.ipynb ├── opencv.py ├── pipeline.config ├── run_cloudml.sh ├── ssd_mobilenet_v1_coco_11_06_2017 │ ├── graph.pbtxt │ ├── model.ckpt.index │ └── model.ckpt.meta ├── tensorboard.sh ├── test │ └── popcorn_test.png ├── tfjs-models │ ├── group1-shard1of6 │ ├── group1-shard2of6 │ ├── group1-shard3of6 │ ├── group1-shard4of6 │ ├── group1-shard5of6 │ ├── group1-shard6of6 │ └── model.json └── training │ ├── cloud.yml │ ├── faster_rcnn_resnet101_coco.config │ └── pascal_label_map.pbtxt ├── database.rules.json ├── edge-server ├── .gitignore ├── BaseClassifier.js ├── CatClassifier.js ├── CatDetector.js ├── CloudIoTCoreGateway.js ├── DeviceListener.js ├── EXAMPLE_BERRY.jpeg ├── EXAMPLE_JAM.jpeg ├── EdgeServer.js ├── ImageClassifier.js ├── Log.js ├── WebInterface.js ├── edge-server-flowchart.md ├── index.html ├── index.js ├── package-lock.json ├── package.json ├── popcorn.jpeg ├── test_1.jpeg ├── test_2.jpeg ├── test_3.jpeg ├── test_model.js └── util │ └── Bag.js ├── esp32-camera-firmware ├── .gitignore ├── .travis.yml ├── .vscode │ ├── extensions.json │ └── settings.json ├── include │ └── README ├── lib │ ├── README │ ├── esp_camera │ │ ├── camera.c │ │ ├── camera_common.h │ │ ├── esp_camera.h │ │ ├── ov2640.c │ │ ├── ov2640.h │ │ ├── ov2640_regs.h │ │ ├── ov2640_settings.h │ │ ├── ov7725.c │ │ ├── ov7725.h │ │ ├── ov7725_regs.h │ │ ├── sccb.c │ │ ├── sccb.h │ │ ├── sensor.c │ │ ├── sensor.h │ │ ├── twi.c │ │ ├── twi.h │ │ ├── xclk.c │ │ └── xclk.h │ ├── esp_http_server │ │ ├── ctrl_sock.c │ │ ├── ctrl_sock.h │ │ ├── esp_http_server.h │ │ ├── esp_httpd_priv.h │ │ ├── http_server.h │ │ ├── httpd_main.c │ │ ├── httpd_parse.c │ │ ├── httpd_sess.c │ │ ├── httpd_txrx.c │ │ ├── httpd_uri.c │ │ └── osal.h │ └── img_converters │ │ ├── esp_jpg_decode.c │ │ ├── esp_jpg_decode.h │ │ ├── img_converters.h │ │ ├── jpge.cpp │ │ ├── jpge.h │ │ ├── to_bmp.c │ │ ├── to_jpg.cpp │ │ ├── yuv.c │ │ └── yuv.h ├── platformio.ini ├── src │ ├── Kconfig.projbuild │ ├── app_main.c │ ├── component.mk │ ├── sdkconfig.h │ └── sdkconfig.h.bak └── test │ └── README ├── firebase.json ├── functions ├── .eslintrc.json ├── index.js ├── package-lock.json └── package.json ├── generate_key_pair.sh ├── public └── index.html ├── register_device.sh ├── set_env_vars.sh └── setup.sh /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/.DS_Store -------------------------------------------------------------------------------- /.firebaserc: -------------------------------------------------------------------------------- 1 | { 2 | "projects": { 3 | "production": "gcloud-iot-edge" 4 | } 5 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pem 2 | .DS_Store 3 | node_modules 4 | *.log 5 | .firebase 6 | .ipynb_checkpoints 7 | custom_model/**/*.jpg 8 | custom_model/**/*.xml 9 | custom_model/**/*.csv 10 | custom_model/**/*.record 11 | custom_model/**/*.pb 12 | custom_model/**/*.h5 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Asset Tracking using Edge Computing and Computer Vision. 2 | 3 | A bunch of ESP32 with camera, a local server (probablye a raspberry pi ) running image classification and object detection using Tensorflow and sending the data to 4 | processed data to Google Cloud. 5 | 6 | [Work in Progress] 7 | 8 | ### Upload firmware with PlatformIO 9 | 10 | Open `esp32-camera-firmware` folder on PlatformIO. Now the firmware have support for two models of esp32 with camera: 11 | 12 | * ESP32 Cam from M5 Stack 13 | * ESP32 Cam from SeedStudio 14 | 15 | Depending on your model, change on the platformio.ini file the `env_default` configuration depending on your board (`m5cam` or `esp32cam`). Also you need to change the Wifi credentials on the `sdkconfig.h` file ( `CONFIG_WIFI_SSID` and `CONFIG_WIFI_PASSWORD`). 16 | 17 | Then click on upload to flash the firmware into the board. 18 | 19 | ### Run server edge node 20 | 21 | The server was written using NodeJS, Tensorflow.js library and the CocoSSD model to detect objects on the image. 22 | 23 | Run the following commands inside the `edge-server` folder to setup the server: 24 | 25 | * Install dependencies: 26 | * `npm install` 27 | * Run server: 28 | * `npm start` 29 | * Open `localhost:3000` to see the UI 30 | 31 | ### Google Cloud Setup 32 | 33 | [Work in Progress] -------------------------------------------------------------------------------- /custom_model/activate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ~/tensorflow/bin/activate 3 | export TENSORFLOWLIBPATH=/Users/alvaroviebrantz/Documents/Desenvolvimento/Python/models/research 4 | export PYTHONPATH=$PYTHONPATH:$TENSORFLOWLIBPATH:$TENSORFLOWLIBPATH/slim 5 | -------------------------------------------------------------------------------- /custom_model/create_tf_record.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | r"""Convert raw PASCAL dataset to TFRecord for object_detection. 17 | 18 | Example usage: 19 | python create_tf_record.py \ 20 | --data_dir=/home/user/VOCdevkit \ 21 | --output_path=/home/user/pascal.record 22 | """ 23 | from __future__ import absolute_import 24 | from __future__ import division 25 | from __future__ import print_function 26 | 27 | import hashlib 28 | import io 29 | import logging 30 | import os 31 | import glob 32 | import random 33 | 34 | from lxml import etree 35 | import PIL.Image 36 | import tensorflow as tf 37 | 38 | from object_detection.utils import dataset_util 39 | from object_detection.utils import label_map_util 40 | 41 | 42 | flags = tf.app.flags 43 | flags.DEFINE_string('set', 'train', 'Convert training set, validation set or ' 44 | 'merged set.') 45 | flags.DEFINE_string('label_map_path', 'training/pascal_label_map.pbtxt', 46 | 'Path to label map proto') 47 | FLAGS = flags.FLAGS 48 | 49 | SETS = ['train', 'val', 'trainval', 'test'] 50 | 51 | 52 | def dict_to_tf_example(data, full_path, label_map_dict): 53 | 54 | #full_path = os.path.join(data['folder'], data['filename']) 55 | with tf.gfile.GFile(full_path, 'rb') as fid: 56 | encoded_jpg = fid.read() 57 | encoded_jpg_io = io.BytesIO(encoded_jpg) 58 | image = PIL.Image.open(encoded_jpg_io) 59 | 60 | if image.format != 'JPEG': 61 | raise ValueError('Image format not JPEG') 62 | key = hashlib.sha256(encoded_jpg).hexdigest() 63 | 64 | width = int(data['size']['width']) 65 | height = int(data['size']['height']) 66 | 67 | xmin = [] 68 | ymin = [] 69 | xmax = [] 70 | ymax = [] 71 | classes = [] 72 | classes_text = [] 73 | truncated = [] 74 | poses = [] 75 | difficult_obj = [] 76 | if 'object' in data: 77 | for obj in data['object']: 78 | difficult = bool(int(obj['difficult'])) 79 | difficult_obj.append(int(difficult)) 80 | 81 | xmin.append(float(obj['bndbox']['xmin']) / width) 82 | ymin.append(float(obj['bndbox']['ymin']) / height) 83 | xmax.append(float(obj['bndbox']['xmax']) / width) 84 | ymax.append(float(obj['bndbox']['ymax']) / height) 85 | classes_text.append(obj['name'].encode('utf8')) 86 | classes.append(label_map_dict[obj['name']]) 87 | truncated.append(int(obj['truncated'])) 88 | poses.append(obj['pose'].encode('utf8')) 89 | 90 | example = tf.train.Example(features=tf.train.Features(feature={ 91 | 'image/height': dataset_util.int64_feature(height), 92 | 'image/width': dataset_util.int64_feature(width), 93 | 'image/filename': dataset_util.bytes_feature( 94 | data['filename'].encode('utf8')), 95 | 'image/source_id': dataset_util.bytes_feature( 96 | data['filename'].encode('utf8')), 97 | 'image/key/sha256': dataset_util.bytes_feature(key.encode('utf8')), 98 | 'image/encoded': dataset_util.bytes_feature(encoded_jpg), 99 | 'image/format': dataset_util.bytes_feature('jpg'.encode('utf8')), 100 | 'image/object/bbox/xmin': dataset_util.float_list_feature(xmin), 101 | 'image/object/bbox/xmax': dataset_util.float_list_feature(xmax), 102 | 'image/object/bbox/ymin': dataset_util.float_list_feature(ymin), 103 | 'image/object/bbox/ymax': dataset_util.float_list_feature(ymax), 104 | 'image/object/class/text': dataset_util.bytes_list_feature(classes_text), 105 | 'image/object/class/label': dataset_util.int64_list_feature(classes), 106 | 'image/object/difficult': dataset_util.int64_list_feature(difficult_obj), 107 | 'image/object/truncated': dataset_util.int64_list_feature(truncated), 108 | 'image/object/view': dataset_util.bytes_list_feature(poses), 109 | })) 110 | return example 111 | 112 | 113 | def generate_record(examples, annotations_dir, label_map_dict, output_path): 114 | writer = tf.python_io.TFRecordWriter(output_path) 115 | for idx, full_path in enumerate(examples): 116 | if idx % 100 == 0: 117 | logging.info('On image %d of %d', idx, len(examples)) 118 | 119 | filename = os.path.splitext(os.path.basename(full_path))[0] 120 | path = os.path.join(annotations_dir, filename + '.xml') 121 | with tf.gfile.GFile(path, 'r') as fid: 122 | xml_str = fid.read() 123 | xml = etree.fromstring(xml_str) 124 | data = dataset_util.recursive_parse_xml_to_dict(xml)['annotation'] 125 | tf_example = dict_to_tf_example(data, full_path, label_map_dict) 126 | # print(full_path, output_path) 127 | writer.write(tf_example.SerializeToString()) 128 | writer.close() 129 | 130 | 131 | def main(_): 132 | val_output_path = os.path.join('training', 'val.record') 133 | train_output_path = os.path.join('training', 'train.record') 134 | 135 | label_map_dict = label_map_util.get_label_map_dict(FLAGS.label_map_path) 136 | annotations_dir = 'annotations' 137 | 138 | images_path = os.path.join(os.getcwd(), 'images', '*.jpg') 139 | examples = glob.glob(images_path) 140 | 141 | random.shuffle(examples) 142 | 143 | size = len(examples) 144 | train_size = int(0.85 * size) 145 | val_size = size - train_size 146 | train_data = examples[train_size:] 147 | val_data = examples[:val_size] 148 | 149 | print(train_size, val_size) 150 | generate_record(train_data, annotations_dir, 151 | label_map_dict, train_output_path) 152 | generate_record(val_data, annotations_dir, label_map_dict, val_output_path) 153 | 154 | 155 | if __name__ == '__main__': 156 | tf.app.run() 157 | -------------------------------------------------------------------------------- /custom_model/download_model.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export GCS_BUCKET=gcloud-iot-edge-tf-records 3 | 4 | # curl -O http://storage.googleapis.com/download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_coco_11_06_2017.tar.gz 5 | # tar -xvf faster_rcnn_resnet101_coco_11_06_2017.tar.gz 6 | # gsutil cp faster_rcnn_resnet101_coco_11_06_2017/model.ckpt.* gs://${GCS_BUCKET}/data/ 7 | 8 | # curl -O https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/samples/configs/ssd_mobilenet_v1_0.75_depth_quantized_300x300_pets_sync.config 9 | pushd /tmp 10 | curl -O http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_0.75_depth_300x300_coco14_sync_2018_07_03.tar.gz /tmp/ssd_mobilenet_v1_0.75_depth_300x300_coco14_sync_2018_07_03.tar.gz 11 | tar xzf ssd_mobilenet_v1_0.75_depth_300x300_coco14_sync_2018_07_03.tar.gz 12 | gsutil cp /tmp/ssd_mobilenet_v1_0.75_depth_300x300_coco14_sync_2018_07_03/model.ckpt.* gs://${GCS_BUCKET}/data/ 13 | popd -------------------------------------------------------------------------------- /custom_model/download_trained_model.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export GCS_BUCKET=gcloud-iot-edge-tf-records 3 | 4 | export CONFIG_FILE=gs://${GCS_BUCKET}/data/pipeline.config 5 | export CHECKPOINT_PATH=gs://${GCS_BUCKET}/train/model.ckpt-10000 6 | export OUTPUT_DIR=/tmp/tflite 7 | 8 | export TENSORFLOWLIBPATH=/Users/alvaroviebrantz/Documents/Desenvolvimento/Python/models/research 9 | export PYTHONPATH=$PYTHONPATH:$TENSORFLOWLIBPATH:$TENSORFLOWLIBPATH/slim 10 | 11 | python $TENSORFLOWLIBPATH/object_detection/export_inference_graph.py \ 12 | --pipeline_config_path=$CONFIG_FILE \ 13 | --trained_checkpoint_prefix=$CHECKPOINT_PATH \ 14 | --output_directory=$OUTPUT_DIR \ 15 | --add_postprocessing_op=true -------------------------------------------------------------------------------- /custom_model/eval_cloudml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export GCS_BUCKET=gcloud-iot-edge-tf-records 3 | export TENSORFLOWLIB=/Users/alvaroviebrantz/Documents/Desenvolvimento/Python/models/research 4 | 5 | gcloud ml-engine jobs submit training `whoami`_object_detection_eval_validation_`date +%s` \ 6 | --job-dir=gs://${GCS_BUCKET}/train \ 7 | --packages ${TENSORFLOWLIB}/dist/object_detection-0.1.tar.gz,${TENSORFLOWLIB}/slim/dist/slim-0.1.tar.gz,/tmp/pycocotools/pycocotools-2.0.tar.gz \ 8 | --module-name object_detection.model_main \ 9 | --runtime-version 1.12 \ 10 | --python-version 3.5 \ 11 | --scale-tier BASIC_GPU \ 12 | --region us-central1 \ 13 | -- \ 14 | --model_dir=gs://${GCS_BUCKET}/train \ 15 | --pipeline_config_path=gs://${GCS_BUCKET}/data/pipeline.config \ 16 | --checkpoint_dir=gs://${GCS_BUCKET}/train 17 | 18 | -------------------------------------------------------------------------------- /custom_model/export_images.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import xmltodict 4 | import json 5 | 6 | from PIL import Image 7 | 8 | # Look for XML files and parses then as if they were Pascal VOC Files 9 | base_dir = "annotations/" 10 | image_dir = "images/" 11 | save_dir = "output/" 12 | 13 | # Extract image samples and save to output dir 14 | 15 | 16 | def extractDataset(dataset): 17 | objs = dataset['object'] 18 | if type(objs) is not list: 19 | objs = [objs] 20 | print("Found {} objects on image '{}'...".format( 21 | len(dataset['object']), dataset['filename'])) 22 | 23 | # Open image and get ready to process 24 | img = Image.open(image_dir + dataset['filename']) 25 | 26 | try: 27 | os.mkdir(save_dir) 28 | except: 29 | pass 30 | 31 | # Run through each item and save cut image to output folder 32 | print(objs) 33 | for item in objs: 34 | # Convert str to integers 35 | label = item['name'] 36 | current_dir = save_dir + label + '/' 37 | # Create output directory 38 | try: 39 | os.mkdir(current_dir) 40 | except: 41 | pass 42 | bndbox = dict([(a, int(b)) for (a, b) in item['bndbox'].items()]) 43 | # Crop image 44 | im = img.crop((bndbox['xmin'], bndbox['ymin'], 45 | bndbox['xmax'], bndbox['ymax'])) 46 | # Save 47 | count = len(os.listdir(current_dir)) 48 | im.save(current_dir + str(count+1) + '.jpg') 49 | 50 | 51 | def main(): 52 | # Finds all XML files on data/ and append to list 53 | pascal_voc_contents = [] 54 | xmls = glob.glob(base_dir + "*.xml") 55 | print("Found {} files in data directory!".format(str(len(xmls)))) 56 | for xml in xmls: 57 | f_handle = open(xml, 'r') 58 | print("Parsing file '{}'...".format(xml)) 59 | pascal_voc_contents.append(xmltodict.parse(f_handle.read())) 60 | 61 | # Process each file individually 62 | for index in pascal_voc_contents: 63 | image_file = image_dir + index['annotation']['filename'] 64 | # If there's a corresponding file in the folder, 65 | # process the images and save to output folder 66 | if os.path.isfile(image_file): 67 | extractDataset(index['annotation']) 68 | else: 69 | print("Image file '{}' not found, skipping file...".format(image_file)) 70 | 71 | 72 | if __name__ == '__main__': 73 | main() 74 | -------------------------------------------------------------------------------- /custom_model/gcs_deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export GCS_BUCKET=gcloud-iot-edge-tf-records 3 | 4 | gsutil cp training/train.record gs://${GCS_BUCKET}/data/train.record 5 | gsutil cp training/val.record gs://${GCS_BUCKET}/data/val.record 6 | gsutil cp training/pascal_label_map.pbtxt gs://${GCS_BUCKET}/data/pascal_label_map.pbtxt 7 | #gsutil cp training/faster_rcnn_resnet101_coco.config gs://${GCS_BUCKET}/data/faster_rcnn_resnet101_coco.config 8 | gsutil cp pipeline.config gs://${GCS_BUCKET}/data/pipeline.config -------------------------------------------------------------------------------- /custom_model/model/graph.pbtxt: -------------------------------------------------------------------------------- 1 | item { 2 | id: 1 3 | name: 'berry' 4 | } 5 | 6 | item { 7 | id: 2 8 | name: 'jam' 9 | } 10 | 11 | item { 12 | id: 3 13 | name: 'muffin' 14 | } 15 | 16 | item { 17 | id: 4 18 | name: 'popcorn' 19 | } 20 | 21 | item { 22 | id: 5 23 | name: 'raspberry' 24 | } 25 | -------------------------------------------------------------------------------- /custom_model/opencv.py: -------------------------------------------------------------------------------- 1 | from object_detection.utils import visualization_utils as vis_util 2 | from object_detection.utils import label_map_util 3 | from keras.preprocessing import image 4 | import cv2 5 | from PIL import Image 6 | from matplotlib import pyplot as plt 7 | from io import StringIO 8 | from collections import defaultdict 9 | import zipfile 10 | import tensorflow as tf 11 | import time 12 | import sys 13 | import numpy as np 14 | import os 15 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 16 | 17 | 18 | #MODEL_NAME = 'ssd_mobilenet_v1_coco_11_06_2017' 19 | OBJECT_DETECTION_MODEL_NAME = 'ssd_mobilenet_v1_coco_11_06_2017' 20 | OBJECT_DETECTION_PATH_TO_CKPT = OBJECT_DETECTION_MODEL_NAME + \ 21 | '/frozen_inference_graph.pb' 22 | OBJECT_DETECTION_PATH_TO_LABELS = os.path.join( 23 | OBJECT_DETECTION_MODEL_NAME, 'graph.pbtxt') 24 | OBJECT_DETECTION_NUM_CLASSES = 90 25 | 26 | MODEL_NAME = 'model' 27 | PATH_TO_CKPT = MODEL_NAME + '/cats_model.pb' 28 | PATH_TO_LABELS = os.path.join(MODEL_NAME, 'graph.pbtxt') 29 | NUM_CLASSES = 5 30 | 31 | 32 | PATH_TO_TEST_IMAGES_DIR = 'test_images' 33 | TEST_IMAGE_PATHS = [os.path.join( 34 | PATH_TO_TEST_IMAGES_DIR, 'image{}.jpg'.format(i)) for i in range(1, 7)] 35 | 36 | # Size, in inches, of the output images. 37 | IMAGE_SIZE = (12, 8) 38 | 39 | 40 | def load_image_into_numpy_array(image): 41 | (im_width, im_height) = image.size 42 | return np.array(image.getdata()).reshape( 43 | (im_height, im_width, 3)).astype(np.uint8) 44 | 45 | 46 | def draw_boxes_with_class(image, boxes, classes, scores): 47 | vis_util.visualize_boxes_and_labels_on_image_array( 48 | image, 49 | np.squeeze(boxes), 50 | np.squeeze(classes).astype(np.int32), 51 | np.squeeze(scores), 52 | category_index, 53 | use_normalized_coordinates=True, 54 | line_thickness=8) 55 | return image 56 | 57 | 58 | def image_resize(image, width=None, height=None, inter=cv2.INTER_AREA): 59 | # initialize the dimensions of the image to be resized and 60 | # grab the image size 61 | dim = None 62 | (h, w) = image.shape[:2] 63 | 64 | # if both the width and height are None, then return the 65 | # original image 66 | if width is None and height is None: 67 | return image 68 | 69 | # check to see if the width is None 70 | if width is None: 71 | # calculate the ratio of the height and construct the 72 | # dimensions 73 | r = height / float(h) 74 | dim = (int(w * r), height) 75 | 76 | # otherwise, the height is None 77 | else: 78 | # calculate the ratio of the width and construct the 79 | # dimensions 80 | r = width / float(w) 81 | dim = (width, int(h * r)) 82 | 83 | # resize the image 84 | resized = cv2.resize(image, dim, interpolation=inter) 85 | 86 | # return the resized image 87 | return resized 88 | 89 | 90 | def classify_image(frame, sess, graph): 91 | img_tensor = image.img_to_array(frame) 92 | # print(frame, img_tensor) 93 | img_tensor = np.expand_dims(img_tensor, axis=0) 94 | img_tensor /= 255. 95 | # print(frame, img_tensor) 96 | image_tensor = graph.get_tensor_by_name('input_4:0') 97 | classes = graph.get_tensor_by_name('dense_12/Softmax:0') 98 | (classes) = sess.run( 99 | [classes], 100 | feed_dict={image_tensor: img_tensor}) 101 | local_classes = np.squeeze(classes) 102 | min_score_thresh = .5 103 | # print('classify image, found ', local_classes) 104 | classes = [] 105 | for i in range(len(local_classes)): 106 | score = local_classes[i] 107 | if score > min_score_thresh: 108 | class_name = cats_category_index[i+1]['name'] 109 | print('found ', i, class_name, score) 110 | classes.append((class_name, score)) 111 | return classes 112 | 113 | 114 | def detect_objects(frame, sess, detection_graph): 115 | # Expand dimensions since the model expects images to have shape: [1, None, None, 3] 116 | image_np_expanded = np.expand_dims(frame, axis=0) 117 | image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') 118 | # Each box represents a part of the image where a particular object was detected. 119 | boxes = detection_graph.get_tensor_by_name('detection_boxes:0') 120 | # Each score represent how level of confidence for each of the objects. 121 | # Score is shown on the result image, together with the class label. 122 | scores = detection_graph.get_tensor_by_name('detection_scores:0') 123 | classes = detection_graph.get_tensor_by_name('detection_classes:0') 124 | num_detections = detection_graph.get_tensor_by_name( 125 | 'num_detections:0') 126 | # Actual detection. 127 | (boxes, scores, classes, num_detections) = sess.run( 128 | [boxes, scores, classes, num_detections], 129 | feed_dict={image_tensor: image_np_expanded}) 130 | 131 | min_score_thresh = .5 132 | im_width = image_np_expanded.shape[2] 133 | im_height = image_np_expanded.shape[1] 134 | local_boxes = np.squeeze(boxes) 135 | local_classes = np.squeeze(classes).astype(np.int32) 136 | local_scores = np.squeeze(scores) 137 | for i in range(local_boxes.shape[0]): 138 | if local_scores[i] > min_score_thresh: 139 | box = tuple(local_boxes[i].tolist()) 140 | class_name = 'N/A' 141 | if local_classes[i] in category_index.keys(): 142 | class_name = category_index[local_classes[i]]['name'] 143 | 144 | if(class_name == 'cat'): 145 | print('found a cat', box) 146 | ymin, xmin, ymax, xmax = box 147 | (xminn, xmaxx, yminn, ymaxx) = (xmin * im_width, 148 | xmax * im_width, 149 | ymin * im_height, 150 | ymax * im_height) 151 | print('found a cat', xminn, xmaxx, yminn, ymaxx) 152 | # crop_img = tf.image.crop_to_bounding_box(frame,int(yminn), int(xminn), int(ymaxx-yminn), int(xmaxx-xminn)) 153 | crop_img = frame[int(yminn):int(ymaxx), int(xminn):int(xmaxx)] 154 | # cv2.imshow("Found a Cat", crop_img) 155 | findings = classify_image(crop_img, classification_sess, 156 | classification_graph) 157 | for item in findings: 158 | cv2.imshow("Found a Cat: " + item[0], crop_img) 159 | 160 | # Visualization of the results of a detection. 161 | # print scores, classes, num_detections 162 | frame = draw_boxes_with_class(frame, boxes, classes, scores) 163 | return frame 164 | 165 | 166 | start_time = time.time() 167 | 168 | label_map = label_map_util.load_labelmap(OBJECT_DETECTION_PATH_TO_LABELS) 169 | categories = label_map_util.convert_label_map_to_categories( 170 | label_map, max_num_classes=OBJECT_DETECTION_NUM_CLASSES, use_display_name=True) 171 | category_index = label_map_util.create_category_index(categories) 172 | 173 | cats_label_map = label_map_util.load_labelmap(PATH_TO_LABELS) 174 | cats_categories = label_map_util.convert_label_map_to_categories( 175 | cats_label_map, max_num_classes=NUM_CLASSES, use_display_name=True) 176 | cats_category_index = label_map_util.create_category_index(cats_categories) 177 | 178 | detection_graph = tf.Graph() 179 | with detection_graph.as_default(): 180 | od_graph_def = tf.GraphDef() 181 | with tf.gfile.GFile(OBJECT_DETECTION_PATH_TO_CKPT, 'rb') as fid: 182 | serialized_graph = fid.read() 183 | od_graph_def.ParseFromString(serialized_graph) 184 | tf.import_graph_def(od_graph_def, name='') 185 | 186 | sess = tf.Session(graph=detection_graph) 187 | 188 | classification_graph = tf.Graph() 189 | with classification_graph.as_default(): 190 | od_graph_def = tf.GraphDef() 191 | with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid: 192 | serialized_graph = fid.read() 193 | od_graph_def.ParseFromString(serialized_graph) 194 | tf.import_graph_def(od_graph_def, name='') 195 | 196 | classification_sess = tf.Session(graph=classification_graph) 197 | 198 | 199 | print("--- %s seconds for initialization ---" % (time.time() - start_time)) 200 | 201 | ''' 202 | for image_path in TEST_IMAGE_PATHS: 203 | start_time = time.time() 204 | image = Image.open(image_path) 205 | # the array based representation of the image will be used later in order to prepare the 206 | # result image with boxes and labels on it. 207 | image_np = load_image_into_numpy_array(image) 208 | image_np = detect_objects(image_np, sess, detection_graph) 209 | # plt.figure(figsize=IMAGE_SIZE) 210 | # plt.imgshow(image_np) 211 | print("--- %s seconds for image %s---" % 212 | (time.time() - start_time, image_path)) 213 | cv2.imshow(image_path, image_np) 214 | 215 | cv2.waitKey(0) 216 | cv2.destroyAllWindows() 217 | ''' 218 | 219 | cap = cv2.VideoCapture(0) 220 | 221 | while(True): 222 | # Capture frame-by-frame 223 | ret, frame = cap.read() 224 | proportion = 0.5 225 | small = cv2.resize(frame, (0, 0), fx=proportion, fy=proportion) 226 | # small = image_resize(frame, height=300) 227 | # (h, w) = small.shape[:2] 228 | # y = 0 229 | # x = round((w-300)/2) 230 | # h = 300 231 | # w = 300 232 | # print(h, w, x, y) 233 | # small = small[y:y+h, x:x+w] 234 | # small = img[y:y+h, x:x+w] 235 | 236 | # Our operations on the frame come here 237 | # gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) 238 | start_time = time.time() 239 | image_with_box = detect_objects(small, sess, detection_graph) 240 | # print("--- %s seconds for frame ---" % (time.time() - start_time)) 241 | # Display the resulting frame 242 | cv2.imshow('frame', image_with_box) 243 | 244 | if cv2.waitKey(1) & 0xFF == ord('q'): 245 | break 246 | 247 | # When everything done, release the capture 248 | cap.release() 249 | cv2.destroyAllWindows() 250 | -------------------------------------------------------------------------------- /custom_model/pipeline.config: -------------------------------------------------------------------------------- 1 | # SSD with Mobilenet v1 0.75 depth multiplied feature extractor, focal loss and 2 | # quantized training. 3 | # Trained on IIIT-Oxford pets, initialized from COCO detection checkpoint 4 | 5 | # This config is TPU compatible 6 | 7 | model { 8 | ssd { 9 | inplace_batchnorm_update: true 10 | freeze_batchnorm: false 11 | num_classes: 5 12 | box_coder { 13 | faster_rcnn_box_coder { 14 | y_scale: 10.0 15 | x_scale: 10.0 16 | height_scale: 5.0 17 | width_scale: 5.0 18 | } 19 | } 20 | matcher { 21 | argmax_matcher { 22 | matched_threshold: 0.5 23 | unmatched_threshold: 0.5 24 | ignore_thresholds: false 25 | negatives_lower_than_unmatched: true 26 | force_match_for_each_row: true 27 | use_matmul_gather: true 28 | } 29 | } 30 | similarity_calculator { 31 | iou_similarity { 32 | } 33 | } 34 | encode_background_as_zeros: true 35 | anchor_generator { 36 | ssd_anchor_generator { 37 | num_layers: 6 38 | min_scale: 0.2 39 | max_scale: 0.95 40 | aspect_ratios: 1.0 41 | aspect_ratios: 2.0 42 | aspect_ratios: 0.5 43 | aspect_ratios: 3.0 44 | aspect_ratios: 0.3333 45 | } 46 | } 47 | image_resizer { 48 | fixed_shape_resizer { 49 | height: 300 50 | width: 300 51 | } 52 | } 53 | box_predictor { 54 | convolutional_box_predictor { 55 | min_depth: 0 56 | max_depth: 0 57 | num_layers_before_predictor: 0 58 | use_dropout: false 59 | dropout_keep_probability: 0.8 60 | kernel_size: 1 61 | box_code_size: 4 62 | apply_sigmoid_to_scores: false 63 | class_prediction_bias_init: -4.6 64 | conv_hyperparams { 65 | activation: RELU_6, 66 | regularizer { 67 | l2_regularizer { 68 | weight: 0.00004 69 | } 70 | } 71 | initializer { 72 | random_normal_initializer { 73 | stddev: 0.01 74 | mean: 0.0 75 | } 76 | } 77 | batch_norm { 78 | train: true, 79 | scale: true, 80 | center: true, 81 | decay: 0.9, 82 | epsilon: 0.001, 83 | } 84 | } 85 | } 86 | } 87 | feature_extractor { 88 | type: 'ssd_mobilenet_v1' 89 | min_depth: 16 90 | depth_multiplier: 0.75 91 | conv_hyperparams { 92 | activation: RELU_6, 93 | regularizer { 94 | l2_regularizer { 95 | weight: 0.00004 96 | } 97 | } 98 | initializer { 99 | truncated_normal_initializer { 100 | stddev: 0.03 101 | mean: 0.0 102 | } 103 | } 104 | batch_norm { 105 | scale: true, 106 | center: true, 107 | decay: 0.9, 108 | epsilon: 0.001, 109 | } 110 | } 111 | override_base_feature_extractor_hyperparams: true 112 | } 113 | loss { 114 | classification_loss { 115 | weighted_sigmoid_focal { 116 | alpha: 0.75, 117 | gamma: 2.0 118 | } 119 | } 120 | localization_loss { 121 | weighted_smooth_l1 { 122 | delta: 1.0 123 | } 124 | } 125 | classification_weight: 1.0 126 | localization_weight: 1.0 127 | } 128 | normalize_loss_by_num_matches: true 129 | normalize_loc_loss_by_codesize: true 130 | post_processing { 131 | batch_non_max_suppression { 132 | score_threshold: 1e-8 133 | iou_threshold: 0.6 134 | max_detections_per_class: 100 135 | max_total_detections: 100 136 | } 137 | score_converter: SIGMOID 138 | } 139 | } 140 | } 141 | 142 | train_config: { 143 | fine_tune_checkpoint: "gs://gcloud-iot-edge-tf-records/data/model.ckpt" 144 | fine_tune_checkpoint_type: "detection" 145 | load_all_detection_checkpoint_vars: true 146 | batch_size: 128 147 | sync_replicas: true 148 | startup_delay_steps: 0 149 | replicas_to_aggregate: 8 150 | num_steps: 10000 151 | data_augmentation_options { 152 | random_horizontal_flip { 153 | } 154 | } 155 | data_augmentation_options { 156 | ssd_random_crop { 157 | } 158 | } 159 | optimizer { 160 | momentum_optimizer: { 161 | learning_rate: { 162 | cosine_decay_learning_rate { 163 | learning_rate_base: 0.2 164 | total_steps: 10000 165 | warmup_steps: 0 166 | } 167 | } 168 | momentum_optimizer_value: 0.9 169 | } 170 | use_moving_average: false 171 | } 172 | max_number_of_boxes: 100 173 | unpad_groundtruth_tensors: false 174 | } 175 | 176 | train_input_reader: { 177 | tf_record_input_reader { 178 | input_path: "gs://gcloud-iot-edge-tf-records/data/train.record" 179 | } 180 | label_map_path: "gs://gcloud-iot-edge-tf-records/data/pascal_label_map.pbtxt" 181 | } 182 | 183 | eval_config: { 184 | metrics_set: "coco_detection_metrics" 185 | use_moving_averages: false 186 | num_examples: 1100 187 | } 188 | 189 | eval_input_reader: { 190 | tf_record_input_reader { 191 | input_path: "gs://gcloud-iot-edge-tf-records/data/val.record" 192 | } 193 | label_map_path: "gs://gcloud-iot-edge-tf-records/data/pascal_label_map.pbtxt" 194 | shuffle: false 195 | num_readers: 1 196 | } 197 | 198 | graph_rewriter { 199 | quantization { 200 | delay: 1800 201 | activation_bits: 8 202 | weight_bits: 8 203 | } 204 | } -------------------------------------------------------------------------------- /custom_model/run_cloudml.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export GCS_BUCKET=gcloud-iot-edge-tf-records 3 | export TENSORFLOWLIB=/Users/alvaroviebrantz/Documents/Desenvolvimento/Python/models/research 4 | 5 | gcloud ml-engine jobs submit training `whoami`_object_detection_`date +%s` \ 6 | --job-dir=gs://${GCS_BUCKET}/train \ 7 | --packages ${TENSORFLOWLIB}/dist/object_detection-0.1.tar.gz,${TENSORFLOWLIB}/slim/dist/slim-0.1.tar.gz,/tmp/pycocotools/pycocotools-2.0.tar.gz \ 8 | --module-name object_detection.model_tpu_main \ 9 | --runtime-version 1.12 \ 10 | --python-version 3.5 \ 11 | --scale-tier BASIC_TPU \ 12 | --region us-central1 \ 13 | -- \ 14 | --model_dir=gs://${GCS_BUCKET}/train \ 15 | --tpu_zone us-central1 \ 16 | --pipeline_config_path=gs://${GCS_BUCKET}/data/pipeline.config -------------------------------------------------------------------------------- /custom_model/ssd_mobilenet_v1_coco_11_06_2017/graph.pbtxt: -------------------------------------------------------------------------------- 1 | item { 2 | name: "/m/01g317" 3 | id: 1 4 | display_name: "person" 5 | } 6 | item { 7 | name: "/m/0199g" 8 | id: 2 9 | display_name: "bicycle" 10 | } 11 | item { 12 | name: "/m/0k4j" 13 | id: 3 14 | display_name: "car" 15 | } 16 | item { 17 | name: "/m/04_sv" 18 | id: 4 19 | display_name: "motorcycle" 20 | } 21 | item { 22 | name: "/m/05czz6l" 23 | id: 5 24 | display_name: "airplane" 25 | } 26 | item { 27 | name: "/m/01bjv" 28 | id: 6 29 | display_name: "bus" 30 | } 31 | item { 32 | name: "/m/07jdr" 33 | id: 7 34 | display_name: "train" 35 | } 36 | item { 37 | name: "/m/07r04" 38 | id: 8 39 | display_name: "truck" 40 | } 41 | item { 42 | name: "/m/019jd" 43 | id: 9 44 | display_name: "boat" 45 | } 46 | item { 47 | name: "/m/015qff" 48 | id: 10 49 | display_name: "traffic light" 50 | } 51 | item { 52 | name: "/m/01pns0" 53 | id: 11 54 | display_name: "fire hydrant" 55 | } 56 | item { 57 | name: "/m/02pv19" 58 | id: 13 59 | display_name: "stop sign" 60 | } 61 | item { 62 | name: "/m/015qbp" 63 | id: 14 64 | display_name: "parking meter" 65 | } 66 | item { 67 | name: "/m/0cvnqh" 68 | id: 15 69 | display_name: "bench" 70 | } 71 | item { 72 | name: "/m/015p6" 73 | id: 16 74 | display_name: "bird" 75 | } 76 | item { 77 | name: "/m/01yrx" 78 | id: 17 79 | display_name: "cat" 80 | } 81 | item { 82 | name: "/m/0bt9lr" 83 | id: 18 84 | display_name: "dog" 85 | } 86 | item { 87 | name: "/m/03k3r" 88 | id: 19 89 | display_name: "horse" 90 | } 91 | item { 92 | name: "/m/07bgp" 93 | id: 20 94 | display_name: "sheep" 95 | } 96 | item { 97 | name: "/m/01xq0k1" 98 | id: 21 99 | display_name: "cow" 100 | } 101 | item { 102 | name: "/m/0bwd_0j" 103 | id: 22 104 | display_name: "elephant" 105 | } 106 | item { 107 | name: "/m/01dws" 108 | id: 23 109 | display_name: "bear" 110 | } 111 | item { 112 | name: "/m/0898b" 113 | id: 24 114 | display_name: "zebra" 115 | } 116 | item { 117 | name: "/m/03bk1" 118 | id: 25 119 | display_name: "giraffe" 120 | } 121 | item { 122 | name: "/m/01940j" 123 | id: 27 124 | display_name: "backpack" 125 | } 126 | item { 127 | name: "/m/0hnnb" 128 | id: 28 129 | display_name: "umbrella" 130 | } 131 | item { 132 | name: "/m/080hkjn" 133 | id: 31 134 | display_name: "handbag" 135 | } 136 | item { 137 | name: "/m/01rkbr" 138 | id: 32 139 | display_name: "tie" 140 | } 141 | item { 142 | name: "/m/01s55n" 143 | id: 33 144 | display_name: "suitcase" 145 | } 146 | item { 147 | name: "/m/02wmf" 148 | id: 34 149 | display_name: "frisbee" 150 | } 151 | item { 152 | name: "/m/071p9" 153 | id: 35 154 | display_name: "skis" 155 | } 156 | item { 157 | name: "/m/06__v" 158 | id: 36 159 | display_name: "snowboard" 160 | } 161 | item { 162 | name: "/m/018xm" 163 | id: 37 164 | display_name: "sports ball" 165 | } 166 | item { 167 | name: "/m/02zt3" 168 | id: 38 169 | display_name: "kite" 170 | } 171 | item { 172 | name: "/m/03g8mr" 173 | id: 39 174 | display_name: "baseball bat" 175 | } 176 | item { 177 | name: "/m/03grzl" 178 | id: 40 179 | display_name: "baseball glove" 180 | } 181 | item { 182 | name: "/m/06_fw" 183 | id: 41 184 | display_name: "skateboard" 185 | } 186 | item { 187 | name: "/m/019w40" 188 | id: 42 189 | display_name: "surfboard" 190 | } 191 | item { 192 | name: "/m/0dv9c" 193 | id: 43 194 | display_name: "tennis racket" 195 | } 196 | item { 197 | name: "/m/04dr76w" 198 | id: 44 199 | display_name: "bottle" 200 | } 201 | item { 202 | name: "/m/09tvcd" 203 | id: 46 204 | display_name: "wine glass" 205 | } 206 | item { 207 | name: "/m/08gqpm" 208 | id: 47 209 | display_name: "cup" 210 | } 211 | item { 212 | name: "/m/0dt3t" 213 | id: 48 214 | display_name: "fork" 215 | } 216 | item { 217 | name: "/m/04ctx" 218 | id: 49 219 | display_name: "knife" 220 | } 221 | item { 222 | name: "/m/0cmx8" 223 | id: 50 224 | display_name: "spoon" 225 | } 226 | item { 227 | name: "/m/04kkgm" 228 | id: 51 229 | display_name: "bowl" 230 | } 231 | item { 232 | name: "/m/09qck" 233 | id: 52 234 | display_name: "banana" 235 | } 236 | item { 237 | name: "/m/014j1m" 238 | id: 53 239 | display_name: "apple" 240 | } 241 | item { 242 | name: "/m/0l515" 243 | id: 54 244 | display_name: "sandwich" 245 | } 246 | item { 247 | name: "/m/0cyhj_" 248 | id: 55 249 | display_name: "orange" 250 | } 251 | item { 252 | name: "/m/0hkxq" 253 | id: 56 254 | display_name: "broccoli" 255 | } 256 | item { 257 | name: "/m/0fj52s" 258 | id: 57 259 | display_name: "carrot" 260 | } 261 | item { 262 | name: "/m/01b9xk" 263 | id: 58 264 | display_name: "hot dog" 265 | } 266 | item { 267 | name: "/m/0663v" 268 | id: 59 269 | display_name: "pizza" 270 | } 271 | item { 272 | name: "/m/0jy4k" 273 | id: 60 274 | display_name: "donut" 275 | } 276 | item { 277 | name: "/m/0fszt" 278 | id: 61 279 | display_name: "cake" 280 | } 281 | item { 282 | name: "/m/01mzpv" 283 | id: 62 284 | display_name: "chair" 285 | } 286 | item { 287 | name: "/m/02crq1" 288 | id: 63 289 | display_name: "couch" 290 | } 291 | item { 292 | name: "/m/03fp41" 293 | id: 64 294 | display_name: "potted plant" 295 | } 296 | item { 297 | name: "/m/03ssj5" 298 | id: 65 299 | display_name: "bed" 300 | } 301 | item { 302 | name: "/m/04bcr3" 303 | id: 67 304 | display_name: "dining table" 305 | } 306 | item { 307 | name: "/m/09g1w" 308 | id: 70 309 | display_name: "toilet" 310 | } 311 | item { 312 | name: "/m/07c52" 313 | id: 72 314 | display_name: "tv" 315 | } 316 | item { 317 | name: "/m/01c648" 318 | id: 73 319 | display_name: "laptop" 320 | } 321 | item { 322 | name: "/m/020lf" 323 | id: 74 324 | display_name: "mouse" 325 | } 326 | item { 327 | name: "/m/0qjjc" 328 | id: 75 329 | display_name: "remote" 330 | } 331 | item { 332 | name: "/m/01m2v" 333 | id: 76 334 | display_name: "keyboard" 335 | } 336 | item { 337 | name: "/m/050k8" 338 | id: 77 339 | display_name: "cell phone" 340 | } 341 | item { 342 | name: "/m/0fx9l" 343 | id: 78 344 | display_name: "microwave" 345 | } 346 | item { 347 | name: "/m/029bxz" 348 | id: 79 349 | display_name: "oven" 350 | } 351 | item { 352 | name: "/m/01k6s3" 353 | id: 80 354 | display_name: "toaster" 355 | } 356 | item { 357 | name: "/m/0130jx" 358 | id: 81 359 | display_name: "sink" 360 | } 361 | item { 362 | name: "/m/040b_t" 363 | id: 82 364 | display_name: "refrigerator" 365 | } 366 | item { 367 | name: "/m/0bt_c3" 368 | id: 84 369 | display_name: "book" 370 | } 371 | item { 372 | name: "/m/01x3z" 373 | id: 85 374 | display_name: "clock" 375 | } 376 | item { 377 | name: "/m/02s195" 378 | id: 86 379 | display_name: "vase" 380 | } 381 | item { 382 | name: "/m/01lsmm" 383 | id: 87 384 | display_name: "scissors" 385 | } 386 | item { 387 | name: "/m/0kmg4" 388 | id: 88 389 | display_name: "teddy bear" 390 | } 391 | item { 392 | name: "/m/03wvsk" 393 | id: 89 394 | display_name: "hair drier" 395 | } 396 | item { 397 | name: "/m/012xff" 398 | id: 90 399 | display_name: "toothbrush" 400 | } -------------------------------------------------------------------------------- /custom_model/ssd_mobilenet_v1_coco_11_06_2017/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/custom_model/ssd_mobilenet_v1_coco_11_06_2017/model.ckpt.index -------------------------------------------------------------------------------- /custom_model/ssd_mobilenet_v1_coco_11_06_2017/model.ckpt.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/custom_model/ssd_mobilenet_v1_coco_11_06_2017/model.ckpt.meta -------------------------------------------------------------------------------- /custom_model/tensorboard.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export GCS_BUCKET=gcloud-iot-edge-tf-records 3 | 4 | tensorboard --logdir=gs://${GCS_BUCKET}/train 5 | -------------------------------------------------------------------------------- /custom_model/test/popcorn_test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/custom_model/test/popcorn_test.png -------------------------------------------------------------------------------- /custom_model/tfjs-models/group1-shard1of6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/custom_model/tfjs-models/group1-shard1of6 -------------------------------------------------------------------------------- /custom_model/tfjs-models/group1-shard2of6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/custom_model/tfjs-models/group1-shard2of6 -------------------------------------------------------------------------------- /custom_model/tfjs-models/group1-shard3of6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/custom_model/tfjs-models/group1-shard3of6 -------------------------------------------------------------------------------- /custom_model/tfjs-models/group1-shard4of6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/custom_model/tfjs-models/group1-shard4of6 -------------------------------------------------------------------------------- /custom_model/tfjs-models/group1-shard5of6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/custom_model/tfjs-models/group1-shard5of6 -------------------------------------------------------------------------------- /custom_model/tfjs-models/group1-shard6of6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/custom_model/tfjs-models/group1-shard6of6 -------------------------------------------------------------------------------- /custom_model/training/cloud.yml: -------------------------------------------------------------------------------- 1 | trainingInput: 2 | runtimeVersion: "1.0" 3 | scaleTier: CUSTOM 4 | masterType: standard_gpu 5 | workerCount: 5 6 | workerType: standard_gpu 7 | parameterServerCount: 3 8 | parameterServerType: standard -------------------------------------------------------------------------------- /custom_model/training/faster_rcnn_resnet101_coco.config: -------------------------------------------------------------------------------- 1 | # Faster R-CNN with Resnet-101 (v1), configuration for MSCOCO Dataset. 2 | # Users should configure the fine_tune_checkpoint field in the train config as 3 | # well as the label_map_path and input_path fields in the train_input_reader and 4 | # eval_input_reader. Search for "gs://gcloud-iot-edge-tf-records/data" to find the fields that 5 | # should be configured. 6 | 7 | model { 8 | faster_rcnn { 9 | num_classes: 5 10 | image_resizer { 11 | keep_aspect_ratio_resizer { 12 | min_dimension: 600 13 | max_dimension: 1024 14 | } 15 | } 16 | feature_extractor { 17 | type: 'faster_rcnn_resnet101' 18 | first_stage_features_stride: 16 19 | } 20 | first_stage_anchor_generator { 21 | grid_anchor_generator { 22 | scales: [0.25, 0.5, 1.0, 2.0] 23 | aspect_ratios: [0.5, 1.0, 2.0] 24 | height_stride: 16 25 | width_stride: 16 26 | } 27 | } 28 | first_stage_box_predictor_conv_hyperparams { 29 | op: CONV 30 | regularizer { 31 | l2_regularizer { 32 | weight: 0.0 33 | } 34 | } 35 | initializer { 36 | truncated_normal_initializer { 37 | stddev: 0.01 38 | } 39 | } 40 | } 41 | first_stage_nms_score_threshold: 0.0 42 | first_stage_nms_iou_threshold: 0.7 43 | first_stage_max_proposals: 300 44 | first_stage_localization_loss_weight: 2.0 45 | first_stage_objectness_loss_weight: 1.0 46 | initial_crop_size: 14 47 | maxpool_kernel_size: 2 48 | maxpool_stride: 2 49 | second_stage_box_predictor { 50 | mask_rcnn_box_predictor { 51 | use_dropout: false 52 | dropout_keep_probability: 1.0 53 | fc_hyperparams { 54 | op: FC 55 | regularizer { 56 | l2_regularizer { 57 | weight: 0.0 58 | } 59 | } 60 | initializer { 61 | variance_scaling_initializer { 62 | factor: 1.0 63 | uniform: true 64 | mode: FAN_AVG 65 | } 66 | } 67 | } 68 | } 69 | } 70 | second_stage_post_processing { 71 | batch_non_max_suppression { 72 | score_threshold: 0.0 73 | iou_threshold: 0.6 74 | max_detections_per_class: 100 75 | max_total_detections: 300 76 | } 77 | score_converter: SOFTMAX 78 | } 79 | second_stage_localization_loss_weight: 2.0 80 | second_stage_classification_loss_weight: 1.0 81 | } 82 | } 83 | 84 | train_config: { 85 | batch_size: 1 86 | optimizer { 87 | momentum_optimizer: { 88 | learning_rate: { 89 | manual_step_learning_rate { 90 | initial_learning_rate: 0.0003 91 | schedule { 92 | step: 900000 93 | learning_rate: .00003 94 | } 95 | schedule { 96 | step: 1200000 97 | learning_rate: .000003 98 | } 99 | } 100 | } 101 | momentum_optimizer_value: 0.9 102 | } 103 | use_moving_average: false 104 | } 105 | gradient_clipping_by_norm: 10.0 106 | fine_tune_checkpoint: "gs://gcloud-iot-edge-tf-records/data/model.ckpt" 107 | from_detection_checkpoint: true 108 | data_augmentation_options { 109 | random_horizontal_flip { 110 | } 111 | } 112 | } 113 | 114 | train_input_reader: { 115 | tf_record_input_reader { 116 | input_path: "gs://gcloud-iot-edge-tf-records/data/train.record" 117 | } 118 | label_map_path: "gs://gcloud-iot-edge-tf-records/data/pascal_label_map.pbtxt" 119 | } 120 | 121 | eval_config: { 122 | num_examples: 8000 123 | # Note: The below line limits the evaluation process to 10 evaluations. 124 | # Remove the below line to evaluate indefinitely. 125 | max_evals: 10 126 | } 127 | 128 | eval_input_reader: { 129 | tf_record_input_reader { 130 | input_path: "gs://gcloud-iot-edge-tf-records/data/val.record" 131 | } 132 | label_map_path: "gs://gcloud-iot-edge-tf-records/data/pascal_label_map.pbtxt" 133 | shuffle: false 134 | num_readers: 1 135 | } -------------------------------------------------------------------------------- /custom_model/training/pascal_label_map.pbtxt: -------------------------------------------------------------------------------- 1 | item { 2 | id: 1 3 | name: 'raspberry' 4 | } 5 | 6 | item { 7 | id: 2 8 | name: 'muffin' 9 | } 10 | 11 | item { 12 | id: 3 13 | name: 'berry' 14 | } 15 | 16 | item { 17 | id: 4 18 | name: 'popcorn' 19 | } 20 | 21 | item { 22 | id: 5 23 | name: 'jam' 24 | } -------------------------------------------------------------------------------- /database.rules.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | ".read": "true", 4 | ".write": "true" 5 | } 6 | } -------------------------------------------------------------------------------- /edge-server/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | *.log -------------------------------------------------------------------------------- /edge-server/BaseClassifier.js: -------------------------------------------------------------------------------- 1 | const tf = require('@tensorflow/tfjs') 2 | 3 | const fs = require('fs') 4 | const jpeg = require('jpeg-js') 5 | 6 | class BaseClassifier { 7 | 8 | readImage(path) { 9 | const buf = fs.readFileSync(path) 10 | const pixels = jpeg.decode(buf, true) 11 | return pixels 12 | } 13 | 14 | imageByteArray(image, numChannels) { 15 | const pixels = image.data 16 | const numPixels = image.width * image.height 17 | const values = new Int32Array(numPixels * numChannels) 18 | 19 | for (let i = 0; i < numPixels; i++) { 20 | for (let channel = 0; channel < numChannels; ++channel) { 21 | values[i * numChannels + channel] = pixels[i * 4 + channel] 22 | } 23 | } 24 | 25 | return values 26 | } 27 | 28 | imageToInput(image, numChannels) { 29 | const values = this.imageByteArray(image, numChannels) 30 | const outShape = [image.height, image.width, numChannels] 31 | const input = tf.tensor3d(values, outShape, 'int32') 32 | 33 | return input 34 | } 35 | 36 | async classifyFromFile(path) { 37 | const image = this.readImage(path) 38 | return this.classifyFromImage(image) 39 | } 40 | 41 | async classifyFromImage(image) { 42 | throw new Error('Not implemented') 43 | } 44 | } 45 | 46 | module.exports = BaseClassifier 47 | -------------------------------------------------------------------------------- /edge-server/CatClassifier.js: -------------------------------------------------------------------------------- 1 | const tf = require('@tensorflow/tfjs') 2 | 3 | const BaseClassifier = require('./BaseClassifier') 4 | 5 | const NUMBER_OF_CHANNELS = 3 6 | 7 | class CatClassifier extends BaseClassifier { 8 | /* 9 | * @param mode - detect or classify model 10 | */ 11 | constructor({ modelPath, labels }) { 12 | super() 13 | this.modelPath = modelPath 14 | this.labels = labels 15 | } 16 | 17 | async load() { 18 | const model = await tf.loadGraphModel(this.modelPath, { 19 | strict: false, 20 | }) 21 | // model.summary() 22 | this.model = model 23 | } 24 | 25 | async classifyFromImage(image) { 26 | let input = this.imageToInput(image, NUMBER_OF_CHANNELS) 27 | input = tf.image.resizeNearestNeighbor(input, [150, 150]) 28 | input = input.expandDims(0).div(255.0) 29 | return this.classifyFromInput(input) 30 | } 31 | 32 | async classifyFromInput(input) { 33 | const output = await this.model.executeAsync(input) 34 | output.print() 35 | const prediction = output.argMax(1).dataSync() 36 | const labelIndex = prediction[0] 37 | const score = output.dataSync()[labelIndex] 38 | tf.dispose(input) 39 | return { class: this.labels[labelIndex], score } 40 | } 41 | } 42 | 43 | module.exports = CatClassifier 44 | -------------------------------------------------------------------------------- /edge-server/CatDetector.js: -------------------------------------------------------------------------------- 1 | const tf = require('@tensorflow/tfjs') 2 | const jpeg = require('jpeg-js') 3 | const fs = require('fs') 4 | 5 | const BaseClassifier = require('./BaseClassifier') 6 | const CatClassifier = require('./CatClassifier') 7 | const ImageClassifier = require('./ImageClassifier') 8 | 9 | class CatDetector extends BaseClassifier { 10 | 11 | constructor({ modelPath, labels, threshold, trackingTags }) { 12 | super() 13 | const mode = 'detect' 14 | this.classifier = new CatClassifier({ labels, modelPath }) 15 | this.detector = new ImageClassifier({ mode, threshold, trackingTags }) 16 | this.groupClasses = this.detector.groupClasses.bind(this.detector) 17 | } 18 | 19 | async load() { 20 | await this.classifier.load() 21 | await this.detector.load() 22 | } 23 | 24 | saveImage(tensor, path) { 25 | const [batch, height, width, channels] = tensor.shape 26 | //create an Image data var 27 | const buffer = Buffer.alloc(width * height * 4) 28 | //get the tensor values as data 29 | const data = tensor.dataSync() 30 | //map the values to the buffer 31 | let i = 0 32 | for (let y = 0; y < height; y++) { 33 | for (let x = 0; x < width; x++) { 34 | const pos = (y * width + x) * 4 // position in buffer based on x and y 35 | buffer[pos] = data[i] // some R value [0, 255] 36 | buffer[pos + 1] = data[i + 1] // some G value 37 | buffer[pos + 2] = data[i + 2] // some B value 38 | buffer[pos + 3] = 255 // set alpha channel 39 | i += 3 40 | } 41 | } 42 | 43 | const rawImageData = { 44 | data: buffer, 45 | width: width, 46 | height: height 47 | }; 48 | const jpegImageData = jpeg.encode(rawImageData, 50) 49 | fs.writeFile(path, jpegImageData.data, err => { 50 | 51 | }) 52 | } 53 | 54 | 55 | async classifyFromImage(image) { 56 | const input = this.classifier.imageToInput(image, 3) 57 | const objects = await this.detector.classifyFromImage(image) 58 | 59 | //console.log(objects) 60 | 61 | const classifyInput = input 62 | .asType('float32') 63 | .expandDims(0) 64 | 65 | const [height, width, channels] = input.shape 66 | 67 | const promises = objects 68 | .filter(obj => obj.class === 'cat') 69 | .map(async (obj, i) => { 70 | const [x1, y1, w, h] = obj.bbox 71 | const y2 = (y1 + h) / height 72 | const x2 = (x1 + w) / width 73 | const box = [[y1 / height, x1 / width, y2, x2]] 74 | const crop = tf.image.cropAndResize(classifyInput, box, [0], [h, w]) 75 | 76 | const result = await this.classifier.classifyFromInput(crop.div(255.0)) 77 | //console.log(result) 78 | objects.push(result) 79 | this.saveImage(crop, `test_${i}.jpeg`) 80 | }) 81 | await Promise.all(promises) 82 | tf.dispose(input) 83 | return objects 84 | } 85 | } 86 | 87 | module.exports = CatDetector 88 | -------------------------------------------------------------------------------- /edge-server/CloudIoTCoreGateway.js: -------------------------------------------------------------------------------- 1 | const fs = require( 'fs' ) 2 | const jwt = require( 'jsonwebtoken' ) 3 | const mqtt = require( 'async-mqtt' ) 4 | 5 | const createLogger = require( './Log' ) 6 | 7 | const logger = createLogger( 'CloudIoTCoreGateway' ) 8 | 9 | class CloudIoTCoreGateway { 10 | constructor( { projectId, cloudRegion, registryId, gatewayId, privateKeyFile } ) { 11 | this.projectId = projectId 12 | this.cloudRegion = cloudRegion 13 | this.registryId = registryId 14 | this.gatewayId = gatewayId 15 | this.privateKeyFile = privateKeyFile 16 | 17 | this.tokenExpMins = 20 // Token expiration time in minutes 18 | } 19 | 20 | start() { 21 | this.connect() 22 | this.connectionTicker = setInterval( this.checkConnection.bind( this ), 10000 ) 23 | } 24 | 25 | stop() { 26 | logger.info( 'Closing...' ) 27 | if ( this.client ) { 28 | this.client.end() 29 | } 30 | if ( this.connectionTicker ) { 31 | clearInterval( this.connectionTicker ) 32 | } 33 | logger.info( 'Done' ) 34 | } 35 | 36 | connect() { 37 | if ( this.client ) { 38 | this.client.end() 39 | } 40 | 41 | // Cloud iot core requires a specific client id 42 | const clientId = `projects/${this.projectId}/locations/${ 43 | this.cloudRegion 44 | }/registries/${this.registryId}/devices/${this.gatewayId}` 45 | 46 | const connectionArgs = { 47 | host : 'mqtt.googleapis.com', 48 | port : 8883, 49 | clientId, 50 | username : 'unused', 51 | password : this.createJwt(), 52 | protocol : 'mqtts', 53 | secureProtocol : 'TLSv1_2_method', 54 | } 55 | 56 | // Create a client, and connect to the Google MQTT bridge 57 | this.iatTime = parseInt( Date.now() / 1000 ) 58 | this.client = mqtt.connect( connectionArgs ) 59 | this.client.on( 'connect', ( success ) => { 60 | if ( success ) { 61 | logger.info( 'Client connected.' ) 62 | } else { 63 | logger.info( 'Client not connected.' ) 64 | } 65 | } ) 66 | } 67 | 68 | checkConnection() { 69 | const secsFromIssue = parseInt( Date.now() / 1000 ) - this.iatTime 70 | if ( secsFromIssue > this.tokenExpMins * 60 ) { 71 | logger.info( `Refreshing token after ${secsFromIssue} seconds.` ) 72 | this.connect() 73 | } 74 | } 75 | 76 | /* 77 | * Create a Cloud IoT Core JWT for the given project id, signed with the given 78 | * private key. 79 | */ 80 | createJwt() { 81 | const algorithm = 'ES256' 82 | const token = { 83 | iat : parseInt( Date.now() / 1000 ), 84 | exp : parseInt( Date.now() / 1000 ) + this.tokenExpMins * 60, 85 | aud : this.projectId, 86 | } 87 | const privateKey = fs.readFileSync( this.privateKeyFile ) 88 | return jwt.sign( token, privateKey, { algorithm } ) 89 | } 90 | 91 | async publish( deviceId, payload, eventType ) { 92 | let finalPayload = payload 93 | if ( payload instanceof Object ) { 94 | finalPayload = JSON.stringify( payload ) 95 | } 96 | 97 | const mqttTopic = `/devices/${deviceId}/${eventType}` 98 | return this.client.publish( mqttTopic, finalPayload, { qos : 0 } ) 99 | } 100 | 101 | attachDevice( deviceId ) { 102 | return this.publish( deviceId, {}, 'attach' ) 103 | } 104 | 105 | detachDevice( deviceId ) { 106 | return this.publish( deviceId, {}, 'detach' ) 107 | } 108 | 109 | publishDeviceTelemetry( deviceId, payload ) { 110 | return this.publish( deviceId, payload, 'events' ) 111 | } 112 | 113 | publishDeviceState( deviceId, payload ) { 114 | return this.publish( deviceId, payload, 'state' ) 115 | } 116 | 117 | publishGatewayTelemetry( payload ) { 118 | return this.publish( this.gatewayId, payload, 'events' ) 119 | } 120 | 121 | publishGatewayState( payload ) { 122 | return this.publish( this.gatewayId, payload, 'state' ) 123 | } 124 | } 125 | 126 | module.exports = CloudIoTCoreGateway 127 | -------------------------------------------------------------------------------- /edge-server/DeviceListener.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events') 2 | const mdns = require('mdns') 3 | const AbortController = require('abort-controller') 4 | const fetch = require('node-fetch') 5 | const jpeg = require('jpeg-js') 6 | 7 | const createLogger = require('./Log') 8 | 9 | const logger = createLogger('DeviceListener') 10 | 11 | class CameraDevice { 12 | constructor(service) { 13 | this.name = service.name 14 | this.host = service.host 15 | this.addresses = service.addresses 16 | } 17 | 18 | /* 19 | * Using ip address directly is faster 20 | */ 21 | getImageUrl() { 22 | let host = `${this.name}.local` 23 | if (this.addresses && this.addresses.length > 0) { 24 | host = `${this.addresses[0]}` 25 | } 26 | return `http://${host}/jpg` 27 | } 28 | 29 | async fetchImage() { 30 | const controller = new AbortController() 31 | const timeout = setTimeout( 32 | () => { controller.abort() }, 33 | 5000, 34 | ) 35 | 36 | const url = this.getImageUrl() 37 | try { 38 | const res = await fetch(url, { signal: controller.signal }) 39 | const buffer = await res.arrayBuffer() 40 | const image = jpeg.decode(buffer, true) 41 | return image 42 | } catch (e) { 43 | throw new Error('Error Fetching Image') 44 | } finally { 45 | clearTimeout(timeout) 46 | } 47 | } 48 | } 49 | 50 | 51 | class DeviceListener extends EventEmitter { 52 | constructor() { 53 | super() 54 | const sequence = [ 55 | mdns.rst.DNSServiceResolve(), 56 | 'DNSServiceGetAddrInfo' in mdns.dns_sd ? mdns.rst.DNSServiceGetAddrInfo() : mdns.rst.getaddrinfo({ families: [0] }), 57 | mdns.rst.makeAddressesUnique() 58 | ] 59 | this.browser = mdns.createBrowser(mdns.tcp('camera'), { resolverSequence: sequence }) 60 | this.devices = {} 61 | } 62 | 63 | start() { 64 | this.browser.start() 65 | this.browser.on('serviceUp', (service) => { 66 | const { name } = service 67 | logger.info(`service up: ${name}`) 68 | this.devices[name] = new CameraDevice(service) 69 | this.emit('deviceAdded', name) 70 | }) 71 | this.browser.on('serviceDown', (service) => { 72 | const { name } = service 73 | logger.info(`service down: ${name}`) 74 | delete this.devices[name] 75 | this.emit('deviceRemoved', name) 76 | }) 77 | } 78 | 79 | onDeviceAdded(callback) { 80 | return this.on('deviceAdded', callback) 81 | } 82 | 83 | onDeviceRemoved(callback) { 84 | return this.on('deviceRemoved', callback) 85 | } 86 | 87 | stop() { 88 | logger.info('Closing...') 89 | this.browser.stop() 90 | this.removeAllListeners() 91 | logger.info('Done') 92 | } 93 | 94 | getDevices() { 95 | return Object.values(this.devices) 96 | } 97 | 98 | } 99 | 100 | module.exports = DeviceListener 101 | -------------------------------------------------------------------------------- /edge-server/EXAMPLE_BERRY.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/edge-server/EXAMPLE_BERRY.jpeg -------------------------------------------------------------------------------- /edge-server/EXAMPLE_JAM.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/edge-server/EXAMPLE_JAM.jpeg -------------------------------------------------------------------------------- /edge-server/EdgeServer.js: -------------------------------------------------------------------------------- 1 | const { RateLimiter } = require('limiter') 2 | 3 | const DeviceListener = require('./DeviceListener') 4 | const ImageClassifier = require('./ImageClassifier') 5 | const CatDetector = require('./CatDetector') 6 | const CloudIoTCoreGateway = require('./CloudIoTCoreGateway') 7 | const WebInterface = require('./WebInterface') 8 | 9 | const createLogger = require('./Log') 10 | 11 | const logger = createLogger('EdgeServer') 12 | 13 | class EdgeServer { 14 | constructor(config) { 15 | const { classifier, web, gateway } = config 16 | 17 | this.deviceListener = new DeviceListener() 18 | if (classifier.mode === 'cats') { 19 | this.classifier = new CatDetector(classifier) 20 | } else { 21 | this.classifier = new ImageClassifier(classifier) 22 | } 23 | this.gateway = new CloudIoTCoreGateway(gateway) 24 | this.web = new WebInterface(web) 25 | this.limiter = new RateLimiter(10, 'minute') 26 | 27 | this.deviceQueue = {} 28 | this.lastDeviceData = {} 29 | } 30 | 31 | async start() { 32 | this.stopped = false 33 | await this.classifier.load() 34 | this.deviceListener.start() 35 | this.web.start() 36 | await this.gateway.start() 37 | 38 | this.deviceListener.onDeviceAdded(async (deviceId) => { 39 | await this.gateway.attachDevice(deviceId) 40 | this.gateway.publishDeviceState(deviceId, { status: 'online', type: 'camera' }) 41 | }) 42 | 43 | this.deviceListener.onDeviceRemoved(async (deviceId) => { 44 | this.gateway.publishDeviceState(deviceId, { status: 'offline' }) 45 | this.gateway.detachDevice(deviceId) 46 | }) 47 | 48 | this.run() 49 | // this.ticker = setInterval( this.run.bind( this ), 10000 ) 50 | 51 | const serverInfo = this.web.getServerInfo() 52 | await this.gateway.publishGatewayState({ status: 'online', server: serverInfo, type: 'gateway' }) 53 | } 54 | 55 | hasChanges() { 56 | const devices = Object.keys(this.deviceQueue) 57 | if (devices.length !== Object.keys(this.lastDeviceData).length) { 58 | return true 59 | } 60 | const hasChanged = devices.some((deviceId) => { 61 | return JSON.stringify(this.deviceQueue[deviceId]) !== JSON.stringify(this.lastDeviceData[deviceId]) 62 | }) 63 | return hasChanged 64 | } 65 | 66 | queueData(device, { classes, trackedClasses, countClasses }) { 67 | const { name } = device 68 | let deviceData = this.deviceQueue[name] 69 | if (!deviceData) { 70 | deviceData = { 71 | name, 72 | classes: [], 73 | trackedClasses: [], 74 | countClasses: {}, 75 | } 76 | this.deviceQueue[name] = deviceData 77 | } 78 | 79 | const classesSet = new Set(deviceData.classes.concat(classes)) 80 | const nClasses = [...classesSet] 81 | 82 | const trackedClassesSet = new Set(deviceData.trackedClasses.concat(trackedClasses)) 83 | const nTrackedClasses = [...trackedClassesSet] 84 | 85 | const nCountClasses = { 86 | ...deviceData.countClasses, 87 | ...countClasses 88 | } 89 | Object.keys(nCountClasses).forEach((key) => { 90 | nCountClasses[key] = Math.max(deviceData.countClasses[key], countClasses[key]) || 1 91 | }) 92 | 93 | const nDeviceData = { 94 | name, 95 | classes: nClasses, 96 | trackedClasses: nTrackedClasses, 97 | countClasses: nCountClasses 98 | } 99 | 100 | this.deviceQueue[name] = nDeviceData 101 | } 102 | 103 | clearQueue() { 104 | this.deviceQueue = {} 105 | this.lastDeviceData = {} 106 | } 107 | 108 | async run() { 109 | const devices = this.deviceListener.getDevices() 110 | const findings = {} 111 | const promises = devices.map(async (device) => { 112 | logger.info(`Looking for image from ${device.name}`) 113 | try { 114 | logger.profile(`FetchImage-${device.name}`) 115 | const image = await device.fetchImage() 116 | logger.info(`Fetched image from ${device.name}`) 117 | logger.profile(`FetchImage-${device.name}`) 118 | 119 | logger.profile(`ClassifyImage-${device.name}`) 120 | const predictions = await this.classifier.classifyFromImage(image) 121 | logger.profile(`ClassifyImage-${device.name}`) 122 | 123 | const grouped = this.classifier.groupClasses(predictions) 124 | const { countClasses } = grouped 125 | logger.info(`Found classes ${device.name} - ${JSON.stringify(countClasses)}`) 126 | 127 | this.queueData(device, grouped) 128 | findings[device.name] = { 129 | name: device.name, 130 | ...grouped 131 | } 132 | } catch (e) { 133 | logger.error('Error fetching image from device', device.name, e.message) 134 | findings[device.name] = { 135 | name: device.name 136 | } 137 | } 138 | }) 139 | 140 | // Wait for inference results and filter for valid ones 141 | try { 142 | await Promise.all(promises) 143 | } catch (e) { 144 | logger.error('Error running inference on some devices', e.message) 145 | } 146 | 147 | // Update local web interface 148 | logger.info(`Device queue : ${JSON.stringify(this.deviceQueue)}`) 149 | logger.info(`Last device data : ${JSON.stringify(this.lastDeviceData)}`) 150 | logger.info(`Findings : ${JSON.stringify(findings)}`) 151 | const deviceData = Object.values(findings) 152 | this.web.broadcastData('devices', deviceData) 153 | 154 | // Send data to cloud iot core 155 | try { 156 | if (this.hasChanges()) { 157 | if (this.limiter.tryRemoveTokens(1)) { 158 | logger.info('[PublishData] Sending data to cloud iot core.') 159 | const publishPromises = Object.keys(this.deviceQueue).map((deviceId) => { 160 | const res = this.deviceQueue[deviceId] 161 | this.lastDeviceData[deviceId] = res 162 | return this.gateway.publishDeviceTelemetry(deviceId, { classes: res.countClasses }) 163 | }) 164 | await Promise.all(publishPromises) 165 | this.clearQueue() 166 | } else { 167 | logger.info('[PublishData] Publishing throttled.') 168 | } 169 | } 170 | } catch (err) { 171 | logger.error(`Error sending data to cloud iot core ${err}`, err) 172 | } 173 | 174 | if (!this.stopped) { 175 | setTimeout(this.run.bind(this), 100) 176 | } 177 | } 178 | 179 | async stop() { 180 | logger.info('Closing...') 181 | this.stopped = true 182 | if (this.ticker) { 183 | clearInterval(this.ticker) 184 | } 185 | 186 | const devices = this.deviceListener.getDevices() 187 | 188 | this.deviceListener.stop() 189 | this.web.stop() 190 | 191 | logger.info('Sending offline events') 192 | try { 193 | logger.info('Sending gateway offline event') 194 | await this.gateway.publishGatewayState({ status: 'offline' }) 195 | const publishPromises = devices.map((device) => { 196 | logger.info(`Sending offline event for device ${device.name}`) 197 | return this.gateway.publishDeviceState(device.name, { status: 'offline' }) 198 | }) 199 | await Promise.all(publishPromises) 200 | logger.info('All offline events sent') 201 | await new Promise(resolve => setTimeout(resolve, 2000)) 202 | } catch (err) { 203 | logger.error(`Error sending data to cloud iot core ${err}`, err) 204 | } 205 | 206 | this.gateway.stop() 207 | logger.info('Done') 208 | } 209 | } 210 | 211 | module.exports = EdgeServer 212 | -------------------------------------------------------------------------------- /edge-server/ImageClassifier.js: -------------------------------------------------------------------------------- 1 | const tf = require('@tensorflow/tfjs') 2 | 3 | const mobilenet = require('@tensorflow-models/mobilenet') 4 | const cocossd = require('@tensorflow-models/coco-ssd') 5 | 6 | const fs = require('fs') 7 | const jpeg = require('jpeg-js') 8 | const flatMap = require('flatmap') 9 | 10 | const BaseClassifier = require('./BaseClassifier') 11 | const Bag = require('./util/Bag') 12 | 13 | const NUMBER_OF_CHANNELS = 3 14 | 15 | class ImageClassifier extends BaseClassifier { 16 | /* 17 | * @param mode - detect or classify model 18 | */ 19 | constructor({ mode, trackingTags, threshold }) { 20 | super() 21 | let model = null 22 | if (mode === 'detect') { 23 | model = cocossd 24 | } else if (mode === 'classify') { 25 | model = mobilenet 26 | } else { 27 | throw new Error('Unknow classifier mode') 28 | } 29 | this.unloadedModel = model 30 | this.mode = mode 31 | this.trackingTags = trackingTags 32 | this.threshold = threshold 33 | } 34 | 35 | async load() { 36 | const model = await this.unloadedModel.load() 37 | this.model = model 38 | } 39 | 40 | mapPredictions(predictions) { 41 | return flatMap(predictions, (prediction) => { 42 | if (this.mode === 'detect') { 43 | return prediction 44 | } 45 | // this.mode === 'classify' 46 | const classes = prediction.className.split(', ') 47 | return classes.map(clss => ({ 48 | score: prediction.probability, 49 | class: clss 50 | })) 51 | }) 52 | } 53 | 54 | groupClasses(predictions) { 55 | const trackingSet = new Set(this.trackingTags) 56 | const all = new Bag() 57 | predictions.forEach((prediction) => { 58 | if (prediction.score >= this.threshold) { 59 | all.add(prediction.class) 60 | } 61 | }) 62 | const classes = all.toArray() 63 | const countClasses = all.toObject() 64 | const trackedClasses = classes.filter(x => trackingSet.has(x)) 65 | return { 66 | classes, 67 | trackedClasses, 68 | countClasses, 69 | } 70 | } 71 | 72 | async classifyFromImage(image) { 73 | const input = this.imageToInput(image, NUMBER_OF_CHANNELS) 74 | return this.classifyFromInput(input) 75 | } 76 | 77 | async classifyFromInput(input) { 78 | let predictions 79 | if (this.mode === 'detect') { 80 | predictions = await this.model.detect(input) 81 | } else { 82 | predictions = await this.model.classify(input) 83 | } 84 | tf.dispose(input) 85 | return this.mapPredictions(predictions) 86 | } 87 | } 88 | 89 | module.exports = ImageClassifier 90 | -------------------------------------------------------------------------------- /edge-server/Log.js: -------------------------------------------------------------------------------- 1 | const { createLogger, format, transports } = require( 'winston' ) 2 | 3 | const { combine, timestamp, label, printf } = format 4 | 5 | const appFormat = printf( ( info ) => { 6 | return `${info.timestamp} ${info.level}: [${info.label}] ${info.message}` 7 | } ) 8 | 9 | function create( customLabel ) { 10 | const logger = createLogger( { 11 | format : combine( 12 | label( { label : customLabel } ), 13 | timestamp(), 14 | appFormat 15 | ), 16 | exitOnError : false, 17 | transports : [ 18 | new transports.File( { filename : 'error.log', level : 'error' } ), 19 | new transports.File( { filename : 'combined.log' } ) 20 | ] 21 | } ) 22 | 23 | if ( process.env.NODE_ENV !== 'production' ) { 24 | logger.add( new transports.Console() ) 25 | } 26 | 27 | return logger 28 | } 29 | 30 | module.exports = create 31 | -------------------------------------------------------------------------------- /edge-server/WebInterface.js: -------------------------------------------------------------------------------- 1 | const express = require( 'express' ) 2 | const ip = require( 'ip' ) 3 | const { Server } = require( 'http' ) 4 | const SocketIO = require( 'socket.io' ) 5 | 6 | const createLogger = require( './Log' ) 7 | 8 | const logger = createLogger( 'WebInterface' ) 9 | 10 | class WebInterface { 11 | constructor( { port } ) { 12 | this.app = express() 13 | this.server = Server( this.app ) 14 | this.io = SocketIO( this.server ) 15 | 16 | this.port = port || 3000 17 | 18 | this.app.get( '/', this.indexHandler.bind( this ) ) 19 | 20 | } 21 | 22 | start() { 23 | this.server.listen( this.port ) 24 | } 25 | 26 | stop() { 27 | logger.info( 'Closing...' ) 28 | this.server.close( () => { 29 | logger.info( 'Done' ) 30 | } ) 31 | } 32 | 33 | getServerInfo() { 34 | return { 35 | port : this.port, 36 | ip : ip.address(), 37 | } 38 | } 39 | 40 | indexHandler( req, res ) { 41 | res.sendFile( `${__dirname}/index.html` ) 42 | } 43 | 44 | broadcastData( channel, data ) { 45 | this.io.emit( channel, data ) 46 | } 47 | } 48 | 49 | module.exports = WebInterface 50 | -------------------------------------------------------------------------------- /edge-server/edge-server-flowchart.md: -------------------------------------------------------------------------------- 1 | graph TD 2 | A[Edge Server] --> B(Device Listener) 3 | A[Edge Server] --> C(Image Classifier) 4 | A[Edge Server] --> D(CloudIoTCoreGateway) 5 | A[Edge Server] --> E(WebInterface) 6 | B --> |Search via mDns| F[fa:fa-camera Cameras] 7 | C --> |Classify with Tensorflow| G[Queue Data] 8 | D --> |Send Queue Data| H[Google Cloud] 9 | E --> |Web Socket| Z(Socket.io) 10 | E --> X(Web) 11 | -------------------------------------------------------------------------------- /edge-server/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 53 | IoT Edge Server Monitor 54 | 55 | 56 | 57 |
58 | 59 | 60 | IoT Edge - Local Monitor 61 | 62 | 63 | 65 | 66 |
67 | 68 | 69 |
{{device.name}}
70 |
{{device.updatedString}}
71 | 72 | 73 | 74 | camera 75 | 76 | 77 | 78 | 79 | {{c}}({{device.countClasses[c]}}) 80 | 81 | 82 | Nothing found. 83 | 84 | 85 | 86 |
87 | 88 | 89 |
90 | 91 | 92 | 93 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /edge-server/index.js: -------------------------------------------------------------------------------- 1 | require('@tensorflow/tfjs-node') 2 | // global.fetch = require( 'node-fetch' ) 3 | 4 | const EdgeServer = require('./EdgeServer') 5 | 6 | const config = { 7 | classifier: { 8 | trackingTags: ['cats', 'person', 'dogs'], 9 | threshold: 0.6, 10 | // mode: 'detect', 11 | mode: 'cats', 12 | labels: { 0: 'berry', 1: 'jam', 2: 'muffin', 3: 'popcorn', 4: 'raspberry' }, 13 | modelPath: 'file://../custom_model/tfjs-models/model.json' 14 | }, 15 | web: { 16 | port: 3000 17 | }, 18 | gateway: { 19 | projectId: 'gcloud-iot-edge', 20 | cloudRegion: 'us-central1', 21 | registryId: 'iot-edge-registry', 22 | gatewayId: 'gw-mark-one', 23 | privateKeyFile: '../ec_private.pem' 24 | } 25 | } 26 | 27 | const server = new EdgeServer(config) 28 | server.start() 29 | 30 | async function gracefulShutdown() { 31 | try { 32 | console.info('SIGTERM signal received.') 33 | console.info('Shutting down server.') 34 | await Promise.race([ 35 | server.stop(), 36 | new Promise((resolve, reject) => { 37 | setTimeout(() => { 38 | reject() 39 | }, 5000) 40 | }) 41 | ]) 42 | process.exit(0) 43 | } catch (err) { 44 | console.error('Forced shutdown', err) 45 | process.exit(1) 46 | } 47 | } 48 | 49 | process.on('SIGTERM', gracefulShutdown) 50 | process.on('SIGINT', gracefulShutdown) 51 | -------------------------------------------------------------------------------- /edge-server/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "edge-server", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "dependencies": { 7 | "@tensorflow-models/coco-ssd": "^1.0.2", 8 | "@tensorflow-models/mobilenet": "^1.0.1", 9 | "@tensorflow/tfjs": "^1.1.2", 10 | "@tensorflow/tfjs-node": "^1.1.2", 11 | "abort-controller": "^2.0.0", 12 | "async-mqtt": "^2.0.0", 13 | "express": "^4.16.4", 14 | "flatmap": "0.0.3", 15 | "ip": "^1.1.5", 16 | "jpeg-js": "^0.4.0", 17 | "jsonwebtoken": "^8.4.0", 18 | "limiter": "^1.1.3", 19 | "mdns": "^2.4.0", 20 | "mqtt": "^2.18.8", 21 | "node-fetch": "^2.3.0", 22 | "socket.io": "^2.2.0", 23 | "winston": "^3.1.0" 24 | }, 25 | "devDependencies": {}, 26 | "scripts": { 27 | "start": "node index.js", 28 | "test": "echo \"Error: no test specified\" && exit 1" 29 | }, 30 | "author": "", 31 | "license": "SEE LICENSE IN License.md" 32 | } 33 | -------------------------------------------------------------------------------- /edge-server/popcorn.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/edge-server/popcorn.jpeg -------------------------------------------------------------------------------- /edge-server/test_1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/edge-server/test_1.jpeg -------------------------------------------------------------------------------- /edge-server/test_2.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/edge-server/test_2.jpeg -------------------------------------------------------------------------------- /edge-server/test_3.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alvarowolfx/gcloud-iot-edge-tensorflow/af12aa60c9baf11d55b2672990aeb55afd2848a2/edge-server/test_3.jpeg -------------------------------------------------------------------------------- /edge-server/test_model.js: -------------------------------------------------------------------------------- 1 | require('@tensorflow/tfjs-node') 2 | const tf = require('@tensorflow/tfjs') 3 | const jpeg = require('jpeg-js') 4 | const fs = require('fs') 5 | 6 | const CatClassifier = require('./CatClassifier') 7 | const ImageClassifier = require('./ImageClassifier') 8 | 9 | function saveImage(tensor, path) { 10 | const [batch, height, width, channels] = tensor.shape 11 | //create an Image data var 12 | const buffer = Buffer.alloc(width * height * 4) 13 | //get the tensor values as data 14 | const data = tensor.dataSync() 15 | //map the values to the buffer 16 | let i = 0 17 | for (let y = 0; y < height; y++) { 18 | for (let x = 0; x < width; x++) { 19 | const pos = (y * width + x) * 4 // position in buffer based on x and y 20 | buffer[pos] = data[i] // some R value [0, 255] 21 | buffer[pos + 1] = data[i + 1] // some G value 22 | buffer[pos + 2] = data[i + 2] // some B value 23 | buffer[pos + 3] = 255 // set alpha channel 24 | i += 3 25 | } 26 | } 27 | 28 | const rawImageData = { 29 | data: buffer, 30 | width: width, 31 | height: height 32 | }; 33 | const jpegImageData = jpeg.encode(rawImageData, 50) 34 | fs.writeFile(path, jpegImageData.data, err => { 35 | 36 | }) 37 | } 38 | 39 | async function main() { 40 | try { 41 | const labels = { 0: 'berry', 1: 'jam', 2: 'muffin', 3: 'popcorn', 4: 'raspberry' } 42 | const modelPath = 'file://../custom_model/tfjs-models/model.json' 43 | const classifier = new CatClassifier({ labels, modelPath }) 44 | await classifier.load() 45 | 46 | const trackingTags = ['cat'] 47 | const threshold = 0.6 48 | const mode = 'detect' 49 | const detector = new ImageClassifier({ mode, threshold, trackingTags }) 50 | await detector.load() 51 | console.log("Loaded") 52 | 53 | const index = Math.floor((Math.random() * 9) + 1) 54 | const catIndex = Math.floor((Math.random() * 4)) 55 | const cat = Object.values(labels)[catIndex] 56 | //const path = `../custom_model/output/${cat}/${index}.jpg` 57 | //const path = '../custom_model/images/cat00005.jpg' 58 | const path = './test_3.jpeg' 59 | console.log(`Reading image ${path}`) 60 | 61 | const result = await classifier.classifyFromFile(path) 62 | console.log(result) 63 | 64 | return; 65 | 66 | const image = classifier.readImage(path) 67 | const input = classifier.imageToInput(image, 3) 68 | 69 | const objects = await detector.classifyFromFile(path) 70 | console.log(objects) 71 | 72 | const classifyInput = input 73 | .asType('float32') 74 | .expandDims(0) 75 | 76 | const [height, width, channels] = input.shape 77 | 78 | objects 79 | .filter(obj => obj.class === 'cat') 80 | .forEach(async (obj, i) => { 81 | const [x1, y1, w, h] = obj.bbox 82 | const y2 = (y1 + h) / height 83 | const x2 = (x1 + w) / width 84 | const box = [[y1 / height, x1 / width, y2, x2]] 85 | const crop = tf.image.cropAndResize(classifyInput, box, [0], [h, w]) 86 | 87 | const result = await classifier.classifyFromInput(crop.div(255.0)) 88 | console.log(result) 89 | saveImage(crop, `test_${i}.jpeg`) 90 | }) 91 | 92 | } catch (e) { 93 | console.error(e) 94 | } 95 | } 96 | 97 | main() 98 | -------------------------------------------------------------------------------- /edge-server/util/Bag.js: -------------------------------------------------------------------------------- 1 | 2 | class Bag { 3 | constructor() { 4 | this.bag = {} 5 | } 6 | 7 | keyExtractor( item ) { 8 | return item 9 | } 10 | 11 | add( item ) { 12 | const key = this.keyExtractor( item ) 13 | if ( this.contains( item ) ) { 14 | this.bag[key] += 1 15 | } else { 16 | this.bag[key] = 1 17 | } 18 | } 19 | 20 | remove( item ) { 21 | const key = this.keyExtractor( item ) 22 | if ( this.contains( item ) ) { 23 | if ( this.count( item ) > 1 ) { 24 | this.bag[key] -= 1 25 | } else { 26 | delete this.bag[key] 27 | } 28 | } 29 | } 30 | 31 | contains( item ) { 32 | const key = this.keyExtractor( item ) 33 | return !!this.bag[key] 34 | } 35 | 36 | count( item ) { 37 | const key = this.keyExtractor( item ) 38 | return this.bag[key] 39 | } 40 | 41 | toArray() { 42 | return Object.keys( this.bag ) 43 | } 44 | 45 | toObject() { 46 | return { ...this.bag } 47 | } 48 | } 49 | 50 | module.exports = Bag 51 | -------------------------------------------------------------------------------- /esp32-camera-firmware/.gitignore: -------------------------------------------------------------------------------- 1 | .pio 2 | .pioenvs 3 | .piolibdeps 4 | .vscode/.browse.c_cpp.db* 5 | .vscode/c_cpp_properties.json 6 | .vscode/launch.json 7 | -------------------------------------------------------------------------------- /esp32-camera-firmware/.travis.yml: -------------------------------------------------------------------------------- 1 | # Continuous Integration (CI) is the practice, in software 2 | # engineering, of merging all developer working copies with a shared mainline 3 | # several times a day < https://docs.platformio.org/page/ci/index.html > 4 | # 5 | # Documentation: 6 | # 7 | # * Travis CI Embedded Builds with PlatformIO 8 | # < https://docs.travis-ci.com/user/integration/platformio/ > 9 | # 10 | # * PlatformIO integration with Travis CI 11 | # < https://docs.platformio.org/page/ci/travis.html > 12 | # 13 | # * User Guide for `platformio ci` command 14 | # < https://docs.platformio.org/page/userguide/cmd_ci.html > 15 | # 16 | # 17 | # Please choose one of the following templates (proposed below) and uncomment 18 | # it (remove "# " before each line) or use own configuration according to the 19 | # Travis CI documentation (see above). 20 | # 21 | 22 | 23 | # 24 | # Template #1: General project. Test it using existing `platformio.ini`. 25 | # 26 | 27 | # language: python 28 | # python: 29 | # - "2.7" 30 | # 31 | # sudo: false 32 | # cache: 33 | # directories: 34 | # - "~/.platformio" 35 | # 36 | # install: 37 | # - pip install -U platformio 38 | # - platformio update 39 | # 40 | # script: 41 | # - platformio run 42 | 43 | 44 | # 45 | # Template #2: The project is intended to be used as a library with examples. 46 | # 47 | 48 | # language: python 49 | # python: 50 | # - "2.7" 51 | # 52 | # sudo: false 53 | # cache: 54 | # directories: 55 | # - "~/.platformio" 56 | # 57 | # env: 58 | # - PLATFORMIO_CI_SRC=path/to/test/file.c 59 | # - PLATFORMIO_CI_SRC=examples/file.ino 60 | # - PLATFORMIO_CI_SRC=path/to/test/directory 61 | # 62 | # install: 63 | # - pip install -U platformio 64 | # - platformio update 65 | # 66 | # script: 67 | # - platformio ci --lib="." --board=ID_1 --board=ID_2 --board=ID_N 68 | -------------------------------------------------------------------------------- /esp32-camera-firmware/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | // See http://go.microsoft.com/fwlink/?LinkId=827846 3 | // for the documentation about the extensions.json format 4 | "recommendations": [ 5 | "platformio.platformio-ide" 6 | ] 7 | } -------------------------------------------------------------------------------- /esp32-camera-firmware/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "terminal.integrated.env.osx": { 3 | "PATH": "/Users/alvaroviebrantz/.platformio/penv/bin:/Users/alvaroviebrantz/.platformio/penv:/Users/alvaroviebrantz/bin:/Users/alvaroviebrantz/Library/Android/sdk/tools/bin:/Users/alvaroviebrantz/Library/Android/sdk/platform-tools:/Users/alvaroviebrantz/Documents/Desenvolvimento/google-cloud-sdk/bin:/Users/alvaroviebrantz/.mos/bin:/Users/alvaroviebrantz/Documents/Desenvolvimento/Flutter/sdk/bin:/Users/alvaroviebrantz/.nvm/versions/node/v10.13.0/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/go/bin:/Users/alvaroviebrantz/Documents/Desenvolvimento/go/gocode/bin:/Users/alvaroviebrantz/.fastlane/bin:/Users/alvaroviebrantz/Documents/Desenvolvimento/leverege/platform/build-tools/src:/Users/alvaroviebrantz/Documents/Desenvolvimento/leverege/platform/terraform-k8s", 4 | "PLATFORMIO_CALLER": "vscode" 5 | }, 6 | "files.associations": { 7 | "sdkconfig.h": "c", 8 | "qr_recoginize.h": "c", 9 | "ov2640.h": "c", 10 | "sensor.h": "c", 11 | "esp_wifi.h": "c" 12 | } 13 | } -------------------------------------------------------------------------------- /esp32-camera-firmware/include/README: -------------------------------------------------------------------------------- 1 | 2 | This directory is intended for project header files. 3 | 4 | A header file is a file containing C declarations and macro definitions 5 | to be shared between several project source files. You request the use of a 6 | header file in your project source file (C, C++, etc) located in `src` folder 7 | by including it, with the C preprocessing directive `#include'. 8 | 9 | ```src/main.c 10 | 11 | #include "header.h" 12 | 13 | int main (void) 14 | { 15 | ... 16 | } 17 | ``` 18 | 19 | Including a header file produces the same results as copying the header file 20 | into each source file that needs it. Such copying would be time-consuming 21 | and error-prone. With a header file, the related declarations appear 22 | in only one place. If they need to be changed, they can be changed in one 23 | place, and programs that include the header file will automatically use the 24 | new version when next recompiled. The header file eliminates the labor of 25 | finding and changing all the copies as well as the risk that a failure to 26 | find one copy will result in inconsistencies within a program. 27 | 28 | In C, the usual convention is to give header files names that end with `.h'. 29 | It is most portable to use only letters, digits, dashes, and underscores in 30 | header file names, and at most one dot. 31 | 32 | Read more about using header files in official GCC documentation: 33 | 34 | * Include Syntax 35 | * Include Operation 36 | * Once-Only Headers 37 | * Computed Includes 38 | 39 | https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html 40 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/README: -------------------------------------------------------------------------------- 1 | 2 | This directory is intended for project specific (private) libraries. 3 | PlatformIO will compile them to static libraries and link into executable file. 4 | 5 | The source code of each library should be placed in a an own separate directory 6 | ("lib/your_library_name/[here are source files]"). 7 | 8 | For example, see a structure of the following two libraries `Foo` and `Bar`: 9 | 10 | |--lib 11 | | | 12 | | |--Bar 13 | | | |--docs 14 | | | |--examples 15 | | | |--src 16 | | | |- Bar.c 17 | | | |- Bar.h 18 | | | |- library.json (optional, custom build options, etc) https://docs.platformio.org/page/librarymanager/config.html 19 | | | 20 | | |--Foo 21 | | | |- Foo.c 22 | | | |- Foo.h 23 | | | 24 | | |- README --> THIS FILE 25 | | 26 | |- platformio.ini 27 | |--src 28 | |- main.c 29 | 30 | and a contents of `src/main.c`: 31 | ``` 32 | #include 33 | #include 34 | 35 | int main (void) 36 | { 37 | ... 38 | } 39 | 40 | ``` 41 | 42 | PlatformIO Library Dependency Finder will find automatically dependent 43 | libraries scanning project source files. 44 | 45 | More information about PlatformIO Library Dependency Finder 46 | - https://docs.platformio.org/page/librarymanager/ldf.html 47 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/camera_common.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include "rom/lldesc.h" 7 | #include "esp_err.h" 8 | #include "esp_intr_alloc.h" 9 | #include "freertos/FreeRTOS.h" 10 | #include "freertos/semphr.h" 11 | #include "freertos/task.h" 12 | #include "esp_camera.h" 13 | #include "sensor.h" 14 | 15 | typedef union { 16 | struct { 17 | uint8_t sample2; 18 | uint8_t unused2; 19 | uint8_t sample1; 20 | uint8_t unused1; 21 | }; 22 | uint32_t val; 23 | } dma_elem_t; 24 | 25 | typedef enum { 26 | /* camera sends byte sequence: s1, s2, s3, s4, ... 27 | * fifo receives: 00 s1 00 s2, 00 s2 00 s3, 00 s3 00 s4, ... 28 | */ 29 | SM_0A0B_0B0C = 0, 30 | /* camera sends byte sequence: s1, s2, s3, s4, ... 31 | * fifo receives: 00 s1 00 s2, 00 s3 00 s4, ... 32 | */ 33 | SM_0A0B_0C0D = 1, 34 | /* camera sends byte sequence: s1, s2, s3, s4, ... 35 | * fifo receives: 00 s1 00 00, 00 s2 00 00, 00 s3 00 00, ... 36 | */ 37 | SM_0A00_0B00 = 3, 38 | } i2s_sampling_mode_t; 39 | 40 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/esp_camera.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | /* 15 | * Example Use 16 | * 17 | static camera_config_t camera_example_config = { 18 | .pin_pwdn = PIN_PWDN, 19 | .pin_reset = PIN_RESET, 20 | .pin_xclk = PIN_XCLK, 21 | .pin_sscb_sda = PIN_SIOD, 22 | .pin_sscb_scl = PIN_SIOC, 23 | .pin_d7 = PIN_D7, 24 | .pin_d6 = PIN_D6, 25 | .pin_d5 = PIN_D5, 26 | .pin_d4 = PIN_D4, 27 | .pin_d3 = PIN_D3, 28 | .pin_d2 = PIN_D2, 29 | .pin_d1 = PIN_D1, 30 | .pin_d0 = PIN_D0, 31 | .pin_vsync = PIN_VSYNC, 32 | .pin_href = PIN_HREF, 33 | .pin_pclk = PIN_PCLK, 34 | 35 | .xclk_freq_hz = 20000000, 36 | .ledc_timer = LEDC_TIMER_0, 37 | .ledc_channel = LEDC_CHANNEL_0, 38 | .pixel_format = PIXFORMAT_JPEG, 39 | .frame_size = FRAMESIZE_SVGA, 40 | .jpeg_quality = 10, 41 | .fb_count = 2 42 | }; 43 | 44 | esp_err_t camera_example_init(){ 45 | return esp_camera_init(&camera_example_config); 46 | } 47 | 48 | esp_err_t camera_example_capture(){ 49 | //capture a frame 50 | camera_fb_t * fb = esp_camera_fb_get(); 51 | if (!fb) { 52 | ESP_LOGE(TAG, "Frame buffer could not be acquired"); 53 | return ESP_FAIL; 54 | } 55 | 56 | //replace this with your own function 57 | display_image(fb->width, fb->height, fb->pixformat, fb->buf, fb->len); 58 | 59 | //return the frame buffer back to be reused 60 | esp_camera_fb_return(fb); 61 | 62 | return ESP_OK; 63 | } 64 | */ 65 | 66 | #pragma once 67 | 68 | #include "esp_err.h" 69 | #include "driver/ledc.h" 70 | #include "sensor.h" 71 | 72 | #ifdef __cplusplus 73 | extern "C" { 74 | #endif 75 | 76 | /** 77 | * @brief Configuration structure for camera initialization 78 | */ 79 | typedef struct { 80 | int pin_pwdn; /*!< GPIO pin for camera power down line */ 81 | int pin_reset; /*!< GPIO pin for camera reset line */ 82 | int pin_xclk; /*!< GPIO pin for camera XCLK line */ 83 | int pin_sscb_sda; /*!< GPIO pin for camera SDA line */ 84 | int pin_sscb_scl; /*!< GPIO pin for camera SCL line */ 85 | int pin_d7; /*!< GPIO pin for camera D7 line */ 86 | int pin_d6; /*!< GPIO pin for camera D6 line */ 87 | int pin_d5; /*!< GPIO pin for camera D5 line */ 88 | int pin_d4; /*!< GPIO pin for camera D4 line */ 89 | int pin_d3; /*!< GPIO pin for camera D3 line */ 90 | int pin_d2; /*!< GPIO pin for camera D2 line */ 91 | int pin_d1; /*!< GPIO pin for camera D1 line */ 92 | int pin_d0; /*!< GPIO pin for camera D0 line */ 93 | int pin_vsync; /*!< GPIO pin for camera VSYNC line */ 94 | int pin_href; /*!< GPIO pin for camera HREF line */ 95 | int pin_pclk; /*!< GPIO pin for camera PCLK line */ 96 | 97 | int xclk_freq_hz; /*!< Frequency of XCLK signal, in Hz. Either 10KHz or 20KHz */ 98 | 99 | ledc_timer_t ledc_timer; /*!< LEDC timer to be used for generating XCLK */ 100 | ledc_channel_t ledc_channel; /*!< LEDC channel to be used for generating XCLK */ 101 | 102 | pixformat_t pixel_format; /*!< Format of the pixel data: PIXFORMAT_ + YUV422|GRAYSCALE|RGB565|JPEG */ 103 | framesize_t frame_size; /*!< Size of the output image: FRAMESIZE_ + QVGA|CIF|VGA|SVGA|XGA|SXGA|UXGA */ 104 | 105 | int jpeg_quality; /*!< Quality of JPEG output. 0-63 lower means higher quality */ 106 | size_t fb_count; /*!< Number of frame buffers to be allocated. If more than one, then each frame will be acquired (double speed) */ 107 | } camera_config_t; 108 | 109 | /** 110 | * @brief Data structure of camera frame buffer 111 | */ 112 | typedef struct { 113 | uint8_t * buf; /*!< Pointer to the pixel data */ 114 | size_t len; /*!< Length of the buffer in bytes */ 115 | size_t width; /*!< Width of the buffer in pixels */ 116 | size_t height; /*!< Height of the buffer in pixels */ 117 | pixformat_t format; /*!< Format of the pixel data */ 118 | } camera_fb_t; 119 | 120 | #define ESP_ERR_CAMERA_BASE 0x20000 121 | #define ESP_ERR_CAMERA_NOT_DETECTED (ESP_ERR_CAMERA_BASE + 1) 122 | #define ESP_ERR_CAMERA_FAILED_TO_SET_FRAME_SIZE (ESP_ERR_CAMERA_BASE + 2) 123 | #define ESP_ERR_CAMERA_NOT_SUPPORTED (ESP_ERR_CAMERA_BASE + 3) 124 | 125 | /** 126 | * @brief Initialize the camera driver 127 | * 128 | * @note call camera_probe before calling this function 129 | * 130 | * This function detects and configures camera over I2C interface, 131 | * allocates framebuffer and DMA buffers, 132 | * initializes parallel I2S input, and sets up DMA descriptors. 133 | * 134 | * Currently this function can only be called once and there is 135 | * no way to de-initialize this module. 136 | * 137 | * @param config Camera configuration parameters 138 | * 139 | * @return ESP_OK on success 140 | */ 141 | esp_err_t esp_camera_init(const camera_config_t* config); 142 | 143 | /** 144 | * @brief Deinitialize the camera driver 145 | * 146 | * @return 147 | * - ESP_OK on success 148 | * - ESP_ERR_INVALID_STATE if the driver hasn't been initialized yet 149 | */ 150 | esp_err_t esp_camera_deinit(); 151 | 152 | /** 153 | * @brief Obtain pointer to a frame buffer. 154 | * 155 | * @return pointer to the frame buffer 156 | */ 157 | camera_fb_t* esp_camera_fb_get(); 158 | 159 | /** 160 | * @brief Return the frame buffer to be reused again. 161 | * 162 | * @param fb Pointer to the frame buffer 163 | */ 164 | void esp_camera_fb_return(camera_fb_t * fb); 165 | 166 | /** 167 | * @brief Get a pointer to the image sensor control structure 168 | * 169 | * @return pointer to the sensor 170 | */ 171 | sensor_t * esp_camera_sensor_get(); 172 | 173 | 174 | #ifdef __cplusplus 175 | } 176 | #endif 177 | 178 | #include "img_converters.h" 179 | 180 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/ov2640.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the OpenMV project. 3 | * Copyright (c) 2013/2014 Ibrahim Abdelkader 4 | * This work is licensed under the MIT license, see the file LICENSE for details. 5 | * 6 | * OV2640 driver. 7 | * 8 | */ 9 | #ifndef __OV2640_H__ 10 | #define __OV2640_H__ 11 | #include "sensor.h" 12 | int ov2640_init(sensor_t *sensor); 13 | #endif // __OV2640_H__ 14 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/ov2640_regs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the OpenMV project. 3 | * Copyright (c) 2013/2014 Ibrahim Abdelkader 4 | * This work is licensed under the MIT license, see the file LICENSE for details. 5 | * 6 | * OV2640 register definitions. 7 | */ 8 | #ifndef __REG_REGS_H__ 9 | #define __REG_REGS_H__ 10 | /* DSP register bank FF=0x00*/ 11 | #define R_BYPASS 0x05 12 | #define QS 0x44 13 | #define CTRLI 0x50 14 | #define HSIZE 0x51 15 | #define VSIZE 0x52 16 | #define XOFFL 0x53 17 | #define YOFFL 0x54 18 | #define VHYX 0x55 19 | #define DPRP 0x56 20 | #define TEST 0x57 21 | #define ZMOW 0x5A 22 | #define ZMOH 0x5B 23 | #define ZMHH 0x5C 24 | #define BPADDR 0x7C 25 | #define BPDATA 0x7D 26 | #define CTRL2 0x86 27 | #define CTRL3 0x87 28 | #define SIZEL 0x8C 29 | #define HSIZE8 0xC0 30 | #define VSIZE8 0xC1 31 | #define CTRL0 0xC2 32 | #define CTRL1 0xC3 33 | #define R_DVP_SP 0xD3 34 | #define IMAGE_MODE 0xDA 35 | #define RESET 0xE0 36 | #define MS_SP 0xF0 37 | #define SS_ID 0xF7 38 | #define SS_CTRL 0xF7 39 | #define MC_BIST 0xF9 40 | #define MC_AL 0xFA 41 | #define MC_AH 0xFB 42 | #define MC_D 0xFC 43 | #define P_CMD 0xFD 44 | #define P_STATUS 0xFE 45 | #define BANK_SEL 0xFF 46 | 47 | #define CTRLI_LP_DP 0x80 48 | #define CTRLI_ROUND 0x40 49 | 50 | #define CTRL0_AEC_EN 0x80 51 | #define CTRL0_AEC_SEL 0x40 52 | #define CTRL0_STAT_SEL 0x20 53 | #define CTRL0_VFIRST 0x10 54 | #define CTRL0_YUV422 0x08 55 | #define CTRL0_YUV_EN 0x04 56 | #define CTRL0_RGB_EN 0x02 57 | #define CTRL0_RAW_EN 0x01 58 | 59 | #define CTRL2_DCW_EN 0x20 60 | #define CTRL2_SDE_EN 0x10 61 | #define CTRL2_UV_ADJ_EN 0x08 62 | #define CTRL2_UV_AVG_EN 0x04 63 | #define CTRL2_CMX_EN 0x01 64 | 65 | #define CTRL3_BPC_EN 0x80 66 | #define CTRL3_WPC_EN 0x40 67 | 68 | #define R_DVP_SP_AUTO_MODE 0x80 69 | 70 | #define R_BYPASS_DSP_EN 0x00 71 | #define R_BYPASS_DSP_BYPAS 0x01 72 | 73 | #define IMAGE_MODE_Y8_DVP_EN 0x40 74 | #define IMAGE_MODE_JPEG_EN 0x10 75 | #define IMAGE_MODE_YUV422 0x00 76 | #define IMAGE_MODE_RAW10 0x04 77 | #define IMAGE_MODE_RGB565 0x08 78 | #define IMAGE_MODE_HREF_VSYNC 0x02 79 | #define IMAGE_MODE_LBYTE_FIRST 0x01 80 | 81 | #define RESET_MICROC 0x40 82 | #define RESET_SCCB 0x20 83 | #define RESET_JPEG 0x10 84 | #define RESET_DVP 0x04 85 | #define RESET_IPU 0x02 86 | #define RESET_CIF 0x01 87 | 88 | #define MC_BIST_RESET 0x80 89 | #define MC_BIST_BOOT_ROM_SEL 0x40 90 | #define MC_BIST_12KB_SEL 0x20 91 | #define MC_BIST_12KB_MASK 0x30 92 | #define MC_BIST_512KB_SEL 0x08 93 | #define MC_BIST_512KB_MASK 0x0C 94 | #define MC_BIST_BUSY_BIT_R 0x02 95 | #define MC_BIST_MC_RES_ONE_SH_W 0x02 96 | #define MC_BIST_LAUNCH 0x01 97 | 98 | 99 | typedef enum { 100 | BANK_DSP, BANK_SENSOR, BANK_MAX 101 | } ov2640_bank_t; 102 | 103 | /* Sensor register bank FF=0x01*/ 104 | #define GAIN 0x00 105 | #define COM1 0x03 106 | #define REG04 0x04 107 | #define REG08 0x08 108 | #define COM2 0x09 109 | #define REG_PID 0x0A 110 | #define REG_VER 0x0B 111 | #define COM3 0x0C 112 | #define COM4 0x0D 113 | #define AEC 0x10 114 | #define CLKRC 0x11 115 | #define COM7 0x12 116 | #define COM8 0x13 117 | #define COM9 0x14 /* AGC gain ceiling */ 118 | #define COM10 0x15 119 | #define HSTART 0x17 120 | #define HSTOP 0x18 121 | #define VSTART 0x19 122 | #define VSTOP 0x1A 123 | #define MIDH 0x1C 124 | #define MIDL 0x1D 125 | #define AEW 0x24 126 | #define AEB 0x25 127 | #define VV 0x26 128 | #define REG2A 0x2A 129 | #define FRARL 0x2B 130 | #define ADDVSL 0x2D 131 | #define ADDVSH 0x2E 132 | #define YAVG 0x2F 133 | #define HSDY 0x30 134 | #define HEDY 0x31 135 | #define REG32 0x32 136 | #define ARCOM2 0x34 137 | #define REG45 0x45 138 | #define FLL 0x46 139 | #define FLH 0x47 140 | #define COM19 0x48 141 | #define ZOOMS 0x49 142 | #define COM22 0x4B 143 | #define COM25 0x4E 144 | #define BD50 0x4F 145 | #define BD60 0x50 146 | #define REG5D 0x5D 147 | #define REG5E 0x5E 148 | #define REG5F 0x5F 149 | #define REG60 0x60 150 | #define HISTO_LOW 0x61 151 | #define HISTO_HIGH 0x62 152 | 153 | #define REG04_DEFAULT 0x28 154 | #define REG04_HFLIP_IMG 0x80 155 | #define REG04_VFLIP_IMG 0x40 156 | #define REG04_VREF_EN 0x10 157 | #define REG04_HREF_EN 0x08 158 | #define REG04_SET(x) (REG04_DEFAULT|x) 159 | 160 | #define COM2_STDBY 0x10 161 | #define COM2_OUT_DRIVE_1x 0x00 162 | #define COM2_OUT_DRIVE_2x 0x01 163 | #define COM2_OUT_DRIVE_3x 0x02 164 | #define COM2_OUT_DRIVE_4x 0x03 165 | 166 | #define COM3_DEFAULT 0x38 167 | #define COM3_BAND_50Hz 0x04 168 | #define COM3_BAND_60Hz 0x00 169 | #define COM3_BAND_AUTO 0x02 170 | #define COM3_BAND_SET(x) (COM3_DEFAULT|x) 171 | 172 | #define COM7_SRST 0x80 173 | #define COM7_RES_UXGA 0x00 /* UXGA */ 174 | #define COM7_RES_SVGA 0x40 /* SVGA */ 175 | #define COM7_RES_CIF 0x20 /* CIF */ 176 | #define COM7_ZOOM_EN 0x04 /* Enable Zoom */ 177 | #define COM7_COLOR_BAR 0x02 /* Enable Color Bar Test */ 178 | 179 | #define COM8_DEFAULT 0xC0 180 | #define COM8_BNDF_EN 0x20 /* Enable Banding filter */ 181 | #define COM8_AGC_EN 0x04 /* AGC Auto/Manual control selection */ 182 | #define COM8_AEC_EN 0x01 /* Auto/Manual Exposure control */ 183 | #define COM8_SET(x) (COM8_DEFAULT|x) 184 | 185 | #define COM9_DEFAULT 0x08 186 | #define COM9_AGC_GAIN_2x 0x00 /* AGC: 2x */ 187 | #define COM9_AGC_GAIN_4x 0x01 /* AGC: 4x */ 188 | #define COM9_AGC_GAIN_8x 0x02 /* AGC: 8x */ 189 | #define COM9_AGC_GAIN_16x 0x03 /* AGC: 16x */ 190 | #define COM9_AGC_GAIN_32x 0x04 /* AGC: 32x */ 191 | #define COM9_AGC_GAIN_64x 0x05 /* AGC: 64x */ 192 | #define COM9_AGC_GAIN_128x 0x06 /* AGC: 128x */ 193 | #define COM9_AGC_SET(x) (COM9_DEFAULT|(x<<5)) 194 | 195 | #define COM10_HREF_EN 0x80 /* HSYNC changes to HREF */ 196 | #define COM10_HSYNC_EN 0x40 /* HREF changes to HSYNC */ 197 | #define COM10_PCLK_FREE 0x20 /* PCLK output option: free running PCLK */ 198 | #define COM10_PCLK_EDGE 0x10 /* Data is updated at the rising edge of PCLK */ 199 | #define COM10_HREF_NEG 0x08 /* HREF negative */ 200 | #define COM10_VSYNC_NEG 0x02 /* VSYNC negative */ 201 | #define COM10_HSYNC_NEG 0x01 /* HSYNC negative */ 202 | 203 | #define CTRL1_AWB 0x08 /* Enable AWB */ 204 | 205 | #define VV_AGC_TH_SET(h,l) ((h<<4)|(l&0x0F)) 206 | 207 | #define REG32_UXGA 0x36 208 | #define REG32_SVGA 0x09 209 | #define REG32_CIF 0x89 210 | 211 | #define CLKRC_2X 0x80 212 | 213 | #endif //__REG_REGS_H__ 214 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/ov7725.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the OpenMV project. 3 | * Copyright (c) 2013/2014 Ibrahim Abdelkader 4 | * This work is licensed under the MIT license, see the file LICENSE for details. 5 | * 6 | * OV7725 driver. 7 | * 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "sccb.h" 14 | #include "ov7725.h" 15 | #include "ov7725_regs.h" 16 | #include "freertos/FreeRTOS.h" 17 | #include "freertos/task.h" 18 | 19 | #if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG) 20 | #include "esp32-hal-log.h" 21 | #else 22 | #include "esp_log.h" 23 | static const char* TAG = "ov7725"; 24 | #endif 25 | 26 | 27 | static const uint8_t default_regs[][2] = { 28 | {COM3, COM3_SWAP_YUV}, 29 | {COM7, COM7_RES_QVGA | COM7_FMT_YUV}, 30 | 31 | {COM4, 0x01}, /* bypass PLL */ 32 | {CLKRC, 0xC0}, /* Res/Bypass pre-scalar */ 33 | 34 | // QVGA Window Size 35 | {HSTART, 0x3F}, 36 | {HSIZE, 0x50}, 37 | {VSTART, 0x03}, 38 | {VSIZE, 0x78}, 39 | {HREF, 0x00}, 40 | 41 | // Scale down to QVGA Resolution 42 | {HOUTSIZE, 0x50}, 43 | {VOUTSIZE, 0x78}, 44 | 45 | {COM12, 0x03}, 46 | {EXHCH, 0x00}, 47 | {TGT_B, 0x7F}, 48 | {FIXGAIN, 0x09}, 49 | {AWB_CTRL0, 0xE0}, 50 | {DSP_CTRL1, 0xFF}, 51 | 52 | {DSP_CTRL2, DSP_CTRL2_VDCW_EN | DSP_CTRL2_HDCW_EN | DSP_CTRL2_HZOOM_EN | DSP_CTRL2_VZOOM_EN}, 53 | 54 | {DSP_CTRL3, 0x00}, 55 | {DSP_CTRL4, 0x00}, 56 | {DSPAUTO, 0xFF}, 57 | 58 | {COM8, 0xF0}, 59 | {COM6, 0xC5}, 60 | {COM9, 0x11}, 61 | {COM10, COM10_VSYNC_NEG | COM10_PCLK_MASK}, //Invert VSYNC and MASK PCLK 62 | {BDBASE, 0x7F}, 63 | {DBSTEP, 0x03}, 64 | {AEW, 0x96}, 65 | {AEB, 0x64}, 66 | {VPT, 0xA1}, 67 | {EXHCL, 0x00}, 68 | {AWB_CTRL3, 0xAA}, 69 | {COM8, 0xFF}, 70 | 71 | //Gamma 72 | {GAM1, 0x0C}, 73 | {GAM2, 0x16}, 74 | {GAM3, 0x2A}, 75 | {GAM4, 0x4E}, 76 | {GAM5, 0x61}, 77 | {GAM6, 0x6F}, 78 | {GAM7, 0x7B}, 79 | {GAM8, 0x86}, 80 | {GAM9, 0x8E}, 81 | {GAM10, 0x97}, 82 | {GAM11, 0xA4}, 83 | {GAM12, 0xAF}, 84 | {GAM13, 0xC5}, 85 | {GAM14, 0xD7}, 86 | {GAM15, 0xE8}, 87 | 88 | {SLOP, 0x20}, 89 | {EDGE1, 0x05}, 90 | {EDGE2, 0x03}, 91 | {EDGE3, 0x00}, 92 | {DNSOFF, 0x01}, 93 | 94 | {MTX1, 0xB0}, 95 | {MTX2, 0x9D}, 96 | {MTX3, 0x13}, 97 | {MTX4, 0x16}, 98 | {MTX5, 0x7B}, 99 | {MTX6, 0x91}, 100 | {MTX_CTRL, 0x1E}, 101 | 102 | {BRIGHTNESS, 0x08}, 103 | {CONTRAST, 0x30}, 104 | {UVADJ0, 0x81}, 105 | {SDE, (SDE_CONT_BRIGHT_EN | SDE_SATURATION_EN)}, 106 | 107 | // For 30 fps/60Hz 108 | {DM_LNL, 0x00}, 109 | {DM_LNH, 0x00}, 110 | {BDBASE, 0x7F}, 111 | {DBSTEP, 0x03}, 112 | 113 | // Lens Correction, should be tuned with real camera module 114 | {LC_RADI, 0x10}, 115 | {LC_COEF, 0x10}, 116 | {LC_COEFB, 0x14}, 117 | {LC_COEFR, 0x17}, 118 | {LC_CTR, 0x05}, 119 | {COM5, 0xF5}, //0x65 120 | 121 | {0x00, 0x00}, 122 | }; 123 | 124 | 125 | static int reset(sensor_t *sensor) 126 | { 127 | int i=0; 128 | const uint8_t (*regs)[2]; 129 | 130 | // Reset all registers 131 | SCCB_Write(sensor->slv_addr, COM7, COM7_RESET); 132 | 133 | // Delay 10 ms 134 | vTaskDelay(10 / portTICK_PERIOD_MS); 135 | 136 | // Write default regsiters 137 | for (i=0, regs = default_regs; regs[i][0]; i++) { 138 | SCCB_Write(sensor->slv_addr, regs[i][0], regs[i][1]); 139 | } 140 | 141 | // Delay 142 | vTaskDelay(30 / portTICK_PERIOD_MS); 143 | 144 | return 0; 145 | } 146 | 147 | 148 | static int set_pixformat(sensor_t *sensor, pixformat_t pixformat) 149 | { 150 | int ret=0; 151 | // Read register COM7 152 | uint8_t reg = SCCB_Read(sensor->slv_addr, COM7); 153 | 154 | switch (pixformat) { 155 | case PIXFORMAT_RGB565: 156 | reg = COM7_SET_RGB(reg, COM7_FMT_RGB565); 157 | break; 158 | case PIXFORMAT_YUV422: 159 | case PIXFORMAT_GRAYSCALE: 160 | reg = COM7_SET_FMT(reg, COM7_FMT_YUV); 161 | break; 162 | default: 163 | return -1; 164 | } 165 | 166 | // Write back register COM7 167 | ret = SCCB_Write(sensor->slv_addr, COM7, reg); 168 | 169 | // Delay 170 | vTaskDelay(30 / portTICK_PERIOD_MS); 171 | 172 | return ret; 173 | } 174 | 175 | static int set_framesize(sensor_t *sensor, framesize_t framesize) 176 | { 177 | int ret=0; 178 | uint16_t w = resolution[framesize][0]; 179 | uint16_t h = resolution[framesize][1]; 180 | 181 | // Write MSBs 182 | ret |= SCCB_Write(sensor->slv_addr, HOUTSIZE, w>>2); 183 | ret |= SCCB_Write(sensor->slv_addr, VOUTSIZE, h>>1); 184 | 185 | // Write LSBs 186 | ret |= SCCB_Write(sensor->slv_addr, EXHCH, ((w&0x3) | ((h&0x1) << 2))); 187 | 188 | if (framesize < FRAMESIZE_VGA) { 189 | // Enable auto-scaling/zooming factors 190 | ret |= SCCB_Write(sensor->slv_addr, DSPAUTO, 0xFF); 191 | } else { 192 | // Disable auto-scaling/zooming factors 193 | ret |= SCCB_Write(sensor->slv_addr, DSPAUTO, 0xF3); 194 | 195 | // Clear auto-scaling/zooming factors 196 | ret |= SCCB_Write(sensor->slv_addr, SCAL0, 0x00); 197 | ret |= SCCB_Write(sensor->slv_addr, SCAL1, 0x00); 198 | ret |= SCCB_Write(sensor->slv_addr, SCAL2, 0x00); 199 | } 200 | 201 | // Delay 202 | vTaskDelay(30 / portTICK_PERIOD_MS); 203 | 204 | return ret; 205 | } 206 | 207 | static int set_colorbar(sensor_t *sensor, int enable) 208 | { 209 | int ret=0; 210 | uint8_t reg; 211 | 212 | // Read reg COM3 213 | reg = SCCB_Read(sensor->slv_addr, COM3); 214 | // Enable colorbar test pattern output 215 | reg = COM3_SET_CBAR(reg, enable); 216 | // Write back COM3 217 | ret |= SCCB_Write(sensor->slv_addr, COM3, reg); 218 | 219 | // Read reg DSP_CTRL3 220 | reg = SCCB_Read(sensor->slv_addr, DSP_CTRL3); 221 | // Enable DSP colorbar output 222 | reg = DSP_CTRL3_SET_CBAR(reg, enable); 223 | // Write back DSP_CTRL3 224 | ret |= SCCB_Write(sensor->slv_addr, DSP_CTRL3, reg); 225 | 226 | return ret; 227 | } 228 | 229 | static int set_whitebal(sensor_t *sensor, int enable) 230 | { 231 | // Read register COM8 232 | uint8_t reg = SCCB_Read(sensor->slv_addr, COM8); 233 | 234 | // Set white bal on/off 235 | reg = COM8_SET_AWB(reg, enable); 236 | 237 | // Write back register COM8 238 | return SCCB_Write(sensor->slv_addr, COM8, reg); 239 | } 240 | 241 | static int set_gain_ctrl(sensor_t *sensor, int enable) 242 | { 243 | // Read register COM8 244 | uint8_t reg = SCCB_Read(sensor->slv_addr, COM8); 245 | 246 | // Set white bal on/off 247 | reg = COM8_SET_AGC(reg, enable); 248 | 249 | // Write back register COM8 250 | return SCCB_Write(sensor->slv_addr, COM8, reg); 251 | } 252 | 253 | static int set_exposure_ctrl(sensor_t *sensor, int enable) 254 | { 255 | // Read register COM8 256 | uint8_t reg = SCCB_Read(sensor->slv_addr, COM8); 257 | 258 | // Set white bal on/off 259 | reg = COM8_SET_AEC(reg, enable); 260 | 261 | // Write back register COM8 262 | return SCCB_Write(sensor->slv_addr, COM8, reg); 263 | } 264 | 265 | static int set_hmirror(sensor_t *sensor, int enable) 266 | { 267 | // Read register COM3 268 | uint8_t reg = SCCB_Read(sensor->slv_addr, COM3); 269 | 270 | // Set mirror on/off 271 | reg = COM3_SET_MIRROR(reg, enable); 272 | 273 | // Write back register COM3 274 | return SCCB_Write(sensor->slv_addr, COM3, reg); 275 | } 276 | 277 | static int set_vflip(sensor_t *sensor, int enable) 278 | { 279 | // Read register COM3 280 | uint8_t reg = SCCB_Read(sensor->slv_addr, COM3); 281 | 282 | // Set mirror on/off 283 | reg = COM3_SET_FLIP(reg, enable); 284 | 285 | // Write back register COM3 286 | return SCCB_Write(sensor->slv_addr, COM3, reg); 287 | } 288 | 289 | int ov7725_init(sensor_t *sensor) 290 | { 291 | // Set function pointers 292 | sensor->reset = reset; 293 | sensor->set_pixformat = set_pixformat; 294 | sensor->set_framesize = set_framesize; 295 | sensor->set_colorbar = set_colorbar; 296 | sensor->set_whitebal = set_whitebal; 297 | sensor->set_gain_ctrl = set_gain_ctrl; 298 | sensor->set_exposure_ctrl = set_exposure_ctrl; 299 | sensor->set_hmirror = set_hmirror; 300 | sensor->set_vflip = set_vflip; 301 | 302 | // Retrieve sensor's signature 303 | sensor->id.MIDH = SCCB_Read(sensor->slv_addr, REG_MIDH); 304 | sensor->id.MIDL = SCCB_Read(sensor->slv_addr, REG_MIDL); 305 | sensor->id.PID = SCCB_Read(sensor->slv_addr, REG_PID); 306 | sensor->id.VER = SCCB_Read(sensor->slv_addr, REG_VER); 307 | 308 | ESP_LOGD(TAG, "OV7725 Attached"); 309 | 310 | return 0; 311 | } 312 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/ov7725.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the OpenMV project. 3 | * Copyright (c) 2013/2014 Ibrahim Abdelkader 4 | * This work is licensed under the MIT license, see the file LICENSE for details. 5 | * 6 | * OV7725 driver. 7 | * 8 | */ 9 | #ifndef __OV7725_H__ 10 | #define __OV7725_H__ 11 | #include "sensor.h" 12 | 13 | int ov7725_init(sensor_t *sensor); 14 | #endif // __OV7725_H__ 15 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/sccb.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the OpenMV project. 3 | * Copyright (c) 2013/2014 Ibrahim Abdelkader 4 | * This work is licensed under the MIT license, see the file LICENSE for details. 5 | * 6 | * SCCB (I2C like) driver. 7 | * 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include "sccb.h" 13 | #include "twi.h" 14 | #include 15 | #include "sdkconfig.h" 16 | #if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG) 17 | #include "esp32-hal-log.h" 18 | #else 19 | #include "esp_log.h" 20 | static const char* TAG = "sccb"; 21 | #endif 22 | 23 | 24 | #define SCCB_FREQ (100000) // We don't need fast I2C. 100KHz is fine here. 25 | #define TIMEOUT (1000) /* Can't be sure when I2C routines return. Interrupts 26 | while polling hardware may result in unknown delays. */ 27 | 28 | 29 | int SCCB_Init(int pin_sda, int pin_scl) 30 | { 31 | twi_init(pin_sda, pin_scl); 32 | return 0; 33 | } 34 | 35 | uint8_t SCCB_Probe() 36 | { 37 | uint8_t reg = 0x00; 38 | uint8_t slv_addr = 0x00; 39 | 40 | for (uint8_t i=0; i<127; i++) { 41 | if (twi_writeTo(i, ®, 1, true) == 0) { 42 | slv_addr = i; 43 | break; 44 | } 45 | 46 | if (i!=126) { 47 | vTaskDelay(1 / portTICK_PERIOD_MS); // Necessary for OV7725 camera (not for OV2640). 48 | } 49 | } 50 | return slv_addr; 51 | } 52 | 53 | uint8_t SCCB_Read(uint8_t slv_addr, uint8_t reg) 54 | { 55 | uint8_t data=0; 56 | 57 | int rc = twi_writeTo(slv_addr, ®, 1, true); 58 | if (rc != 0) { 59 | data = 0xff; 60 | } else { 61 | rc = twi_readFrom(slv_addr, &data, 1, true); 62 | if (rc != 0) { 63 | data=0xFF; 64 | } 65 | } 66 | if (rc != 0) { 67 | ESP_LOGE(TAG, "SCCB_Read [%02x] failed rc=%d\n", reg, rc); 68 | } 69 | return data; 70 | } 71 | 72 | uint8_t SCCB_Write(uint8_t slv_addr, uint8_t reg, uint8_t data) 73 | { 74 | uint8_t ret=0; 75 | uint8_t buf[] = {reg, data}; 76 | 77 | if(twi_writeTo(slv_addr, buf, 2, true) != 0) { 78 | ret=0xFF; 79 | } 80 | if (ret != 0) { 81 | printf("SCCB_Write [%02x]=%02x failed\n", reg, data); 82 | } 83 | return ret; 84 | } 85 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/sccb.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the OpenMV project. 3 | * Copyright (c) 2013/2014 Ibrahim Abdelkader 4 | * This work is licensed under the MIT license, see the file LICENSE for details. 5 | * 6 | * SCCB (I2C like) driver. 7 | * 8 | */ 9 | #ifndef __SCCB_H__ 10 | #define __SCCB_H__ 11 | #include 12 | int SCCB_Init(int pin_sda, int pin_scl); 13 | uint8_t SCCB_Probe(); 14 | uint8_t SCCB_Read(uint8_t slv_addr, uint8_t reg); 15 | uint8_t SCCB_Write(uint8_t slv_addr, uint8_t reg, uint8_t data); 16 | #endif // __SCCB_H__ 17 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/sensor.c: -------------------------------------------------------------------------------- 1 | 2 | const int resolution[][2] = { 3 | { 160, 120 }, /* QQVGA */ 4 | { 128, 160 }, /* QQVGA2*/ 5 | { 176, 144 }, /* QCIF */ 6 | { 240, 176 }, /* HQVGA */ 7 | { 320, 240 }, /* QVGA */ 8 | { 400, 296 }, /* CIF */ 9 | { 640, 480 }, /* VGA */ 10 | { 800, 600 }, /* SVGA */ 11 | { 1024, 768 }, /* XGA */ 12 | { 1280, 1024 }, /* SXGA */ 13 | { 1600, 1200 }, /* UXGA */ 14 | }; 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/sensor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the OpenMV project. 3 | * Copyright (c) 2013/2014 Ibrahim Abdelkader 4 | * This work is licensed under the MIT license, see the file LICENSE for details. 5 | * 6 | * Sensor abstraction layer. 7 | * 8 | */ 9 | #ifndef __SENSOR_H__ 10 | #define __SENSOR_H__ 11 | #include 12 | 13 | #define OV9650_PID (0x96) 14 | #define OV2640_PID (0x26) 15 | #define OV7725_PID (0x77) 16 | 17 | typedef enum { 18 | PIXFORMAT_RGB565, // 2BPP/RGB565 19 | PIXFORMAT_YUV422, // 2BPP/YUV422 20 | PIXFORMAT_GRAYSCALE, // 1BPP/GRAYSCALE 21 | PIXFORMAT_JPEG, // JPEG/COMPRESSED 22 | PIXFORMAT_RGB888, // 3BPP/RGB888 23 | } pixformat_t; 24 | 25 | typedef enum { 26 | FRAMESIZE_QQVGA, // 160x120 27 | FRAMESIZE_QQVGA2, // 128x160 28 | FRAMESIZE_QCIF, // 176x144 29 | FRAMESIZE_HQVGA, // 240x176 30 | FRAMESIZE_QVGA, // 320x240 31 | FRAMESIZE_CIF, // 400x296 32 | FRAMESIZE_VGA, // 640x480 33 | FRAMESIZE_SVGA, // 800x600 34 | FRAMESIZE_XGA, // 1024x768 35 | FRAMESIZE_SXGA, // 1280x1024 36 | FRAMESIZE_UXGA, // 1600x1200 37 | } framesize_t; 38 | 39 | typedef enum { 40 | GAINCEILING_2X, 41 | GAINCEILING_4X, 42 | GAINCEILING_8X, 43 | GAINCEILING_16X, 44 | GAINCEILING_32X, 45 | GAINCEILING_64X, 46 | GAINCEILING_128X, 47 | } gainceiling_t; 48 | 49 | typedef struct { 50 | uint8_t MIDH; 51 | uint8_t MIDL; 52 | uint8_t PID; 53 | uint8_t VER; 54 | } sensor_id_t; 55 | 56 | typedef struct { 57 | framesize_t framesize;//0 - 10 58 | uint8_t quality;//0 - 63 59 | int8_t brightness;//-2 - 2 60 | int8_t contrast;//-2 - 2 61 | int8_t saturation;//-2 - 2 62 | uint8_t special_effect;//0 - 6 63 | uint8_t wb_mode;//0 - 4 64 | uint8_t awb; 65 | uint8_t awb_gain; 66 | uint8_t aec; 67 | uint8_t aec2; 68 | int8_t ae_level;//-2 - 2 69 | uint16_t aec_value;//0 - 1200 70 | uint8_t agc; 71 | uint8_t agc_gain;//0 - 30 72 | uint8_t gainceiling;//0 - 6 73 | uint8_t bpc; 74 | uint8_t wpc; 75 | uint8_t raw_gma; 76 | uint8_t lenc; 77 | uint8_t hmirror; 78 | uint8_t vflip; 79 | uint8_t dcw; 80 | uint8_t colorbar; 81 | } camera_status_t; 82 | 83 | typedef struct _sensor sensor_t; 84 | typedef struct _sensor { 85 | sensor_id_t id; // Sensor ID. 86 | uint8_t slv_addr; // Sensor I2C slave address. 87 | pixformat_t pixformat; 88 | camera_status_t status; 89 | 90 | // Sensor function pointers 91 | int (*init_status) (sensor_t *sensor); 92 | int (*reset) (sensor_t *sensor); 93 | int (*set_pixformat) (sensor_t *sensor, pixformat_t pixformat); 94 | int (*set_framesize) (sensor_t *sensor, framesize_t framesize); 95 | int (*set_contrast) (sensor_t *sensor, int level); 96 | int (*set_brightness) (sensor_t *sensor, int level); 97 | int (*set_saturation) (sensor_t *sensor, int level); 98 | int (*set_gainceiling) (sensor_t *sensor, gainceiling_t gainceiling); 99 | int (*set_quality) (sensor_t *sensor, int quality); 100 | int (*set_colorbar) (sensor_t *sensor, int enable); 101 | int (*set_whitebal) (sensor_t *sensor, int enable); 102 | int (*set_gain_ctrl) (sensor_t *sensor, int enable); 103 | int (*set_exposure_ctrl) (sensor_t *sensor, int enable); 104 | int (*set_hmirror) (sensor_t *sensor, int enable); 105 | int (*set_vflip) (sensor_t *sensor, int enable); 106 | 107 | int (*set_aec2) (sensor_t *sensor, int enable); 108 | int (*set_awb_gain) (sensor_t *sensor, int enable); 109 | int (*set_agc_gain) (sensor_t *sensor, int gain); 110 | int (*set_aec_value) (sensor_t *sensor, int gain); 111 | 112 | int (*set_special_effect) (sensor_t *sensor, int effect); 113 | int (*set_wb_mode) (sensor_t *sensor, int mode); 114 | int (*set_ae_level) (sensor_t *sensor, int level); 115 | 116 | int (*set_dcw) (sensor_t *sensor, int enable); 117 | int (*set_bpc) (sensor_t *sensor, int enable); 118 | int (*set_wpc) (sensor_t *sensor, int enable); 119 | 120 | int (*set_raw_gma) (sensor_t *sensor, int enable); 121 | int (*set_lenc) (sensor_t *sensor, int enable); 122 | } sensor_t; 123 | 124 | // Resolution table (in camera.c) 125 | extern const int resolution[][2]; 126 | 127 | #endif /* __SENSOR_H__ */ 128 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/twi.h: -------------------------------------------------------------------------------- 1 | /* 2 | twi.h - Software I2C library for ESP31B 3 | 4 | Copyright (c) 2015 Hristo Gochkov. All rights reserved. 5 | This file is part of the ESP31B core for Arduino environment. 6 | 7 | This library is free software; you can redistribute it and/or 8 | modify it under the terms of the GNU Lesser General Public 9 | License as published by the Free Software Foundation; either 10 | version 2.1 of the License, or (at your option) any later version. 11 | 12 | This library is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 | Lesser General Public License for more details. 16 | 17 | You should have received a copy of the GNU Lesser General Public 18 | License along with this library; if not, write to the Free Software 19 | Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 20 | */ 21 | #ifndef SI2C_h 22 | #define SI2C_h 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | void twi_init(unsigned char sda, unsigned char scl); 29 | void twi_stop(void); 30 | void twi_setClock(unsigned int freq); 31 | uint8_t twi_writeTo(unsigned char address, unsigned char * buf, unsigned int len, unsigned char sendStop); 32 | uint8_t twi_readFrom(unsigned char address, unsigned char * buf, unsigned int len, unsigned char sendStop); 33 | 34 | #ifdef __cplusplus 35 | } 36 | #endif 37 | 38 | #endif -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/xclk.c: -------------------------------------------------------------------------------- 1 | #include "driver/gpio.h" 2 | #include "driver/ledc.h" 3 | #include "esp_err.h" 4 | #include "esp_log.h" 5 | #include "xclk.h" 6 | 7 | #if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG) 8 | #include "esp32-hal-log.h" 9 | #else 10 | #include "esp_log.h" 11 | static const char* TAG = "camera_xclk"; 12 | #endif 13 | 14 | esp_err_t camera_enable_out_clock(camera_config_t* config) 15 | { 16 | periph_module_enable(PERIPH_LEDC_MODULE); 17 | 18 | ledc_timer_config_t timer_conf; 19 | timer_conf.duty_resolution = 2; 20 | timer_conf.freq_hz = config->xclk_freq_hz; 21 | timer_conf.speed_mode = LEDC_HIGH_SPEED_MODE; 22 | timer_conf.timer_num = config->ledc_timer; 23 | esp_err_t err = ledc_timer_config(&timer_conf); 24 | if (err != ESP_OK) { 25 | ESP_LOGE(TAG, "ledc_timer_config failed, rc=%x", err); 26 | return err; 27 | } 28 | 29 | ledc_channel_config_t ch_conf; 30 | ch_conf.gpio_num = config->pin_xclk; 31 | ch_conf.speed_mode = LEDC_HIGH_SPEED_MODE; 32 | ch_conf.channel = config->ledc_channel; 33 | ch_conf.intr_type = LEDC_INTR_DISABLE; 34 | ch_conf.timer_sel = config->ledc_timer; 35 | ch_conf.duty = 2; 36 | ch_conf.hpoint = 0; 37 | err = ledc_channel_config(&ch_conf); 38 | if (err != ESP_OK) { 39 | ESP_LOGE(TAG, "ledc_channel_config failed, rc=%x", err); 40 | return err; 41 | } 42 | return ESP_OK; 43 | } 44 | 45 | void camera_disable_out_clock() 46 | { 47 | periph_module_disable(PERIPH_LEDC_MODULE); 48 | } 49 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_camera/xclk.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "camera_common.h" 4 | 5 | esp_err_t camera_enable_out_clock(); 6 | 7 | void camera_disable_out_clock(); 8 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_http_server/ctrl_sock.c: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "ctrl_sock.h" 23 | 24 | /* Control socket, because in some network stacks select can't be woken up any 25 | * other way 26 | */ 27 | int cs_create_ctrl_sock(int port) 28 | { 29 | int fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); 30 | if (fd < 0) { 31 | return -1; 32 | } 33 | 34 | int ret; 35 | struct sockaddr_in addr; 36 | memset(&addr, 0, sizeof(addr)); 37 | addr.sin_family = AF_INET; 38 | addr.sin_port = htons(port); 39 | inet_aton("127.0.0.1", &addr.sin_addr); 40 | ret = bind(fd, (struct sockaddr *)&addr, sizeof(addr)); 41 | if (ret < 0) { 42 | close(fd); 43 | return -1; 44 | } 45 | return fd; 46 | } 47 | 48 | void cs_free_ctrl_sock(int fd) 49 | { 50 | close(fd); 51 | } 52 | 53 | int cs_send_to_ctrl_sock(int send_fd, int port, void *data, unsigned int data_len) 54 | { 55 | int ret; 56 | struct sockaddr_in to_addr; 57 | to_addr.sin_family = AF_INET; 58 | to_addr.sin_port = htons(port); 59 | inet_aton("127.0.0.1", &to_addr.sin_addr); 60 | ret = sendto(send_fd, data, data_len, 0, (struct sockaddr *)&to_addr, sizeof(to_addr)); 61 | 62 | if (ret < 0) { 63 | return -1; 64 | } 65 | return ret; 66 | } 67 | 68 | int cs_recv_from_ctrl_sock(int fd, void *data, unsigned int data_len) 69 | { 70 | int ret; 71 | ret = recvfrom(fd, data, data_len, 0, NULL, NULL); 72 | 73 | if (ret < 0) { 74 | return -1; 75 | } 76 | return ret; 77 | } 78 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_http_server/ctrl_sock.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /** 16 | * \file ctrl_sock.h 17 | * \brief Control Socket for select() wakeup 18 | * 19 | * LWIP doesn't allow an easy mechanism to on-demand wakeup a thread 20 | * sleeping on select. This is a common requirement for sending 21 | * control commands to a network server. This control socket API 22 | * facilitates the same. 23 | */ 24 | #ifndef _CTRL_SOCK_H_ 25 | #define _CTRL_SOCK_H_ 26 | 27 | #ifdef __cplusplus 28 | extern "C" { 29 | #endif 30 | 31 | /** 32 | * @brief Create a control socket 33 | * 34 | * LWIP doesn't allow an easy mechanism to on-demand wakeup a thread 35 | * sleeping on select. This is a common requirement for sending 36 | * control commands to a network server. This control socket API 37 | * facilitates the same. 38 | * 39 | * This API will create a UDP control socket on the specified port. It 40 | * will return a socket descriptor that can then be added to your 41 | * fd_set in select() 42 | * 43 | * @param[in] port the local port on which the control socket will listen 44 | * 45 | * @return - the socket descriptor that can be added to the fd_set in select. 46 | * - an error code if less than zero 47 | */ 48 | int cs_create_ctrl_sock(int port); 49 | 50 | /** 51 | * @brief Free the control socket 52 | * 53 | * This frees up the control socket that was earlier created using 54 | * cs_create_ctrl_sock() 55 | * 56 | * @param[in] fd the socket descriptor associated with this control socket 57 | */ 58 | void cs_free_ctrl_sock(int fd); 59 | 60 | /** 61 | * @brief Send data to control socket 62 | * 63 | * This API sends data to the control socket. If a server is blocked 64 | * on select() with the control socket, this call will wake up that 65 | * server. 66 | * 67 | * @param[in] send_fd the socket for sending ctrl messages 68 | * @param[in] port the port on which the control socket was created 69 | * @param[in] data pointer to a buffer that contains data to send on the socket 70 | * @param[in] data_len the length of the data contained in the buffer pointed to be data 71 | * 72 | * @return - the number of bytes sent to the control socket 73 | * - an error code if less than zero 74 | */ 75 | int cs_send_to_ctrl_sock(int send_fd, int port, void *data, unsigned int data_len); 76 | 77 | /** 78 | * @brief Receive data from control socket 79 | * 80 | * This API receives any data that was sent to the control 81 | * socket. This will be typically called from the server thread to 82 | * process any commands on this socket. 83 | * 84 | * @param[in] fd the socket descriptor of the control socket 85 | * @param[in] data pointer to a buffer that will be used to store 86 | * received from the control socket 87 | * @param[in] data_len the maximum length of the data that can be 88 | * stored in the buffer pointed by data 89 | * 90 | * @return - the number of bytes received from the control socket 91 | * - an error code if less than zero 92 | */ 93 | int cs_recv_from_ctrl_sock(int fd, void *data, unsigned int data_len); 94 | 95 | #ifdef __cplusplus 96 | } 97 | #endif 98 | 99 | #endif /* ! _CTRL_SOCK_H_ */ 100 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_http_server/http_server.h: -------------------------------------------------------------------------------- 1 | #warning http_server.h has been renamed to esp_http_server.h, please update include directives 2 | #include "esp_http_server.h" 3 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_http_server/httpd_sess.c: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include "esp_httpd_priv.h" 22 | 23 | static const char *TAG = "httpd_sess"; 24 | 25 | bool httpd_is_sess_available(struct httpd_data *hd) 26 | { 27 | int i; 28 | for (i = 0; i < hd->config.max_open_sockets; i++) { 29 | if (hd->hd_sd[i].fd == -1) { 30 | return true; 31 | } 32 | } 33 | return false; 34 | } 35 | 36 | static struct sock_db *httpd_sess_get(struct httpd_data *hd, int newfd) 37 | { 38 | int i; 39 | for (i = 0; i < hd->config.max_open_sockets; i++) { 40 | if (hd->hd_sd[i].fd == newfd) { 41 | return &hd->hd_sd[i]; 42 | } 43 | } 44 | return NULL; 45 | } 46 | 47 | esp_err_t httpd_sess_new(struct httpd_data *hd, int newfd) 48 | { 49 | ESP_LOGD(TAG, LOG_FMT("fd = %d"), newfd); 50 | 51 | if (httpd_sess_get(hd, newfd)) { 52 | ESP_LOGE(TAG, LOG_FMT("session already exists with fd = %d"), newfd); 53 | return ESP_FAIL; 54 | } 55 | 56 | int i; 57 | for (i = 0; i < hd->config.max_open_sockets; i++) { 58 | if (hd->hd_sd[i].fd == -1) { 59 | memset(&hd->hd_sd[i], 0, sizeof(hd->hd_sd[i])); 60 | hd->hd_sd[i].fd = newfd; 61 | hd->hd_sd[i].handle = (httpd_handle_t) hd; 62 | hd->hd_sd[i].send_fn = httpd_default_send; 63 | hd->hd_sd[i].recv_fn = httpd_default_recv; 64 | return ESP_OK; 65 | } 66 | } 67 | ESP_LOGD(TAG, LOG_FMT("unable to launch session for fd = %d"), newfd); 68 | return ESP_FAIL; 69 | } 70 | 71 | void *httpd_sess_get_ctx(httpd_handle_t handle, int sockfd) 72 | { 73 | if (handle == NULL) { 74 | return NULL; 75 | } 76 | 77 | struct httpd_data *hd = (struct httpd_data *) handle; 78 | struct sock_db *sd = httpd_sess_get(hd, sockfd); 79 | if (sd == NULL) { 80 | return NULL; 81 | } 82 | 83 | return sd->ctx; 84 | } 85 | 86 | void httpd_sess_set_descriptors(struct httpd_data *hd, 87 | fd_set *fdset, int *maxfd) 88 | { 89 | int i; 90 | *maxfd = -1; 91 | for (i = 0; i < hd->config.max_open_sockets; i++) { 92 | if (hd->hd_sd[i].fd != -1) { 93 | FD_SET(hd->hd_sd[i].fd, fdset); 94 | if (hd->hd_sd[i].fd > *maxfd) { 95 | *maxfd = hd->hd_sd[i].fd; 96 | } 97 | } 98 | } 99 | } 100 | 101 | int httpd_sess_delete(struct httpd_data *hd, int fd) 102 | { 103 | ESP_LOGD(TAG, LOG_FMT("fd = %d"), fd); 104 | int i; 105 | int pre_sess_fd = -1; 106 | for (i = 0; i < hd->config.max_open_sockets; i++) { 107 | if (hd->hd_sd[i].fd == fd) { 108 | hd->hd_sd[i].fd = -1; 109 | if (hd->hd_sd[i].ctx) { 110 | if (hd->hd_sd[i].free_ctx) { 111 | hd->hd_sd[i].free_ctx(hd->hd_sd[i].ctx); 112 | } else { 113 | free(hd->hd_sd[i].ctx); 114 | } 115 | hd->hd_sd[i].ctx = NULL; 116 | hd->hd_sd[i].free_ctx = NULL; 117 | } 118 | break; 119 | } else if (hd->hd_sd[i].fd != -1) { 120 | /* Return the fd just preceding the one being 121 | * deleted so that iterator can continue from 122 | * the correct fd */ 123 | pre_sess_fd = hd->hd_sd[i].fd; 124 | } 125 | } 126 | return pre_sess_fd; 127 | } 128 | 129 | void httpd_sess_init(struct httpd_data *hd) 130 | { 131 | int i; 132 | for (i = 0; i < hd->config.max_open_sockets; i++) { 133 | hd->hd_sd[i].fd = -1; 134 | hd->hd_sd[i].ctx = NULL; 135 | } 136 | } 137 | 138 | bool httpd_sess_pending(struct httpd_data *hd, int fd) 139 | { 140 | struct sock_db *sd = httpd_sess_get(hd, fd); 141 | if (! sd) { 142 | return ESP_FAIL; 143 | } 144 | 145 | return (sd->pending_len != 0); 146 | } 147 | 148 | /* This MUST return ESP_OK on successful execution. If any other 149 | * value is returned, everything related to this socket will be 150 | * cleaned up and the socket will be closed. 151 | */ 152 | esp_err_t httpd_sess_process(struct httpd_data *hd, int newfd) 153 | { 154 | struct sock_db *sd = httpd_sess_get(hd, newfd); 155 | if (! sd) { 156 | return ESP_FAIL; 157 | } 158 | 159 | ESP_LOGD(TAG, LOG_FMT("httpd_req_new")); 160 | if (httpd_req_new(hd, sd) != ESP_OK) { 161 | return ESP_FAIL; 162 | } 163 | ESP_LOGD(TAG, LOG_FMT("httpd_req_delete")); 164 | if (httpd_req_delete(hd) != ESP_OK) { 165 | return ESP_FAIL; 166 | } 167 | ESP_LOGD(TAG, LOG_FMT("success")); 168 | sd->timestamp = httpd_os_get_timestamp(); 169 | return ESP_OK; 170 | } 171 | 172 | esp_err_t httpd_sess_update_timestamp(httpd_handle_t handle, int sockfd) 173 | { 174 | if (handle == NULL) { 175 | return ESP_ERR_INVALID_ARG; 176 | } 177 | 178 | /* Search for the socket database entry */ 179 | struct httpd_data *hd = (struct httpd_data *) handle; 180 | int i; 181 | for (i = 0; i < hd->config.max_open_sockets; i++) { 182 | if (hd->hd_sd[i].fd == sockfd) { 183 | hd->hd_sd[i].timestamp = httpd_os_get_timestamp(); 184 | return ESP_OK; 185 | } 186 | } 187 | return ESP_ERR_NOT_FOUND; 188 | } 189 | 190 | esp_err_t httpd_sess_close_lru(struct httpd_data *hd) 191 | { 192 | int64_t timestamp = INT64_MAX; 193 | int lru_fd = -1; 194 | int i; 195 | for (i = 0; i < hd->config.max_open_sockets; i++) { 196 | /* If a descriptor is -1, there is no need to close any session. 197 | * So, we can return from here, without finding the Least Recently Used 198 | * session 199 | */ 200 | if (hd->hd_sd[i].fd == -1) { 201 | return ESP_OK; 202 | } 203 | if (hd->hd_sd[i].timestamp < timestamp) { 204 | timestamp = hd->hd_sd[i].timestamp; 205 | lru_fd = hd->hd_sd[i].fd; 206 | } 207 | } 208 | ESP_LOGD(TAG, LOG_FMT("fd = %d"), lru_fd); 209 | return httpd_trigger_sess_close(hd, lru_fd); 210 | } 211 | 212 | int httpd_sess_iterate(struct httpd_data *hd, int start_fd) 213 | { 214 | int start_index = 0; 215 | int i; 216 | 217 | if (start_fd != -1) { 218 | /* Take our index to where this fd is stored */ 219 | for (i = 0; i < hd->config.max_open_sockets; i++) { 220 | if (hd->hd_sd[i].fd == start_fd) { 221 | start_index = i + 1; 222 | break; 223 | } 224 | } 225 | } 226 | 227 | for (i = start_index; i < hd->config.max_open_sockets; i++) { 228 | if (hd->hd_sd[i].fd != -1) { 229 | return hd->hd_sd[i].fd; 230 | } 231 | } 232 | return -1; 233 | } 234 | 235 | static void httpd_sess_close(void *arg) 236 | { 237 | struct sock_db *sock_db = (struct sock_db *)arg; 238 | if (sock_db) { 239 | int fd = sock_db->fd; 240 | struct httpd_data *hd = (struct httpd_data *) sock_db->handle; 241 | httpd_sess_delete(hd, fd); 242 | close(fd); 243 | } 244 | } 245 | 246 | esp_err_t httpd_trigger_sess_close(httpd_handle_t handle, int sockfd) 247 | { 248 | if (handle == NULL) { 249 | return ESP_ERR_INVALID_ARG; 250 | } 251 | 252 | struct httpd_data *hd = (struct httpd_data *) handle; 253 | struct sock_db *sock_db = httpd_sess_get(hd, sockfd); 254 | if (sock_db) { 255 | return httpd_queue_work(handle, httpd_sess_close, sock_db); 256 | } 257 | 258 | return ESP_ERR_NOT_FOUND; 259 | } 260 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_http_server/httpd_uri.c: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | #include "esp_httpd_priv.h" 23 | 24 | static const char *TAG = "httpd_uri"; 25 | 26 | static int httpd_find_uri_handler(struct httpd_data *hd, 27 | const char* uri, 28 | httpd_method_t method) 29 | { 30 | for (int i = 0; i < hd->config.max_uri_handlers; i++) { 31 | if (hd->hd_calls[i]) { 32 | ESP_LOGD(TAG, LOG_FMT("[%d] = %s"), i, hd->hd_calls[i]->uri); 33 | if ((hd->hd_calls[i]->method == method) && // First match methods 34 | (strcmp(hd->hd_calls[i]->uri, uri) == 0)) { // Then match uri strings 35 | return i; 36 | } 37 | } 38 | } 39 | return -1; 40 | } 41 | 42 | esp_err_t httpd_register_uri_handler(httpd_handle_t handle, 43 | const httpd_uri_t *uri_handler) 44 | { 45 | if (handle == NULL || uri_handler == NULL) { 46 | return ESP_ERR_INVALID_ARG; 47 | } 48 | 49 | struct httpd_data *hd = (struct httpd_data *) handle; 50 | 51 | /* Make sure another handler with same URI and method 52 | * is not already registered 53 | */ 54 | if (httpd_find_uri_handler(handle, uri_handler->uri, 55 | uri_handler->method) != -1) { 56 | ESP_LOGW(TAG, LOG_FMT("handler %s with method %d already registered"), 57 | uri_handler->uri, uri_handler->method); 58 | return ESP_ERR_HTTPD_HANDLER_EXISTS; 59 | } 60 | 61 | for (int i = 0; i < hd->config.max_uri_handlers; i++) { 62 | if (hd->hd_calls[i] == NULL) { 63 | hd->hd_calls[i] = malloc(sizeof(httpd_uri_t)); 64 | if (hd->hd_calls[i] == NULL) { 65 | /* Failed to allocate memory */ 66 | return ESP_ERR_HTTPD_ALLOC_MEM; 67 | } 68 | 69 | /* Copy URI string */ 70 | hd->hd_calls[i]->uri = strdup(uri_handler->uri); 71 | if (hd->hd_calls[i]->uri == NULL) { 72 | /* Failed to allocate memory */ 73 | free(hd->hd_calls[i]); 74 | return ESP_ERR_HTTPD_ALLOC_MEM; 75 | } 76 | 77 | /* Copy remaining members */ 78 | hd->hd_calls[i]->method = uri_handler->method; 79 | hd->hd_calls[i]->handler = uri_handler->handler; 80 | hd->hd_calls[i]->user_ctx = uri_handler->user_ctx; 81 | ESP_LOGD(TAG, LOG_FMT("[%d] installed %s"), i, uri_handler->uri); 82 | return ESP_OK; 83 | } 84 | ESP_LOGD(TAG, LOG_FMT("[%d] exists %s"), i, hd->hd_calls[i]->uri); 85 | } 86 | ESP_LOGW(TAG, LOG_FMT("no slots left for registering handler")); 87 | return ESP_ERR_HTTPD_HANDLERS_FULL; 88 | } 89 | 90 | esp_err_t httpd_unregister_uri_handler(httpd_handle_t handle, 91 | const char *uri, httpd_method_t method) 92 | { 93 | if (handle == NULL || uri == NULL) { 94 | return ESP_ERR_INVALID_ARG; 95 | } 96 | 97 | struct httpd_data *hd = (struct httpd_data *) handle; 98 | int i = httpd_find_uri_handler(hd, uri, method); 99 | 100 | if (i != -1) { 101 | ESP_LOGD(TAG, LOG_FMT("[%d] removing %s"), i, hd->hd_calls[i]->uri); 102 | 103 | free((char*)hd->hd_calls[i]->uri); 104 | free(hd->hd_calls[i]); 105 | hd->hd_calls[i] = NULL; 106 | return ESP_OK; 107 | } 108 | ESP_LOGW(TAG, LOG_FMT("handler %s with method %d not found"), uri, method); 109 | return ESP_ERR_NOT_FOUND; 110 | } 111 | 112 | esp_err_t httpd_unregister_uri(httpd_handle_t handle, const char *uri) 113 | { 114 | if (handle == NULL || uri == NULL) { 115 | return ESP_ERR_INVALID_ARG; 116 | } 117 | 118 | struct httpd_data *hd = (struct httpd_data *) handle; 119 | bool found = false; 120 | 121 | for (int i = 0; i < hd->config.max_uri_handlers; i++) { 122 | if ((hd->hd_calls[i] != NULL) && 123 | (strcmp(hd->hd_calls[i]->uri, uri) == 0)) { 124 | ESP_LOGD(TAG, LOG_FMT("[%d] removing %s"), i, uri); 125 | 126 | free((char*)hd->hd_calls[i]->uri); 127 | free(hd->hd_calls[i]); 128 | hd->hd_calls[i] = NULL; 129 | found = true; 130 | } 131 | } 132 | if (!found) { 133 | ESP_LOGW(TAG, LOG_FMT("no handler found for URI %s"), uri); 134 | } 135 | return (found ? ESP_OK : ESP_ERR_NOT_FOUND); 136 | } 137 | 138 | void httpd_unregister_all_uri_handlers(struct httpd_data *hd) 139 | { 140 | for (unsigned i = 0; i < hd->config.max_uri_handlers; i++) { 141 | if (hd->hd_calls[i]) { 142 | ESP_LOGD(TAG, LOG_FMT("[%d] removing %s"), i, hd->hd_calls[i]->uri); 143 | 144 | free((char*)hd->hd_calls[i]->uri); 145 | free(hd->hd_calls[i]); 146 | } 147 | } 148 | } 149 | 150 | /* Alternate implmentation of httpd_find_uri_handler() 151 | * which takes a uri_len field. This is useful when the URI 152 | * string contains extra parameters that are not to be included 153 | * while matching with the registered URI_handler strings 154 | */ 155 | static httpd_uri_t* httpd_find_uri_handler2(httpd_err_resp_t *err, 156 | struct httpd_data *hd, 157 | const char *uri, size_t uri_len, 158 | httpd_method_t method) 159 | { 160 | *err = 0; 161 | for (int i = 0; i < hd->config.max_uri_handlers; i++) { 162 | if (hd->hd_calls[i]) { 163 | ESP_LOGD(TAG, LOG_FMT("[%d] = %s"), i, hd->hd_calls[i]->uri); 164 | if ((strlen(hd->hd_calls[i]->uri) == uri_len) && // First match uri length 165 | (strncmp(hd->hd_calls[i]->uri, uri, uri_len) == 0)) { // Then match uri strings 166 | if (hd->hd_calls[i]->method == method) { // Finally match methods 167 | return hd->hd_calls[i]; 168 | } 169 | /* URI found but method not allowed. 170 | * If URI IS found later then this 171 | * error is to be neglected */ 172 | *err = HTTPD_405_METHOD_NOT_ALLOWED; 173 | } 174 | } 175 | } 176 | if (*err == 0) { 177 | *err = HTTPD_404_NOT_FOUND; 178 | } 179 | return NULL; 180 | } 181 | 182 | esp_err_t httpd_uri(struct httpd_data *hd) 183 | { 184 | httpd_uri_t *uri = NULL; 185 | httpd_req_t *req = &hd->hd_req; 186 | struct http_parser_url *res = &hd->hd_req_aux.url_parse_res; 187 | 188 | /* For conveying URI not found/method not allowed */ 189 | httpd_err_resp_t err = 0; 190 | 191 | ESP_LOGD(TAG, LOG_FMT("request for %s with type %d"), req->uri, req->method); 192 | /* URL parser result contains offset and length of path string */ 193 | if (res->field_set & (1 << UF_PATH)) { 194 | uri = httpd_find_uri_handler2(&err, hd, 195 | req->uri + res->field_data[UF_PATH].off, 196 | res->field_data[UF_PATH].len, 197 | req->method); 198 | } 199 | 200 | /* If URI with method not found, respond with error code */ 201 | if (uri == NULL) { 202 | switch (err) { 203 | case HTTPD_404_NOT_FOUND: 204 | return httpd_resp_send_err(req, HTTPD_404_NOT_FOUND); 205 | case HTTPD_405_METHOD_NOT_ALLOWED: 206 | return httpd_resp_send_err(req, HTTPD_405_METHOD_NOT_ALLOWED); 207 | default: 208 | return ESP_FAIL; 209 | } 210 | } 211 | 212 | /* Attach user context data (passed during URI registration) into request */ 213 | req->user_ctx = uri->user_ctx; 214 | 215 | /* Invoke handler */ 216 | if (uri->handler(req) != ESP_OK) { 217 | /* Handler returns error, this socket should be closed */ 218 | ESP_LOGW(TAG, LOG_FMT("uri handler execution failed")); 219 | return ESP_FAIL; 220 | } 221 | return ESP_OK; 222 | } 223 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/esp_http_server/osal.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef _OSAL_H_ 16 | #define _OSAL_H_ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | #define OS_SUCCESS ESP_OK 29 | #define OS_FAIL ESP_FAIL 30 | 31 | typedef TaskHandle_t othread_t; 32 | 33 | static inline int httpd_os_thread_create(othread_t *thread, 34 | const char *name, uint16_t stacksize, int prio, 35 | void (*thread_routine)(void *arg), void *arg) 36 | { 37 | int ret = xTaskCreate(thread_routine, name, stacksize, arg, prio, thread); 38 | if (ret == pdPASS) { 39 | return OS_SUCCESS; 40 | } 41 | return OS_FAIL; 42 | } 43 | 44 | /* Only self delete is supported */ 45 | static inline void httpd_os_thread_delete() 46 | { 47 | vTaskDelete(xTaskGetCurrentTaskHandle()); 48 | } 49 | 50 | static inline void httpd_os_thread_sleep(int msecs) 51 | { 52 | vTaskDelay(msecs / portTICK_RATE_MS); 53 | } 54 | 55 | static inline int64_t httpd_os_get_timestamp() 56 | { 57 | return esp_timer_get_time(); 58 | } 59 | 60 | static inline othread_t httpd_os_thread_handle() 61 | { 62 | return xTaskGetCurrentTaskHandle(); 63 | } 64 | 65 | #ifdef __cplusplus 66 | } 67 | #endif 68 | 69 | #endif /* ! _OSAL_H_ */ 70 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/img_converters/esp_jpg_decode.c: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "esp_jpg_decode.h" 15 | #include "rom/tjpgd.h" 16 | 17 | #if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG) 18 | #include "esp32-hal-log.h" 19 | #define TAG "" 20 | #else 21 | #include "esp_log.h" 22 | static const char* TAG = "esp_jpg_decode"; 23 | #endif 24 | 25 | typedef struct { 26 | jpg_scale_t scale; 27 | jpg_reader_cb reader; 28 | jpg_writer_cb writer; 29 | void * arg; 30 | size_t len; 31 | size_t index; 32 | } esp_jpg_decoder_t; 33 | 34 | static const char * jd_errors[] = { 35 | "Succeeded", 36 | "Interrupted by output function", 37 | "Device error or wrong termination of input stream", 38 | "Insufficient memory pool for the image", 39 | "Insufficient stream input buffer", 40 | "Parameter error", 41 | "Data format error", 42 | "Right format but not supported", 43 | "Not supported JPEG standard" 44 | }; 45 | 46 | static uint32_t _jpg_write(JDEC *decoder, void *bitmap, JRECT *rect) 47 | { 48 | uint16_t x = rect->left; 49 | uint16_t y = rect->top; 50 | uint16_t w = rect->right + 1 - x; 51 | uint16_t h = rect->bottom + 1 - y; 52 | uint8_t *data = (uint8_t *)bitmap; 53 | 54 | esp_jpg_decoder_t * jpeg = (esp_jpg_decoder_t *)decoder->device; 55 | 56 | if (jpeg->writer) { 57 | return jpeg->writer(jpeg->arg, x, y, w, h, data); 58 | } 59 | return 0; 60 | } 61 | 62 | static uint32_t _jpg_read(JDEC *decoder, uint8_t *buf, uint32_t len) 63 | { 64 | esp_jpg_decoder_t * jpeg = (esp_jpg_decoder_t *)decoder->device; 65 | if (jpeg->len && len > (jpeg->len - jpeg->index)) { 66 | len = jpeg->len - jpeg->index; 67 | } 68 | if (len) { 69 | len = jpeg->reader(jpeg->arg, jpeg->index, buf, len); 70 | if (!len) { 71 | ESP_LOGE(TAG, "Read Fail at %u/%u", jpeg->index, jpeg->len); 72 | } 73 | jpeg->index += len; 74 | } 75 | return len; 76 | } 77 | 78 | esp_err_t esp_jpg_decode(size_t len, jpg_scale_t scale, jpg_reader_cb reader, jpg_writer_cb writer, void * arg) 79 | { 80 | static uint8_t work[3100]; 81 | JDEC decoder; 82 | esp_jpg_decoder_t jpeg; 83 | 84 | jpeg.len = len; 85 | jpeg.reader = reader; 86 | jpeg.writer = writer; 87 | jpeg.arg = arg; 88 | jpeg.scale = scale; 89 | jpeg.index = 0; 90 | 91 | JRESULT jres = jd_prepare(&decoder, _jpg_read, work, 3100, &jpeg); 92 | if(jres != JDR_OK){ 93 | ESP_LOGE(TAG, "JPG Header Parse Failed! %s", jd_errors[jres]); 94 | return ESP_FAIL; 95 | } 96 | 97 | uint16_t output_width = decoder.width / (1 << (uint8_t)(jpeg.scale)); 98 | uint16_t output_height = decoder.height / (1 << (uint8_t)(jpeg.scale)); 99 | 100 | //output start 101 | writer(arg, 0, 0, output_width, output_height, NULL); 102 | //output write 103 | jres = jd_decomp(&decoder, _jpg_write, (uint8_t)jpeg.scale); 104 | //output end 105 | writer(arg, output_width, output_height, output_width, output_height, NULL); 106 | 107 | if (jres != JDR_OK) { 108 | ESP_LOGE(TAG, "JPG Decompression Failed! %s", jd_errors[jres]); 109 | return ESP_FAIL; 110 | } 111 | //check if all data has been consumed. 112 | if (len && jpeg.index < len) { 113 | _jpg_read(&decoder, NULL, len - jpeg.index); 114 | } 115 | 116 | return ESP_OK; 117 | } 118 | 119 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/img_converters/esp_jpg_decode.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef _ESP_JPG_DECODE_H_ 15 | #define _ESP_JPG_DECODE_H_ 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | #include 22 | #include 23 | #include 24 | #include "esp_err.h" 25 | 26 | typedef enum { 27 | JPG_SCALE_NONE, 28 | JPG_SCALE_2X, 29 | JPG_SCALE_4X, 30 | JPG_SCALE_8X, 31 | JPG_SCALE_MAX = JPG_SCALE_8X 32 | } jpg_scale_t; 33 | 34 | typedef size_t (* jpg_reader_cb)(void * arg, size_t index, uint8_t *buf, size_t len); 35 | typedef bool (* jpg_writer_cb)(void * arg, uint16_t x, uint16_t y, uint16_t w, uint16_t h, uint8_t *data); 36 | 37 | esp_err_t esp_jpg_decode(size_t len, jpg_scale_t scale, jpg_reader_cb reader, jpg_writer_cb writer, void * arg); 38 | 39 | #ifdef __cplusplus 40 | } 41 | #endif 42 | 43 | #endif /* _ESP_JPG_DECODE_H_ */ 44 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/img_converters/img_converters.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef _IMG_CONVERTERS_H_ 15 | #define _IMG_CONVERTERS_H_ 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | #include 22 | #include 23 | #include 24 | #include "esp_camera.h" 25 | 26 | typedef size_t (* jpg_out_cb)(void * arg, size_t index, const void* data, size_t len); 27 | 28 | /** 29 | * @brief Convert image buffer to JPEG 30 | * 31 | * @param src Source buffer in RGB565, RGB888, YUYV or GRAYSCALE format 32 | * @param src_len Length in bytes of the source buffer 33 | * @param width Width in pixels of the source image 34 | * @param height Height in pixels of the source image 35 | * @param format Format of the source image 36 | * @param quality JPEG quality of the resulting image 37 | * @param cp Callback to be called to write the bytes of the output JPEG 38 | * @param arg Pointer to be passed to the callback 39 | * 40 | * @return true on success 41 | */ 42 | bool fmt2jpg_cb(uint8_t *src, size_t src_len, uint16_t width, uint16_t height, pixformat_t format, uint8_t quality, jpg_out_cb cb, void * arg); 43 | 44 | /** 45 | * @brief Convert camera frame buffer to JPEG 46 | * 47 | * @param fb Source camera frame buffer 48 | * @param quality JPEG quality of the resulting image 49 | * @param cp Callback to be called to write the bytes of the output JPEG 50 | * @param arg Pointer to be passed to the callback 51 | * 52 | * @return true on success 53 | */ 54 | bool frame2jpg_cb(camera_fb_t * fb, uint8_t quality, jpg_out_cb cb, void * arg); 55 | 56 | /** 57 | * @brief Convert image buffer to JPEG buffer 58 | * 59 | * @param src Source buffer in RGB565, RGB888, YUYV or GRAYSCALE format 60 | * @param src_len Length in bytes of the source buffer 61 | * @param width Width in pixels of the source image 62 | * @param height Height in pixels of the source image 63 | * @param format Format of the source image 64 | * @param quality JPEG quality of the resulting image 65 | * @param out Pointer to be populated with the address of the resulting buffer 66 | * @param out_len Pointer to be populated with the length of the output buffer 67 | * 68 | * @return true on success 69 | */ 70 | bool fmt2jpg(uint8_t *src, size_t src_len, uint16_t width, uint16_t height, pixformat_t format, uint8_t quality, uint8_t ** out, size_t * out_len); 71 | 72 | /** 73 | * @brief Convert camera frame buffer to JPEG buffer 74 | * 75 | * @param fb Source camera frame buffer 76 | * @param quality JPEG quality of the resulting image 77 | * @param out Pointer to be populated with the address of the resulting buffer 78 | * @param out_len Pointer to be populated with the length of the output buffer 79 | * 80 | * @return true on success 81 | */ 82 | bool frame2jpg(camera_fb_t * fb, uint8_t quality, uint8_t ** out, size_t * out_len); 83 | 84 | /** 85 | * @brief Convert image buffer to BMP buffer 86 | * 87 | * @param src Source buffer in JPEG, RGB565, RGB888, YUYV or GRAYSCALE format 88 | * @param src_len Length in bytes of the source buffer 89 | * @param width Width in pixels of the source image 90 | * @param height Height in pixels of the source image 91 | * @param format Format of the source image 92 | * @param out Pointer to be populated with the address of the resulting buffer 93 | * @param out_len Pointer to be populated with the length of the output buffer 94 | * 95 | * @return true on success 96 | */ 97 | bool fmt2bmp(uint8_t *src, size_t src_len, uint16_t width, uint16_t height, pixformat_t format, uint8_t ** out, size_t * out_len); 98 | 99 | /** 100 | * @brief Convert camera frame buffer to BMP buffer 101 | * 102 | * @param fb Source camera frame buffer 103 | * @param out Pointer to be populated with the address of the resulting buffer 104 | * @param out_len Pointer to be populated with the length of the output buffer 105 | * 106 | * @return true on success 107 | */ 108 | bool frame2bmp(camera_fb_t * fb, uint8_t ** out, size_t * out_len); 109 | 110 | /** 111 | * @brief Convert image buffer to RGB888 buffer (used for face detection) 112 | * 113 | * @param src Source buffer in JPEG, RGB565, RGB888, YUYV or GRAYSCALE format 114 | * @param src_len Length in bytes of the source buffer 115 | * @param format Format of the source image 116 | * @param rgb_buf Pointer to the output buffer (width * height * 3) 117 | * 118 | * @return true on success 119 | */ 120 | bool fmt2rgb888(const uint8_t *src_buf, size_t src_len, pixformat_t format, uint8_t * rgb_buf); 121 | 122 | #ifdef __cplusplus 123 | } 124 | #endif 125 | 126 | #endif /* _IMG_CONVERTERS_H_ */ 127 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/img_converters/jpge.h: -------------------------------------------------------------------------------- 1 | // jpge.h - C++ class for JPEG compression. 2 | // Public domain, Rich Geldreich 3 | // Alex Evans: Added RGBA support, linear memory allocator. 4 | #ifndef JPEG_ENCODER_H 5 | #define JPEG_ENCODER_H 6 | 7 | namespace jpge 8 | { 9 | typedef unsigned char uint8; 10 | typedef signed short int16; 11 | typedef signed int int32; 12 | typedef unsigned short uint16; 13 | typedef unsigned int uint32; 14 | typedef unsigned int uint; 15 | 16 | // JPEG chroma subsampling factors. Y_ONLY (grayscale images) and H2V2 (color images) are the most common. 17 | enum subsampling_t { Y_ONLY = 0, H1V1 = 1, H2V1 = 2, H2V2 = 3 }; 18 | 19 | // JPEG compression parameters structure. 20 | struct params { 21 | inline params() : m_quality(85), m_subsampling(H2V2) { } 22 | 23 | inline bool check() const { 24 | if ((m_quality < 1) || (m_quality > 100)) { 25 | return false; 26 | } 27 | if ((uint)m_subsampling > (uint)H2V2) { 28 | return false; 29 | } 30 | return true; 31 | } 32 | 33 | // Quality: 1-100, higher is better. Typical values are around 50-95. 34 | int m_quality; 35 | 36 | // m_subsampling: 37 | // 0 = Y (grayscale) only 38 | // 1 = H1V1 subsampling (YCbCr 1x1x1, 3 blocks per MCU) 39 | // 2 = H2V1 subsampling (YCbCr 2x1x1, 4 blocks per MCU) 40 | // 3 = H2V2 subsampling (YCbCr 4x1x1, 6 blocks per MCU-- very common) 41 | subsampling_t m_subsampling; 42 | }; 43 | 44 | // Output stream abstract class - used by the jpeg_encoder class to write to the output stream. 45 | // put_buf() is generally called with len==JPGE_OUT_BUF_SIZE bytes, but for headers it'll be called with smaller amounts. 46 | class output_stream { 47 | public: 48 | virtual ~output_stream() { }; 49 | virtual bool put_buf(const void* Pbuf, int len) = 0; 50 | virtual uint get_size() const = 0; 51 | }; 52 | 53 | // Lower level jpeg_encoder class - useful if more control is needed than the above helper functions. 54 | class jpeg_encoder { 55 | public: 56 | jpeg_encoder(); 57 | ~jpeg_encoder(); 58 | 59 | // Initializes the compressor. 60 | // pStream: The stream object to use for writing compressed data. 61 | // params - Compression parameters structure, defined above. 62 | // width, height - Image dimensions. 63 | // channels - May be 1, or 3. 1 indicates grayscale, 3 indicates RGB source data. 64 | // Returns false on out of memory or if a stream write fails. 65 | bool init(output_stream *pStream, int width, int height, int src_channels, const params &comp_params = params()); 66 | 67 | // Call this method with each source scanline. 68 | // width * src_channels bytes per scanline is expected (RGB or Y format). 69 | // You must call with NULL after all scanlines are processed to finish compression. 70 | // Returns false on out of memory or if a stream write fails. 71 | bool process_scanline(const void* pScanline); 72 | 73 | // Deinitializes the compressor, freeing any allocated memory. May be called at any time. 74 | void deinit(); 75 | 76 | private: 77 | jpeg_encoder(const jpeg_encoder &); 78 | jpeg_encoder &operator =(const jpeg_encoder &); 79 | 80 | typedef int32 sample_array_t; 81 | enum { JPGE_OUT_BUF_SIZE = 512 }; 82 | 83 | output_stream *m_pStream; 84 | params m_params; 85 | uint8 m_num_components; 86 | uint8 m_comp_h_samp[3], m_comp_v_samp[3]; 87 | int m_image_x, m_image_y, m_image_bpp, m_image_bpl; 88 | int m_image_x_mcu, m_image_y_mcu; 89 | int m_image_bpl_xlt, m_image_bpl_mcu; 90 | int m_mcus_per_row; 91 | int m_mcu_x, m_mcu_y; 92 | uint8 *m_mcu_lines[16]; 93 | uint8 m_mcu_y_ofs; 94 | sample_array_t m_sample_array[64]; 95 | int16 m_coefficient_array[64]; 96 | 97 | int m_last_dc_val[3]; 98 | uint8 m_out_buf[JPGE_OUT_BUF_SIZE]; 99 | uint8 *m_pOut_buf; 100 | uint m_out_buf_left; 101 | uint32 m_bit_buffer; 102 | uint m_bits_in; 103 | uint8 m_pass_num; 104 | bool m_all_stream_writes_succeeded; 105 | 106 | bool jpg_open(int p_x_res, int p_y_res, int src_channels); 107 | 108 | void flush_output_buffer(); 109 | void put_bits(uint bits, uint len); 110 | 111 | void emit_byte(uint8 i); 112 | void emit_word(uint i); 113 | void emit_marker(int marker); 114 | 115 | void emit_jfif_app0(); 116 | void emit_dqt(); 117 | void emit_sof(); 118 | void emit_dht(uint8 *bits, uint8 *val, int index, bool ac_flag); 119 | void emit_dhts(); 120 | void emit_sos(); 121 | 122 | void compute_quant_table(int32 *dst, const int16 *src); 123 | void load_quantized_coefficients(int component_num); 124 | 125 | void load_block_8_8_grey(int x); 126 | void load_block_8_8(int x, int y, int c); 127 | void load_block_16_8(int x, int c); 128 | void load_block_16_8_8(int x, int c); 129 | 130 | void code_coefficients_pass_two(int component_num); 131 | void code_block(int component_num); 132 | 133 | void process_mcu_row(); 134 | bool process_end_of_image(); 135 | void load_mcu(const void* src); 136 | void clear(); 137 | void init(); 138 | }; 139 | 140 | } // namespace jpge 141 | 142 | #endif // JPEG_ENCODER 143 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/img_converters/to_bmp.c: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | #include 16 | #include "img_converters.h" 17 | #include "esp_spiram.h" 18 | #include "soc/efuse_reg.h" 19 | #include "esp_heap_caps.h" 20 | #include "yuv.h" 21 | #include "sdkconfig.h" 22 | #include "esp_jpg_decode.h" 23 | 24 | #if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG) 25 | #include "esp32-hal-log.h" 26 | #define TAG "" 27 | #else 28 | #include "esp_log.h" 29 | static const char* TAG = "to_bmp"; 30 | #endif 31 | 32 | static const int BMP_HEADER_LEN = 54; 33 | 34 | typedef struct { 35 | uint32_t filesize; 36 | uint32_t reserved; 37 | uint32_t fileoffset_to_pixelarray; 38 | uint32_t dibheadersize; 39 | int32_t width; 40 | int32_t height; 41 | uint16_t planes; 42 | uint16_t bitsperpixel; 43 | uint32_t compression; 44 | uint32_t imagesize; 45 | uint32_t ypixelpermeter; 46 | uint32_t xpixelpermeter; 47 | uint32_t numcolorspallette; 48 | uint32_t mostimpcolor; 49 | } bmp_header_t; 50 | 51 | typedef struct { 52 | uint16_t width; 53 | uint16_t height; 54 | uint16_t data_offset; 55 | const uint8_t *input; 56 | uint8_t *output; 57 | } rgb_jpg_decoder; 58 | 59 | static void *_malloc(size_t size) 60 | { 61 | return heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); 62 | } 63 | 64 | //output buffer and image width 65 | static bool _rgb_write(void * arg, uint16_t x, uint16_t y, uint16_t w, uint16_t h, uint8_t *data) 66 | { 67 | rgb_jpg_decoder * jpeg = (rgb_jpg_decoder *)arg; 68 | if(!data){ 69 | if(x == 0 && y == 0){ 70 | //write start 71 | jpeg->width = w; 72 | jpeg->height = h; 73 | //if output is null, this is BMP 74 | if(!jpeg->output){ 75 | jpeg->output = (uint8_t *)_malloc((w*h*3)+jpeg->data_offset); 76 | if(!jpeg->output){ 77 | return false; 78 | } 79 | } 80 | } else { 81 | //write end 82 | } 83 | return true; 84 | } 85 | 86 | size_t jw = jpeg->width*3; 87 | size_t t = y * jw; 88 | size_t b = t + (h * jw); 89 | size_t l = x * 3; 90 | uint8_t *out = jpeg->output+jpeg->data_offset; 91 | uint8_t *o = out; 92 | size_t iy, ix; 93 | 94 | w = w * 3; 95 | 96 | for(iy=t; iyinput + index, len); 114 | } 115 | return len; 116 | } 117 | 118 | static bool jpg2rgb888(const uint8_t *src, size_t src_len, uint8_t * out, jpg_scale_t scale) 119 | { 120 | rgb_jpg_decoder jpeg; 121 | jpeg.width = 0; 122 | jpeg.height = 0; 123 | jpeg.input = src; 124 | jpeg.output = out; 125 | jpeg.data_offset = 0; 126 | 127 | if(esp_jpg_decode(src_len, scale, _jpg_read, _rgb_write, (void*)&jpeg) != ESP_OK){ 128 | return false; 129 | } 130 | return true; 131 | } 132 | 133 | bool jpg2bmp(const uint8_t *src, size_t src_len, uint8_t ** out, size_t * out_len) 134 | { 135 | 136 | rgb_jpg_decoder jpeg; 137 | jpeg.width = 0; 138 | jpeg.height = 0; 139 | jpeg.input = src; 140 | jpeg.output = NULL; 141 | jpeg.data_offset = BMP_HEADER_LEN; 142 | 143 | if(esp_jpg_decode(src_len, JPG_SCALE_NONE, _jpg_read, _rgb_write, (void*)&jpeg) != ESP_OK){ 144 | return false; 145 | } 146 | 147 | size_t output_size = jpeg.width*jpeg.height*3; 148 | 149 | jpeg.output[0] = 'B'; 150 | jpeg.output[1] = 'M'; 151 | bmp_header_t * bitmap = (bmp_header_t*)&jpeg.output[2]; 152 | bitmap->reserved = 0; 153 | bitmap->filesize = output_size+BMP_HEADER_LEN; 154 | bitmap->fileoffset_to_pixelarray = BMP_HEADER_LEN; 155 | bitmap->dibheadersize = 40; 156 | bitmap->width = jpeg.width; 157 | bitmap->height = -jpeg.height;//set negative for top to bottom 158 | bitmap->planes = 1; 159 | bitmap->bitsperpixel = 24; 160 | bitmap->compression = 0; 161 | bitmap->imagesize = output_size; 162 | bitmap->ypixelpermeter = 0x0B13 ; //2835 , 72 DPI 163 | bitmap->xpixelpermeter = 0x0B13 ; //2835 , 72 DPI 164 | bitmap->numcolorspallette = 0; 165 | bitmap->mostimpcolor = 0; 166 | 167 | *out = jpeg.output; 168 | *out_len = output_size+BMP_HEADER_LEN; 169 | 170 | return true; 171 | } 172 | 173 | bool fmt2rgb888(const uint8_t *src_buf, size_t src_len, pixformat_t format, uint8_t * rgb_buf) 174 | { 175 | int pix_count = 0; 176 | if(format == PIXFORMAT_JPEG) { 177 | return jpg2rgb888(src_buf, src_len, rgb_buf, JPG_SCALE_NONE); 178 | } else if(format == PIXFORMAT_RGB888) { 179 | memcpy(rgb_buf, src_buf, src_len); 180 | } else if(format == PIXFORMAT_RGB565) { 181 | int i; 182 | uint8_t hb, lb; 183 | pix_count = src_len / 2; 184 | for(i=0; i> 3; 189 | *rgb_buf++ = hb & 0xF8; 190 | } 191 | } else if(format == PIXFORMAT_GRAYSCALE) { 192 | int i; 193 | uint8_t b; 194 | pix_count = src_len; 195 | for(i=0; ireserved = 0; 247 | bitmap->filesize = out_size; 248 | bitmap->fileoffset_to_pixelarray = BMP_HEADER_LEN; 249 | bitmap->dibheadersize = 40; 250 | bitmap->width = width; 251 | bitmap->height = -height;//set negative for top to bottom 252 | bitmap->planes = 1; 253 | bitmap->bitsperpixel = 24; 254 | bitmap->compression = 0; 255 | bitmap->imagesize = pix_count * 3; 256 | bitmap->ypixelpermeter = 0x0B13 ; //2835 , 72 DPI 257 | bitmap->xpixelpermeter = 0x0B13 ; //2835 , 72 DPI 258 | bitmap->numcolorspallette = 0; 259 | bitmap->mostimpcolor = 0; 260 | 261 | uint8_t * rgb_buf = out_buf + BMP_HEADER_LEN; 262 | uint8_t * src_buf = src; 263 | 264 | 265 | //convert data to RGB888 266 | if(format == PIXFORMAT_RGB888) { 267 | memcpy(rgb_buf, src_buf, pix_count*3); 268 | } else if(format == PIXFORMAT_RGB565) { 269 | int i; 270 | uint8_t hb, lb; 271 | for(i=0; i> 3; 276 | *rgb_buf++ = hb & 0xF8; 277 | } 278 | } else if(format == PIXFORMAT_GRAYSCALE) { 279 | int i; 280 | uint8_t b; 281 | for(i=0; ibuf, fb->len, fb->width, fb->height, fb->format, out, out_len); 316 | } 317 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/img_converters/to_jpg.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | #include 16 | #include "esp_spiram.h" 17 | #include "esp_attr.h" 18 | #include "soc/efuse_reg.h" 19 | #include "esp_heap_caps.h" 20 | #include "esp_camera.h" 21 | #include "img_converters.h" 22 | #include "jpge.h" 23 | #include "yuv.h" 24 | 25 | #if defined(ARDUINO_ARCH_ESP32) && defined(CONFIG_ARDUHAL_ESP_LOG) 26 | #include "esp32-hal-log.h" 27 | #define TAG "" 28 | #else 29 | #include "esp_log.h" 30 | static const char* TAG = "to_bmp"; 31 | #endif 32 | 33 | static void *_malloc(size_t size) 34 | { 35 | void * res = malloc(size); 36 | if(res) { 37 | return res; 38 | } 39 | return heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); 40 | } 41 | 42 | static IRAM_ATTR void convert_line_format(uint8_t * src, pixformat_t format, uint8_t * dst, size_t width, size_t in_channels, size_t line) 43 | { 44 | int i=0, o=0, l=0; 45 | if(format == PIXFORMAT_GRAYSCALE) { 46 | memcpy(dst, src + line * width, width); 47 | } else if(format == PIXFORMAT_RGB888) { 48 | l = width * 3; 49 | src += l * line; 50 | for(i=0; i> 3; 61 | dst[o++] = (src[i+1] & 0x1F) << 3; 62 | } 63 | } else if(format == PIXFORMAT_YUV422) { 64 | uint8_t y0, y1, u, v; 65 | uint8_t r, g, b; 66 | l = width * 2; 67 | src += l * line; 68 | for(i=0; i 100) { 100 | quality = 100; 101 | } 102 | 103 | jpge::params comp_params = jpge::params(); 104 | comp_params.m_subsampling = subsampling; 105 | comp_params.m_quality = quality; 106 | 107 | jpge::jpeg_encoder dst_image; 108 | 109 | if (!dst_image.init(dst_stream, width, height, num_channels, comp_params)) { 110 | ESP_LOGE(TAG, "JPG encoder init failed"); 111 | return false; 112 | } 113 | 114 | uint8_t* line = (uint8_t*)_malloc(width * num_channels); 115 | if(!line) { 116 | ESP_LOGE(TAG, "Scan line malloc failed"); 117 | return false; 118 | } 119 | 120 | for (int i = 0; i < height; i++) { 121 | convert_line_format(src, format, line, width, num_channels, i); 122 | if (!dst_image.process_scanline(line)) { 123 | ESP_LOGE(TAG, "JPG process line %u failed", i); 124 | free(line); 125 | return false; 126 | } 127 | } 128 | free(line); 129 | 130 | if (!dst_image.process_scanline(NULL)) { 131 | ESP_LOGE(TAG, "JPG image finish failed"); 132 | return false; 133 | } 134 | dst_image.deinit(); 135 | return true; 136 | } 137 | 138 | class callback_stream : public jpge::output_stream { 139 | protected: 140 | jpg_out_cb ocb; 141 | void * oarg; 142 | size_t index; 143 | 144 | public: 145 | callback_stream(jpg_out_cb cb, void * arg) : ocb(cb), oarg(arg), index(0) { } 146 | virtual ~callback_stream() { } 147 | virtual bool put_buf(const void* data, int len) 148 | { 149 | index += ocb(oarg, index, data, len); 150 | return true; 151 | } 152 | virtual size_t get_size() const 153 | { 154 | return index; 155 | } 156 | }; 157 | 158 | bool fmt2jpg_cb(uint8_t *src, size_t src_len, uint16_t width, uint16_t height, pixformat_t format, uint8_t quality, jpg_out_cb cb, void * arg) 159 | { 160 | callback_stream dst_stream(cb, arg); 161 | return convert_image(src, width, height, format, quality, &dst_stream); 162 | } 163 | 164 | bool frame2jpg_cb(camera_fb_t * fb, uint8_t quality, jpg_out_cb cb, void * arg) 165 | { 166 | return fmt2jpg_cb(fb->buf, fb->len, fb->width, fb->height, fb->format, quality, cb, arg); 167 | } 168 | 169 | 170 | 171 | class memory_stream : public jpge::output_stream { 172 | protected: 173 | uint8_t *out_buf; 174 | size_t max_len, index; 175 | 176 | public: 177 | memory_stream(void *pBuf, uint buf_size) : out_buf(static_cast(pBuf)), max_len(buf_size), index(0) { } 178 | 179 | virtual ~memory_stream() { } 180 | 181 | virtual bool put_buf(const void* pBuf, int len) 182 | { 183 | if (!pBuf) { 184 | //end of image 185 | return true; 186 | } 187 | if ((size_t)len > (max_len - index)) { 188 | ESP_LOGW(TAG, "JPG output overflow: %d bytes", len - (max_len - index)); 189 | len = max_len - index; 190 | } 191 | if (len) { 192 | memcpy(out_buf + index, pBuf, len); 193 | index += len; 194 | } 195 | return true; 196 | } 197 | 198 | virtual size_t get_size() const 199 | { 200 | return index; 201 | } 202 | }; 203 | 204 | bool fmt2jpg(uint8_t *src, size_t src_len, uint16_t width, uint16_t height, pixformat_t format, uint8_t quality, uint8_t ** out, size_t * out_len) 205 | { 206 | //todo: allocate proper buffer for holding JPEG data 207 | //this should be enough for CIF frame size 208 | int jpg_buf_len = 24*1024; 209 | 210 | 211 | uint8_t * jpg_buf = (uint8_t *)_malloc(jpg_buf_len); 212 | if(jpg_buf == NULL) { 213 | ESP_LOGE(TAG, "JPG buffer malloc failed"); 214 | return false; 215 | } 216 | memory_stream dst_stream(jpg_buf, jpg_buf_len); 217 | 218 | if(!convert_image(src, width, height, format, quality, &dst_stream)) { 219 | free(jpg_buf); 220 | return false; 221 | } 222 | 223 | *out = jpg_buf; 224 | *out_len = dst_stream.get_size(); 225 | return true; 226 | } 227 | 228 | bool frame2jpg(camera_fb_t * fb, uint8_t quality, uint8_t ** out, size_t * out_len) 229 | { 230 | return fmt2jpg(fb->buf, fb->len, fb->width, fb->height, fb->format, quality, out, out_len); 231 | } 232 | -------------------------------------------------------------------------------- /esp32-camera-firmware/lib/img_converters/yuv.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef _CONVERSIONS_YUV_H_ 15 | #define _CONVERSIONS_YUV_H_ 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | #include 22 | 23 | void yuv2rgb(uint8_t y, uint8_t u, uint8_t v, uint8_t *r, uint8_t *g, uint8_t *b); 24 | 25 | #ifdef __cplusplus 26 | } 27 | #endif 28 | 29 | #endif /* _CONVERSIONS_YUV_H_ */ 30 | -------------------------------------------------------------------------------- /esp32-camera-firmware/platformio.ini: -------------------------------------------------------------------------------- 1 | ; PlatformIO Project Configuration File 2 | ; 3 | ; Build options: build flags, source filter 4 | ; Upload options: custom upload port, speed and extra flags 5 | ; Library options: dependencies, extra library storages 6 | ; Advanced options: extra scripting 7 | ; 8 | ; Please visit documentation for the other options and examples 9 | ; https://docs.platformio.org/page/projectconf.html 10 | 11 | [platformio] 12 | env_default = esp32cam 13 | 14 | [common] 15 | monitor_speed = 115200 16 | platform = espressif32 17 | board = esp32dev 18 | framework = espidf 19 | 20 | [env:esp32cam] 21 | build_flags = -DCONFIG_MODEL=2 22 | platform = ${common.platform} 23 | board = ${common.board} 24 | framework = ${common.framework} 25 | monitor_speed = ${common.monitor_speed} 26 | 27 | [env:m5cam] 28 | build_flags = -DCONFIG_MODEL=1 29 | platform = ${common.platform} 30 | board = ${common.board} 31 | framework = ${common.framework} 32 | monitor_speed = ${common.monitor_speed} -------------------------------------------------------------------------------- /esp32-camera-firmware/src/Kconfig.projbuild: -------------------------------------------------------------------------------- 1 | menu "ESP32 Camera Demo Configuration" 2 | 3 | config WIFI_SSID 4 | string "WiFi SSID" 5 | default "" 6 | help 7 | SSID (network name) for the demo to connect to. 8 | 9 | config WIFI_PASSWORD 10 | string "WiFi Password" 11 | default "" 12 | help 13 | Password for your network. 14 | 15 | config XCLK_FREQ 16 | int "XCLK Frequency" 17 | default "20000000" 18 | help 19 | The XCLK Frequency in Herz. 20 | 21 | 22 | menu "Pin Configuration" 23 | config D0 24 | int "D0" 25 | default "5" 26 | config D1 27 | int "D1" 28 | default "18" 29 | config D2 30 | int "D2" 31 | default "19" 32 | config D3 33 | int "D3" 34 | default "21" 35 | config D4 36 | int "D4" 37 | default "36" 38 | config D5 39 | int "D5" 40 | default "39" 41 | config D6 42 | int "D6" 43 | default "34" 44 | config D7 45 | int "D7" 46 | default "35" 47 | config XCLK 48 | int "XCLK" 49 | default "0" 50 | config PCLK 51 | int "PCLK" 52 | default "22" 53 | config VSYNC 54 | int "VSYNC" 55 | default "25" 56 | config HREF 57 | int "HREF" 58 | default "23" 59 | config SDA 60 | int "SDA" 61 | default "26" 62 | config SCL 63 | int "SCL" 64 | default "27" 65 | config RESET 66 | int "RESET" 67 | default "32" 68 | endmenu 69 | 70 | config QR_RECOGNIZE 71 | bool "QR recognize Support" 72 | default y 73 | help 74 | Enable this option if you want to recognize qr. 75 | endmenu 76 | 77 | -------------------------------------------------------------------------------- /esp32-camera-firmware/src/app_main.c: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2017 Espressif Systems (Shanghai) PTE LTD 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | 19 | #include "freertos/FreeRTOS.h" 20 | #include "freertos/task.h" 21 | #include "freertos/semphr.h" 22 | #include "freertos/event_groups.h" 23 | 24 | #include "esp_camera.h" 25 | #include "esp_system.h" 26 | #include "esp_wifi.h" 27 | #include "mdns.h" 28 | #include "esp_event_loop.h" 29 | #include "esp_log.h" 30 | #include "esp_err.h" 31 | #include "esp_http_server.h" 32 | #include "nvs_flash.h" 33 | #include "driver/gpio.h" 34 | 35 | static esp_err_t jpg_http_handler(httpd_req_t *req); 36 | static esp_err_t event_handler(void *ctx, system_event_t *event); 37 | static void initialise_wifi(void); 38 | static void start_mdns_service(void); 39 | 40 | static const char *TAG = "camera_demo"; 41 | 42 | EventGroupHandle_t s_wifi_event_group; 43 | static const int CONNECTED_BIT = BIT0; 44 | static ip4_addr_t s_ip_addr; 45 | 46 | #define CAMERA_PIXEL_FORMAT CAMERA_PF_JPEG 47 | #if CONFIG_MODEL == CONFIG_MODEL_ESP32CAM 48 | // #define CAMERA_FRAME_SIZE FRAMESIZE_UXGA // Full 2MP 49 | #define CAMERA_FRAME_SIZE FRAMESIZE_SVGA // No PSRAM 50 | #else 51 | #define CAMERA_FRAME_SIZE FRAMESIZE_SVGA // No PSRAM 52 | #endif 53 | 54 | void app_main() 55 | { 56 | 57 | esp_log_level_set("wifi", ESP_LOG_WARN); 58 | esp_log_level_set("gpio", ESP_LOG_WARN); 59 | esp_err_t err = nvs_flash_init(); 60 | if (err != ESP_OK) 61 | { 62 | ESP_ERROR_CHECK(nvs_flash_erase()); 63 | ESP_ERROR_CHECK(nvs_flash_init()); 64 | } 65 | 66 | camera_config_t camera_config = { 67 | .pin_reset = CONFIG_RESET, 68 | .pin_pwdn = CONFIG_PWDN, 69 | .pin_xclk = CONFIG_XCLK, 70 | .pin_sscb_sda = CONFIG_SDA, 71 | .pin_sscb_scl = CONFIG_SCL, 72 | 73 | .pin_d0 = CONFIG_D0, 74 | .pin_d1 = CONFIG_D1, 75 | .pin_d2 = CONFIG_D2, 76 | .pin_d3 = CONFIG_D3, 77 | .pin_d4 = CONFIG_D4, 78 | .pin_d5 = CONFIG_D5, 79 | .pin_d6 = CONFIG_D6, 80 | .pin_d7 = CONFIG_D7, 81 | .pin_vsync = CONFIG_VSYNC, 82 | .pin_href = CONFIG_HREF, 83 | .pin_pclk = CONFIG_PCLK, 84 | 85 | .xclk_freq_hz = CONFIG_XCLK_FREQ, 86 | .ledc_channel = LEDC_CHANNEL_0, 87 | .ledc_timer = LEDC_TIMER_0, 88 | 89 | .pixel_format = PIXFORMAT_JPEG, 90 | .frame_size = CAMERA_FRAME_SIZE, 91 | .jpeg_quality = 12, 92 | .fb_count = 1}; 93 | 94 | err = esp_camera_init(&camera_config); 95 | if (err != ESP_OK) 96 | { 97 | ESP_LOGE(TAG, "Camera init failed with error 0x%x", err); 98 | return; 99 | } 100 | // databuf = (char *) malloc(BUF_SIZE); 101 | initialise_wifi(); 102 | 103 | start_mdns_service(); 104 | 105 | httpd_handle_t server = NULL; 106 | httpd_config_t config = HTTPD_DEFAULT_CONFIG(); 107 | ESP_ERROR_CHECK(httpd_start(&server, &config)); 108 | 109 | httpd_uri_t jpg_uri = { 110 | .uri = "/jpg", 111 | .method = HTTP_GET, 112 | .handler = jpg_http_handler, 113 | .user_ctx = NULL}; 114 | 115 | ESP_ERROR_CHECK(httpd_register_uri_handler(server, &jpg_uri)); 116 | ESP_LOGI(TAG, "Open http://" IPSTR "/jpg for single image/jpg image", IP2STR(&s_ip_addr)); 117 | ESP_LOGI(TAG, "Free heap: %u", xPortGetFreeHeapSize()); 118 | ESP_LOGI(TAG, "Camera demo ready"); 119 | } 120 | 121 | typedef struct 122 | { 123 | httpd_req_t *req; 124 | size_t len; 125 | } jpg_chunking_t; 126 | 127 | static size_t jpg_encode_stream(void *arg, size_t index, const void *data, size_t len) 128 | { 129 | jpg_chunking_t *j = (jpg_chunking_t *)arg; 130 | if (!index) 131 | { 132 | j->len = 0; 133 | } 134 | if (httpd_resp_send_chunk(j->req, (const char *)data, len) != ESP_OK) 135 | { 136 | return 0; 137 | } 138 | j->len += len; 139 | return len; 140 | } 141 | 142 | esp_err_t jpg_http_handler(httpd_req_t *req) 143 | { 144 | camera_fb_t *fb = NULL; 145 | esp_err_t res = ESP_OK; 146 | size_t fb_len = 0; 147 | int64_t fr_start = esp_timer_get_time(); 148 | 149 | fb = esp_camera_fb_get(); 150 | if (!fb) 151 | { 152 | ESP_LOGE(TAG, "Camera capture failed"); 153 | httpd_resp_send_500(req); 154 | return ESP_FAIL; 155 | } 156 | res = httpd_resp_set_type(req, "image/jpeg"); 157 | if (res == ESP_OK) 158 | { 159 | res = httpd_resp_set_hdr(req, "Content-Disposition", "inline; filename=capture.jpg"); 160 | } 161 | 162 | if (res == ESP_OK) 163 | { 164 | if (fb->format == PIXFORMAT_JPEG) 165 | { 166 | fb_len = fb->len; 167 | res = httpd_resp_send(req, (const char *)fb->buf, fb->len); 168 | } 169 | else 170 | { 171 | jpg_chunking_t jchunk = {req, 0}; 172 | res = frame2jpg_cb(fb, 80, jpg_encode_stream, &jchunk) ? ESP_OK : ESP_FAIL; 173 | httpd_resp_send_chunk(req, NULL, 0); 174 | fb_len = jchunk.len; 175 | } 176 | } 177 | esp_camera_fb_return(fb); 178 | int64_t fr_end = esp_timer_get_time(); 179 | ESP_LOGI(TAG, "JPG: %uKB %ums", (uint32_t)(fb_len / 1024), (uint32_t)((fr_end - fr_start) / 1000)); 180 | return res; 181 | } 182 | 183 | static esp_err_t event_handler(void *ctx, system_event_t *event) 184 | { 185 | switch (event->event_id) 186 | { 187 | case SYSTEM_EVENT_STA_START: 188 | esp_wifi_connect(); 189 | break; 190 | case SYSTEM_EVENT_STA_GOT_IP: 191 | xEventGroupSetBits(s_wifi_event_group, CONNECTED_BIT); 192 | s_ip_addr = event->event_info.got_ip.ip_info.ip; 193 | break; 194 | case SYSTEM_EVENT_STA_DISCONNECTED: 195 | esp_wifi_connect(); 196 | xEventGroupClearBits(s_wifi_event_group, CONNECTED_BIT); 197 | break; 198 | default: 199 | break; 200 | } 201 | return ESP_OK; 202 | } 203 | 204 | void start_mdns_service() 205 | { 206 | //initialize mDNS service 207 | esp_err_t err = mdns_init(); 208 | if (err) 209 | { 210 | printf("MDNS Init failed: %d\n", err); 211 | return; 212 | } 213 | 214 | uint64_t chipId = 0LL; 215 | esp_efuse_mac_get_default((uint8_t *)(&chipId)); 216 | char id[4]; 217 | sprintf(id, "%04x", (uint16_t)(chipId >> 32)); 218 | 219 | char name[18] = "indoor-camera-"; 220 | strcat(name, id); 221 | 222 | mdns_hostname_set(name); 223 | mdns_service_add(NULL, "_camera", "_tcp", 80, NULL, 0); 224 | mdns_service_instance_name_set("_camera", "_tcp", name); 225 | 226 | ESP_LOGI(TAG, "MDNS domain: %s", name); 227 | } 228 | 229 | static void initialise_wifi(void) 230 | { 231 | tcpip_adapter_init(); 232 | s_wifi_event_group = xEventGroupCreate(); 233 | ESP_ERROR_CHECK(esp_event_loop_init(event_handler, NULL)); 234 | wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); 235 | ESP_ERROR_CHECK(esp_wifi_init(&cfg)); 236 | // ESP_ERROR_CHECK( esp_wifi_set_storage(WIFI_STORAGE_RAM) ); 237 | wifi_config_t wifi_config = { 238 | .sta = { 239 | .ssid = CONFIG_WIFI_SSID, 240 | .password = CONFIG_WIFI_PASSWORD, 241 | }, 242 | }; 243 | ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA)); 244 | ESP_ERROR_CHECK(esp_wifi_set_config(WIFI_IF_STA, &wifi_config)); 245 | ESP_ERROR_CHECK(esp_wifi_start()); 246 | ESP_ERROR_CHECK(esp_wifi_set_ps(WIFI_PS_NONE)); 247 | ESP_LOGI(TAG, "Connecting to \"%s\"", wifi_config.sta.ssid); 248 | xEventGroupWaitBits(s_wifi_event_group, CONNECTED_BIT, false, true, portMAX_DELAY); 249 | ESP_LOGI(TAG, "Connected"); 250 | } 251 | -------------------------------------------------------------------------------- /esp32-camera-firmware/src/component.mk: -------------------------------------------------------------------------------- 1 | # 2 | # Main Makefile. This is basically the same as a component makefile. 3 | # 4 | # This Makefile should, at the very least, just include $(SDK_PATH)/make/component.mk. By default, 5 | # this will take the sources in the src/ directory, compile them and link them into 6 | # lib(subdirectory_name).a in the build directory. This behaviour is entirely configurable, 7 | # please read the SDK documents if you need to do this. 8 | # 9 | 10 | -------------------------------------------------------------------------------- /esp32-camera-firmware/test/README: -------------------------------------------------------------------------------- 1 | 2 | This directory is intended for PIO Unit Testing and project tests. 3 | 4 | Unit Testing is a software testing method by which individual units of 5 | source code, sets of one or more MCU program modules together with associated 6 | control data, usage procedures, and operating procedures, are tested to 7 | determine whether they are fit for use. Unit testing finds problems early 8 | in the development cycle. 9 | 10 | More information about PIO Unit Testing: 11 | - https://docs.platformio.org/page/plus/unit-testing.html 12 | -------------------------------------------------------------------------------- /firebase.json: -------------------------------------------------------------------------------- 1 | { 2 | "database": { 3 | "rules": "database.rules.json" 4 | }, 5 | "functions": { 6 | "predeploy": [ 7 | "npm --prefix \"$RESOURCE_DIR\" run lint" 8 | ] 9 | }, 10 | "hosting": { 11 | "public": "public", 12 | "ignore": [ 13 | "firebase.json", 14 | "**/.*", 15 | "**/node_modules/**" 16 | ], 17 | "rewrites": [ 18 | { 19 | "source": "**", 20 | "destination": "/index.html" 21 | } 22 | ] 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /functions/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "parserOptions": { 3 | // Required for certain syntax usages 4 | "ecmaVersion": 6 5 | }, 6 | "plugins": [ 7 | "promise" 8 | ], 9 | "extends": "eslint:recommended", 10 | "rules": { 11 | // Removed rule "disallow the use of console" from recommended eslint rules 12 | "no-console": "off", 13 | 14 | // Removed rule "disallow multiple spaces in regular expressions" from recommended eslint rules 15 | "no-regex-spaces": "off", 16 | 17 | // Removed rule "disallow the use of debugger" from recommended eslint rules 18 | "no-debugger": "off", 19 | 20 | // Removed rule "disallow unused variables" from recommended eslint rules 21 | "no-unused-vars": "off", 22 | 23 | // Removed rule "disallow mixed spaces and tabs for indentation" from recommended eslint rules 24 | "no-mixed-spaces-and-tabs": "off", 25 | 26 | // Removed rule "disallow the use of undeclared variables unless mentioned in /*global */ comments" from recommended eslint rules 27 | "no-undef": "off", 28 | 29 | // Warn against template literal placeholder syntax in regular strings 30 | "no-template-curly-in-string": 1, 31 | 32 | // Warn if return statements do not either always or never specify values 33 | "consistent-return": 1, 34 | 35 | // Warn if no return statements in callbacks of array methods 36 | "array-callback-return": 1, 37 | 38 | // Require the use of === and !== 39 | "eqeqeq": 2, 40 | 41 | // Disallow the use of alert, confirm, and prompt 42 | "no-alert": 2, 43 | 44 | // Disallow the use of arguments.caller or arguments.callee 45 | "no-caller": 2, 46 | 47 | // Disallow null comparisons without type-checking operators 48 | "no-eq-null": 2, 49 | 50 | // Disallow the use of eval() 51 | "no-eval": 2, 52 | 53 | // Warn against extending native types 54 | "no-extend-native": 1, 55 | 56 | // Warn against unnecessary calls to .bind() 57 | "no-extra-bind": 1, 58 | 59 | // Warn against unnecessary labels 60 | "no-extra-label": 1, 61 | 62 | // Disallow leading or trailing decimal points in numeric literals 63 | "no-floating-decimal": 2, 64 | 65 | // Warn against shorthand type conversions 66 | "no-implicit-coercion": 1, 67 | 68 | // Warn against function declarations and expressions inside loop statements 69 | "no-loop-func": 1, 70 | 71 | // Disallow new operators with the Function object 72 | "no-new-func": 2, 73 | 74 | // Warn against new operators with the String, Number, and Boolean objects 75 | "no-new-wrappers": 1, 76 | 77 | // Disallow throwing literals as exceptions 78 | "no-throw-literal": 2, 79 | 80 | // Require using Error objects as Promise rejection reasons 81 | "prefer-promise-reject-errors": 2, 82 | 83 | // Enforce “for” loop update clause moving the counter in the right direction 84 | "for-direction": 2, 85 | 86 | // Enforce return statements in getters 87 | "getter-return": 2, 88 | 89 | // Disallow await inside of loops 90 | "no-await-in-loop": 2, 91 | 92 | // Disallow comparing against -0 93 | "no-compare-neg-zero": 2, 94 | 95 | // Warn against catch clause parameters from shadowing variables in the outer scope 96 | "no-catch-shadow": 1, 97 | 98 | // Disallow identifiers from shadowing restricted names 99 | "no-shadow-restricted-names": 2, 100 | 101 | // Enforce return statements in callbacks of array methods 102 | "callback-return": 2, 103 | 104 | // Require error handling in callbacks 105 | "handle-callback-err": 2, 106 | 107 | // Warn against string concatenation with __dirname and __filename 108 | "no-path-concat": 1, 109 | 110 | // Prefer using arrow functions for callbacks 111 | "prefer-arrow-callback": 1, 112 | 113 | // Return inside each then() to create readable and reusable Promise chains. 114 | // Forces developers to return console logs and http calls in promises. 115 | "promise/always-return": 2, 116 | 117 | //Enforces the use of catch() on un-returned promises 118 | "promise/catch-or-return": 2, 119 | 120 | // Warn against nested then() or catch() statements 121 | "promise/no-nesting": 1 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /functions/index.js: -------------------------------------------------------------------------------- 1 | const functions = require('firebase-functions') 2 | const admin = require('firebase-admin') 3 | 4 | admin.initializeApp() 5 | 6 | const db = admin.database() 7 | 8 | exports.processTelemetry = functions 9 | .pubsub 10 | .topic('telemetry') 11 | .onPublish( handleMessage('data') ) 12 | 13 | exports.processState = functions 14 | .pubsub 15 | .topic('state') 16 | .onPublish( handleMessage('meta') ) 17 | 18 | function handleMessage( topic ) { 19 | return ( message, context ) => { 20 | const attributes = message.attributes 21 | const payload = message.json 22 | 23 | const deviceId = attributes['deviceId'] 24 | 25 | const data = Object.assign({}, payload, { 26 | updated: context.timestamp 27 | }) 28 | 29 | return db.ref(`/devices/${deviceId}/${topic}`).update(data) 30 | } 31 | } -------------------------------------------------------------------------------- /functions/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "functions", 3 | "description": "Cloud Functions for Firebase", 4 | "scripts": { 5 | "lint": "eslint .", 6 | "serve": "firebase serve --only functions", 7 | "shell": "firebase functions:shell", 8 | "start": "npm run shell", 9 | "deploy": "firebase deploy --only functions", 10 | "logs": "firebase functions:log" 11 | }, 12 | "engines": { 13 | "node": "8" 14 | }, 15 | "dependencies": { 16 | "firebase-admin": "~6.0.0", 17 | "firebase-functions": "^2.0.3" 18 | }, 19 | "devDependencies": { 20 | "eslint": "^4.12.0", 21 | "eslint-plugin-promise": "^3.6.0" 22 | }, 23 | "private": true 24 | } 25 | -------------------------------------------------------------------------------- /generate_key_pair.sh: -------------------------------------------------------------------------------- 1 | openssl ecparam -genkey -name prime256v1 -noout -out ec_private.pem 2 | openssl ec -in ec_private.pem -pubout -out ec_public.pem 3 | openssl ec -in ec_private.pem -noout -text -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 61 | IoT Cloud Monitor 62 | 63 | 64 |
65 | 66 | 67 | IoT Edge - Cloud Monitor 68 | Create Room 69 | 70 | 71 | 76 | 77 |
78 | {{room.name}} 79 | 80 | No devices assgined. 81 | 82 |
83 | 84 | 85 | 86 | {{device.icon}} 87 | 88 |
{{device.name}}
89 |
90 | [{{device.meta.status}}] {{device.updated}} 91 |
92 | 93 | 94 | 95 | 96 | {{c}} ({{device.classes[c]}}) 97 | 98 | 99 | Nothing found. 100 | 101 | 102 | 103 | 104 | 105 | {{room.name}} 106 | 107 | 108 | Remove from room 109 | 110 | 111 | 112 | 113 | 114 | 115 | Open local monitor 116 | open_in_new 117 | 118 | 119 | 120 |
121 |
122 | 130 | 131 | 132 | 133 |
134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 235 | 236 | 237 | -------------------------------------------------------------------------------- /register_device.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . set_env_vars.sh $1 4 | 5 | DEVICE=$2 6 | 7 | # Create device 8 | gcloud iot devices create $DEVICE \ 9 | --region=$LOCATION \ 10 | --registry=$REGISTRY 11 | 12 | # Bind device to gateway 13 | gcloud beta iot devices gateways "bind" \ 14 | --gateway=$GATEWAY_ID \ 15 | --gateway-region=$LOCATION \ 16 | --gateway-registry=$REGISTRY \ 17 | --device=$DEVICE \ 18 | --device-region=$LOCATION \ 19 | --device-registry=$REGISTRY -------------------------------------------------------------------------------- /set_env_vars.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LOCATION="us-central1" 4 | export GATEWAY_ID="gw-mark-one" 5 | export REGISTRY="iot-edge-registry" 6 | export PUBSUB_TOPIC_EVENTS="telemetry" 7 | export PUBSUB_TOPIC_STATE="state" 8 | 9 | if [ -z "$GCLOUD_PROJECT" ]; then 10 | printf "\nOnce you've logged into your project, enter the project id below." 11 | GCLOUD_PROJECT="$1" 12 | if [ -z "$1" ]; then 13 | echo 14 | read -p 'Please enter project id: ' GCLOUD_PROJECT 15 | echo 16 | fi 17 | if [ -z "$GCLOUD_PROJECT" ]; then 18 | cat<