├── __pycache__
├── al_Alerts.cpython-35.pyc
├── camera_pi.cpython-35.pyc
├── eml_Email.cpython-35.pyc
├── image_draw.cpython-35.pyc
├── base_camera.cpython-35.pyc
└── ms_cognitive_imagerec.cpython-35.pyc
├── templates.yaml
├── templates
├── index.html
├── who_see.html
└── what_see.html
├── camera.py
├── camera_opencv.py
├── camera_pi.py
├── README.md
├── eml_Email.py
├── st_Camera.py
├── al_Alerts.py
├── image_draw.py
├── base_camera.py
├── ms_cognitive_imagerec.py
└── st_main.py
/__pycache__/al_Alerts.cpython-35.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/walkken/SeeTalker/HEAD/__pycache__/al_Alerts.cpython-35.pyc
--------------------------------------------------------------------------------
/__pycache__/camera_pi.cpython-35.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/walkken/SeeTalker/HEAD/__pycache__/camera_pi.cpython-35.pyc
--------------------------------------------------------------------------------
/__pycache__/eml_Email.cpython-35.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/walkken/SeeTalker/HEAD/__pycache__/eml_Email.cpython-35.pyc
--------------------------------------------------------------------------------
/__pycache__/image_draw.cpython-35.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/walkken/SeeTalker/HEAD/__pycache__/image_draw.cpython-35.pyc
--------------------------------------------------------------------------------
/__pycache__/base_camera.cpython-35.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/walkken/SeeTalker/HEAD/__pycache__/base_camera.cpython-35.pyc
--------------------------------------------------------------------------------
/__pycache__/ms_cognitive_imagerec.cpython-35.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/walkken/SeeTalker/HEAD/__pycache__/ms_cognitive_imagerec.cpython-35.pyc
--------------------------------------------------------------------------------
/templates.yaml:
--------------------------------------------------------------------------------
1 | welcome: Welcome to Ken's Raspberry Pi
2 | round: Can you repeat the numbers {{ numbers|join(", ") }} backwards?
3 |
4 | win: Good job!
5 |
6 | lose: Sorry, that's the wrong answer.
--------------------------------------------------------------------------------
/templates/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Video Streaming
4 |
5 |
6 | Live Video Streaming
7 |  }})
8 |
9 |
--------------------------------------------------------------------------------
/templates/who_see.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Who Do I See?
4 |
5 |
6 | {{who_see}}
7 |
8 |
9 | {% if who_see_image != None %}
10 | 
11 | {% endif %}
12 |
13 |
--------------------------------------------------------------------------------
/templates/what_see.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | What do I See?
4 |
5 |
6 | {{what_see}}!
7 |
8 |
9 | {% if what_see_image != None %}
10 | 
11 | {% endif %}
12 |
13 |
--------------------------------------------------------------------------------
/camera.py:
--------------------------------------------------------------------------------
1 | import time
2 | from base_camera import BaseCamera
3 |
4 |
5 | class Camera(BaseCamera):
6 | """An emulated camera implementation that streams a repeated sequence of
7 | files 1.jpg, 2.jpg and 3.jpg at a rate of one frame per second."""
8 | imgs = [open(f + '.jpg', 'rb').read() for f in ['1', '2', '3']]
9 |
10 | @staticmethod
11 | def frames():
12 | while True:
13 | time.sleep(1)
14 | yield Camera.imgs[int(time.time()) % 3]
--------------------------------------------------------------------------------
/camera_opencv.py:
--------------------------------------------------------------------------------
1 | import cv2
2 | from base_camera import BaseCamera
3 |
4 |
5 | class Camera(BaseCamera):
6 | video_source = 0
7 |
8 | @staticmethod
9 | def set_video_source(source):
10 | Camera.video_source = source
11 |
12 | @staticmethod
13 | def frames():
14 | camera = cv2.VideoCapture(Camera.video_source)
15 | if not camera.isOpened():
16 | raise RuntimeError('Could not start camera.')
17 |
18 | while True:
19 | # read current frame
20 | _, img = camera.read()
21 |
22 | # encode as a jpeg image and return it
23 | yield cv2.imencode('.jpg', img)[1].tobytes()
--------------------------------------------------------------------------------
/camera_pi.py:
--------------------------------------------------------------------------------
1 | import io
2 | import time
3 | import picamera
4 | from base_camera import BaseCamera
5 |
6 | class Camera(BaseCamera):
7 | @staticmethod
8 | def frames():
9 | with picamera.PiCamera() as camera:
10 | # let camera warm up
11 | time.sleep(2)
12 |
13 | camera.resolution = (800, 640)
14 | camera.rotation = 270
15 |
16 |
17 | stream = io.BytesIO()
18 | for _ in camera.capture_continuous(stream, 'jpeg',
19 | use_video_port=True):
20 | # return current frame
21 | stream.seek(0)
22 | yield stream.read()
23 |
24 | # reset stream for next frame
25 | stream.seek(0)
26 | stream.truncate()
27 |
28 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # SeeTalker
2 | SeeTalker gives Amazon's Alexa the ability to tell you what it sees. SeeTalker is triggered by Alexa requests to describe what it sees or take a selfie. SeeTalker calls Microsoft's Face API and Vision Services API to interpret video images, returning a natural language response using Alexa.
3 |
4 | SeeTalker was developed by the original developer to explore image recognition and Alexa skill development. The develper also learned Python along the way and how to utlize a Raspberry Pi as a web server and IoT device.
5 |
6 | Key learning opportunities from SeeTalker:
7 | 1. Alexa skill development
8 | 2. Utilize Raspberry Pi as a web server
9 | 3. Microsoft Cognitive Services: Face API, Vision Services API
10 | 4. Alexa skills directives
11 | 5. Sending Email
12 | 6. Python Flask and Amazon Ask web services framework (Flask-Ask)
13 |
14 | An overview and code documentation will be maintained on project Hackster.IO site at:
15 | https://www.hackster.io/ken-walker/raspberry-pi-image-recognition-with-alexa-voice-6b7a01
16 |
--------------------------------------------------------------------------------
/eml_Email.py:
--------------------------------------------------------------------------------
1 | import os
2 | #!/usr/bin/env python
3 | import smtplib
4 | from email.mime.multipart import MIMEMultipart
5 | from email.mime.text import MIMEText
6 | from email.mime.base import MIMEBase
7 | from email import encoders
8 |
9 | # set for gmail
10 | smtp_server = "smtp.gmail.com"
11 |
12 | def eml_SendEmail(fromAddr, toAddr, email_pwd, subject, body, attachFile=None):
13 |
14 | msg = MIMEMultipart()
15 | msg["From"] = fromAddr
16 | msg["To"] = toAddr
17 | msg["Subject"] = subject
18 | msg.attach(MIMEText(body, "plain"))
19 |
20 | if attachFile != None:
21 | attachment = open(attachFile, 'rb')
22 |
23 | part = MIMEBase("application", "octet-stream")
24 | part.set_payload((attachment).read())
25 | encoders.encode_base64(part)
26 | part.add_header('Content-Disposition', 'attachment', filename=os.path.basename(attachFile))
27 | msg.attach(part)
28 |
29 | server = smtplib.SMTP(smtp_server, 587)
30 | server.starttls()
31 | server.login(fromAddr, email_pwd)
32 | text = msg.as_string()
33 |
34 | server.sendmail(fromAddr, toAddr, text)
35 | server.quit()
36 |
--------------------------------------------------------------------------------
/st_Camera.py:
--------------------------------------------------------------------------------
1 | import io
2 | import picamera
3 | import logging
4 |
5 | from threading import Condition
6 | from Thread import Timer
7 |
8 |
9 | class StreamingOutput(object):
10 | def __init__(self):
11 | self.frame = None
12 | self.buffer = io.BytesIO()
13 | self.condition = Condition()
14 |
15 | def write(self, buf):
16 | if buf.startswith(b'\xff\xd8'):
17 | # New frame, copy the existing buffer's content and notify all
18 | # clients it's available
19 | self.buffer.truncate()
20 | with self.condition:
21 | self.frame = self.buffer.getvalue()
22 | self.condition.notify_all()
23 | self.buffer.seek(0)
24 | return self.buffer.write(buf)
25 |
26 |
27 | def cam_StartLiveCamera ():
28 |
29 | camera = picamera.PiCamera(resolution='640x480', framerate=24)
30 |
31 | output = StreamingOutput()
32 | #Uncomment the next line to change your Pi's Camera rotation (in degrees)
33 | camera.rotation = 270
34 | print ("start recording")
35 | camera.start_recording(output, format='mjpeg')
36 |
37 | camera.rotation = 90
38 | camera.vflip = 1
39 |
40 | return(camera, output)
41 |
--------------------------------------------------------------------------------
/al_Alerts.py:
--------------------------------------------------------------------------------
1 | import threading
2 | import time
3 |
4 |
5 | def al_StartAlertTriggers (interval_secs, AlertHandler, fRepeat=True):
6 | if fRepeat: # repeat timer
7 | tmr = RepeatedTimer(interval_secs, AlertHandler)
8 | else: # one-time timer
9 | tmr = threading.Timer(interval_secs, AlertHandler)
10 | tmr.start() # start the timer
11 |
12 | return (tmr)
13 |
14 |
15 | def al_CancelAlertTriggers (tmr):
16 | #tmr.cancel()
17 | tmr.stop()
18 |
19 |
20 | def CheckAlertTriggers ():
21 | print ("Check Alert Triggers")
22 | return (True)
23 |
24 |
25 | class RepeatedTimer(object):
26 | def __init__(self, interval, function, *args, **kwargs):
27 | self._timer = None
28 | self.interval = interval
29 | self.function = function
30 | self.args = args
31 | self.kwargs = kwargs
32 | self.is_running = False
33 | self.next_call = time.time()
34 | self.start()
35 |
36 | def _run(self):
37 | self.is_running = False
38 | self.start()
39 | self.function(*self.args, **self.kwargs)
40 |
41 | def start(self):
42 | if not self.is_running:
43 | self.next_call += self.interval
44 | self._timer = threading.Timer(self.next_call - time.time(), self._run)
45 | self._timer.start()
46 | self.is_running = True
47 |
48 | def stop(self):
49 | self._timer.cancel()
50 | self.is_running = False
--------------------------------------------------------------------------------
/image_draw.py:
--------------------------------------------------------------------------------
1 | import requests
2 | from io import BytesIO
3 | from PIL import Image, ImageDraw, ImageFont
4 |
5 |
6 | #Convert width height to a point in a rectangle
7 | def getRectangle(faceDictionary):
8 | rect = faceDictionary['faceRectangle']
9 | left = rect['left']
10 | top = rect['top']
11 | bottom = top + rect['height']
12 | right = left + rect['width']
13 | return ((left, top), (right, bottom))
14 |
15 |
16 | def drawFaceRectangles(face, image, upperLeftText, upperRightText, lowerText1, lowerText2):
17 |
18 | if len(face) > 0 :
19 |
20 | #For each face returned use the face rectangle and draw a red box.
21 | draw = ImageDraw.Draw(image)
22 |
23 | #font = ImageFont.truetype('/usr/share/fonts/truetype/msttcorefonts/arial.ttf', 10)
24 | #size = font.getsize(showtext)
25 |
26 | (left, top), (right, bottom) = getRectangle(face)
27 | draw.rectangle(getRectangle(face), outline='red')
28 |
29 | font_size = 16
30 | font_name = "FreeSans.ttf"
31 | font_path = "/usr/share/fonts//truetype/freefont/%s"% font_name
32 | font = ImageFont.truetype(font_path, font_size)
33 |
34 | # UpperLeftText needs to be required
35 | (text_width, text_height) = font.getsize(upperLeftText)
36 |
37 | if len(upperLeftText) > 0:
38 | draw.text((left, top-text_height), upperLeftText, font=font)
39 | if len(upperRightText) > 0:
40 | draw.text((right-text_width, top-text_height), upperRightText, font=font)
41 | if len(lowerText1) > 0:
42 | draw.text((left, bottom), lowerText1, font=font)
43 | if len(lowerText2) > 0:
44 | draw.text((left, bottom+text_height), lowerText2, font=font)
45 |
46 | return (True)
47 | else:
48 | return (False)
49 |
--------------------------------------------------------------------------------
/base_camera.py:
--------------------------------------------------------------------------------
1 | import time
2 | import threading
3 | try:
4 | from greenlet import getcurrent as get_ident
5 | except ImportError:
6 | try:
7 | from thread import get_ident
8 | except ImportError:
9 | from _thread import get_ident
10 |
11 |
12 | class CameraEvent(object):
13 | """An Event-like class that signals all active clients when a new frame is
14 | available.
15 | """
16 | def __init__(self):
17 | self.events = {}
18 |
19 | def wait(self):
20 | """Invoked from each client's thread to wait for the next frame."""
21 | ident = get_ident()
22 | if ident not in self.events:
23 | # this is a new client
24 | # add an entry for it in the self.events dict
25 | # each entry has two elements, a threading.Event() and a timestamp
26 | self.events[ident] = [threading.Event(), time.time()]
27 | return self.events[ident][0].wait()
28 |
29 | def set(self):
30 | """Invoked by the camera thread when a new frame is available."""
31 | now = time.time()
32 | remove = None
33 | for ident, event in self.events.items():
34 | if not event[0].isSet():
35 | # if this client's event is not set, then set it
36 | # also update the last set timestamp to now
37 | event[0].set()
38 | event[1] = now
39 | else:
40 | # if the client's event is already set, it means the client
41 | # did not process a previous frame
42 | # if the event stays set for more than 5 seconds, then assume
43 | # the client is gone and remove it
44 | if now - event[1] > 5:
45 | remove = ident
46 | if remove:
47 | del self.events[remove]
48 |
49 | def clear(self):
50 | """Invoked from each client's thread after a frame was processed."""
51 | self.events[get_ident()][0].clear()
52 |
53 |
54 | class BaseCamera(object):
55 | thread = None # background thread that reads frames from camera
56 | frame = None # current frame is stored here by background thread
57 | last_access = 0 # time of last client access to the camera
58 | event = CameraEvent()
59 |
60 | def __init__(self):
61 | """Start the background camera thread if it isn't running yet."""
62 | if BaseCamera.thread is None:
63 | BaseCamera.last_access = time.time()
64 |
65 | # start background frame thread
66 | BaseCamera.thread = threading.Thread(target=self._thread)
67 | BaseCamera.thread.start()
68 |
69 | # wait until frames are available
70 | while self.get_frame() is None:
71 | time.sleep(0)
72 |
73 | def get_frame(self):
74 | """Return the current camera frame."""
75 | BaseCamera.last_access = time.time()
76 |
77 | # wait for a signal from the camera thread
78 | BaseCamera.event.wait()
79 | BaseCamera.event.clear()
80 |
81 | return BaseCamera.frame
82 |
83 | @staticmethod
84 | def frames():
85 | """"Generator that returns frames from the camera."""
86 | raise RuntimeError('Must be implemented by subclasses.')
87 |
88 |
89 | @classmethod
90 | def _thread(cls):
91 | """Camera background thread."""
92 | print('Starting camera thread.')
93 | frames_iterator = cls.frames()
94 | for frame in frames_iterator:
95 | BaseCamera.frame = frame
96 | BaseCamera.event.set() # send signal to clients
97 | time.sleep(0)
98 |
99 | # if there hasn't been any clients asking for frames in
100 | # the last 10 seconds then stop the thread
101 | #if time.time() - BaseCamera.last_access > 60:
102 | # frames_iterator.close()
103 | # print('Stopping camera thread due to inactivity.')
104 | #break
105 | BaseCamera.thread = None
--------------------------------------------------------------------------------
/ms_cognitive_imagerec.py:
--------------------------------------------------------------------------------
1 | import http.client, urllib, base64, json
2 | import requests
3 | from collections import namedtuple
4 | import operator
5 |
6 | # change this
7 | # the subscription and endpoints for Face API and Vision API
8 | # make sure endpoint is correct for your region, default is western us
9 | face_api_sub_key = 'your subscription key'
10 | face_api_endpoint = 'https://westus.api.cognitive.microsoft.com/face/v1.0/detect'
11 |
12 | vision_api_sub_key = 'your subscription key'
13 | vision_api_endpoint = 'westus.api.cognitive.microsoft.com'
14 |
15 |
16 | class FaceAttribs(object):
17 | """__init__() functions as the class constructor"""
18 | def __init__(self, age=None, gender=None, gender_noun=None, gender_possessive=None, glasses=None, glasses_txt=None, top_emotion=None, top_emotion_conf=None, profile_txt=None):
19 | self.age = age
20 | self.gender = gender
21 | self.gender_noun = gender_noun
22 | self.gender_possessive = gender_possessive
23 | self.glasses = glasses
24 | self.top_emotion = top_emotion
25 | self.top_emotion_conf = top_emotion_conf
26 | self.profile_txt = profile_txt
27 | self.glasses_txt = glasses_txt
28 |
29 |
30 | def ms_GetFaceAttribs (face):
31 |
32 | if len(face)> 0:
33 |
34 | faceAttribs = FaceAttribs()
35 | print(face["faceAttributes"]["age"])
36 |
37 | faceAttribs.age = face["faceAttributes"]["age"]
38 | faceAttribs.gender = face["faceAttributes"]["gender"]
39 | faceAttribs.glasses = face["faceAttributes"]["glasses"]
40 |
41 | if faceAttribs.gender == 'male' :
42 | faceAttribs.gender_noun = "He"
43 | faceAttribs.gender_possessive = "His"
44 | else:
45 | faceAttribs.gender_noun= "She"
46 | faceAttribs.gender_possessive = "Her"
47 |
48 | emotion = face["faceAttributes"]["emotion"]
49 | sort_emotion = sorted(emotion.items(), key=operator.itemgetter(1), reverse=True)
50 |
51 | if faceAttribs.glasses == 'NoGlasses':
52 | faceAttribs.glasses_txt = "No Glasses"
53 | else:
54 | faceAttribs.glasses_txt = "wearing %s"% (faceAttribs.glasses)
55 |
56 |
57 | faceAttribs.top_emotion = sort_emotion[0][0]
58 | faceAttribs.top_emotion_conf = sort_emotion[0][1] *100
59 | faceAttribs.profile_txt = "%s age %d"% (faceAttribs.gender, faceAttribs.age)
60 |
61 | return faceAttribs
62 | else:
63 | return False
64 |
65 | def ms_WhoDoYouSee (body):
66 |
67 | # Request headers.
68 | header = {
69 | 'Content-Type': 'application/octet-stream',
70 | 'Ocp-Apim-Subscription-Key': face_api_sub_key
71 | }
72 |
73 | # Request parameters.
74 | params = {
75 | 'returnFaceId': 'true',
76 | # 'returnFaceLandmarks': 'false',
77 | 'returnFaceAttributes': 'age,gender,smile,emotion,glasses',
78 | }
79 |
80 | try:
81 |
82 | api_url = face_api_endpoint
83 |
84 | response = requests.post(api_url, headers=header, data=body, params=params)
85 |
86 | #print (response.json())
87 | #response = conn.request('POST', uri_base + '/face/v1.0/detect', json=body, data=body, headers=headers, params=params)
88 | #data = response.read().decode('utf-8')
89 |
90 | print ('Response:')
91 | parsed = json.loads(response.text)
92 | print (json.dumps(parsed, sort_keys=True, indent=2))
93 |
94 | #faceID = parsed[0]["faceId"]
95 | # print("faceID = %s"% faceID)
96 |
97 | #height = parsed[0]["faceRectangle"]["height"]
98 | #print("height= %d"% height)
99 |
100 | #gender = parsed[0]["gender"]
101 | #print("gender = %s"% gender)
102 |
103 |
104 | except Exception as e:
105 | print('Error:')
106 | print(e)
107 |
108 | return(parsed)
109 |
110 |
111 | def ms_WhatDoYouSee (body):
112 |
113 | headers = {
114 | # Request headers.
115 | 'Content-Type': 'application/octet-stream',
116 | 'Ocp-Apim-Subscription-Key': vision_api_sub_key,
117 | }
118 |
119 | params = urllib.parse.urlencode({
120 | # Request parameters. All of them are optional.
121 | 'visualFeatures': 'Description, Faces',
122 | 'details': '',
123 | 'language': 'en',
124 | })
125 |
126 | # Execute the REST API call and get the response.
127 |
128 | try:
129 | conn = http.client.HTTPSConnection(vision_api_endpoint)
130 | conn.request("POST", "/vision/v1.0/analyze?%s" % params, body, headers)
131 | response = conn.getresponse()
132 | data = response.read().decode('utf-8')
133 |
134 | # 'data' contains the JSON data. The following formats the JSON data for display.
135 | parsed = json.loads(data)
136 | parsedText = "I see %s"% (parsed['description']['captions'][0]['text'])
137 | conn.close()
138 |
139 | except Exception as e:
140 | print('Error:')
141 | print(e) ########### Python 3.6 #############
142 | parsedText = e
143 |
144 | return parsedText
145 |
--------------------------------------------------------------------------------
/st_main.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | from importlib import import_module
3 | import os
4 | from flask import Flask, render_template, Response, send_file
5 |
6 | # Libraries created for this program by Ken
7 | from ms_cognitive_imagerec import ms_WhatDoYouSee, ms_WhoDoYouSee, ms_GetFaceAttribs, FaceAttribs
8 | # for Intel Neural Compute Stick
9 | #from ncs_image_classify import ncs_init, ncs_classify_image, ncs_close
10 | from al_Alerts import al_StartAlertTriggers, al_CancelAlertTriggers
11 | from image_draw import drawFaceRectangles
12 | from eml_Email import eml_SendEmail
13 | # other libraries
14 | import requests
15 | from io import BytesIO
16 | from PIL import Image, ImageDraw
17 | from time import sleep
18 |
19 | # Alexa Flask-Ask
20 | import json
21 | import logging
22 | from flask_ask import Ask, statement, question, context as ask_context, request as ask_request, session as ask_session
23 |
24 |
25 | import operator
26 | import base64
27 | import http.client
28 | import webbrowser
29 |
30 | class SelfieAlert:
31 | countdown_secs = 3
32 | countdown_interval = 1
33 | secs_rem = 2
34 | tmr = None
35 | requestId = None
36 | apiEndpoint = None
37 | apiAccessToken = None
38 | image = None
39 |
40 | # import camera driver
41 | ##if os.environ.get('CAMERA'):
42 | # Camera = import_module('camera_' + os.environ['CAMERA']).Camera
43 | #else:
44 | # from camera import Camera
45 |
46 | from camera_pi import Camera
47 | camera = Camera()
48 |
49 | image_file = 'image_file.png'
50 |
51 | # this sound is not guaranteed to be available. replace sound. use ffmpeg to translate to Alexa requirements
52 | camera_sound_url = 'https://videotalker.blob.core.windows.net/videoblob/camera_sound_conv.mp3'
53 |
54 | # Raspberry Pi camera module (requires picamera package)
55 | # from camera_pi import Camera
56 |
57 | app = Flask(__name__)
58 | ask = Ask(app, "/")
59 |
60 | log = logging.getLogger()
61 | log.addHandler(logging.StreamHandler())
62 | log.setLevel(logging.DEBUG)
63 | logging.getLogger("flask_ask").setLevel(logging.DEBUG)
64 |
65 | @ask.launch
66 | def alexa_launch():
67 |
68 | return question('See Talker Active, How can I help you?')
69 |
70 |
71 |
72 | @ask.intent('Selfie')
73 | def alexa_Selfie():
74 |
75 | # do selfie countdown, capture image and set time to email photo with WhatDoYouSee for email body
76 | selfie(True)
77 |
78 | response_txt = "You're selfie was taken and will be emailed"
79 |
80 | return statement(response_txt)
81 |
82 |
83 | @ask.intent('WhoDoYouSee')
84 | def alexa_WhoDoYouSee():
85 | response = "no response"
86 | response = who_see(fAlexa=True)
87 |
88 | return statement(response)
89 |
90 | @ask.intent('WhatDoYouSee')
91 | def alexa_whatsee():
92 |
93 | response = "no response"
94 | response = what_see(fAlexa=True)
95 | return statement(response)
96 |
97 |
98 | @ask.session_ended
99 | def session_ended():
100 |
101 | return "{}", 200
102 |
103 | @ask.on_session_started
104 | def new_session():
105 | log.info('new session started')
106 |
107 | @app.route('/')
108 | def index():
109 | """Video streaming home page."""
110 | print ("index page")
111 | return render_template('index.html')
112 |
113 |
114 | def gen(camera):
115 | """Video streaming generator function."""
116 | while True:
117 | frame = camera.get_frame()
118 | yield (b'--frame\r\n'
119 | b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
120 |
121 |
122 | @app.route('/video_feed')
123 | def video_feed():
124 | """Video streaming route. Put this in the src attribute of an img tag."""
125 | return Response(gen(Camera()),
126 | mimetype='multipart/x-mixed-replace; boundary=frame')
127 |
128 | @app.route('/what_see')
129 | def what_see(fAlexa=False):
130 |
131 | body = camera.get_frame()
132 |
133 | #body = open(image_file, 'rb').read()
134 |
135 | parsedText = ms_WhatDoYouSee (body)
136 |
137 | imageFile_png = base64.b64encode(body).decode('ascii')
138 |
139 |
140 | if fAlexa:
141 | return (parsedText)
142 | else:
143 | return render_template("what_see.html", what_see = parsedText, what_see_image = imageFile_png)
144 |
145 |
146 | @app.route('/get_image')
147 | def get_image():
148 | body = camera.get_frame()
149 |
150 | with open(image_file, 'wb') as f:
151 | f.write(body)
152 | f.close()
153 |
154 | return send_file(image_file, mimetype='image/jpg')
155 |
156 |
157 |
158 | @app.route('/who_see')
159 | def who_see(fAlexa=False):
160 |
161 | body = camera.get_frame()
162 |
163 | response = ms_WhoDoYouSee (body)
164 |
165 | n=0
166 | if len(response) > 0:
167 | for face in response:
168 | n += 1
169 | faceAttribs = ms_GetFaceAttribs (face)
170 | if n == 1:
171 | img = Image.open(BytesIO(body))
172 |
173 | upperLeftText = faceAttribs.profile_txt
174 | upperRightText = ""
175 |
176 | lowerText1 = "Emotion: %s %2.f%%"% (faceAttribs.top_emotion, faceAttribs.top_emotion_conf)
177 | lowerText2 = faceAttribs.glasses_txt
178 |
179 | drawFaceRectangles(face, img, upperLeftText, upperRightText, lowerText1, lowerText2)
180 |
181 | img.save(image_file, 'PNG')
182 |
183 | img_str = open(image_file, 'rb').read()
184 |
185 | if faceAttribs.glasses == 'NoGlasses':
186 | glasses_txt = ""
187 | else:
188 | glasses_txt = "%s is wearing %s"% (faceAttribs.gender_noun, faceAttribs.glasses)
189 |
190 | top_emotion_txt = "%s top emotion is %s at %2.f %% confidence"% (faceAttribs.gender_possessive, faceAttribs.top_emotion, faceAttribs.top_emotion_conf)
191 |
192 | iSeeText = "I see a %s age %d %s. %s. "% (faceAttribs.gender, faceAttribs.age, top_emotion_txt, glasses_txt)
193 | profile = "%s age %d"% (faceAttribs.gender, faceAttribs.age)
194 |
195 | if n==1:
196 | finalText = iSeeText
197 | else:
198 | finalText = finalText + iSeeText
199 |
200 | # show image on screen which can be saved
201 | img.show()
202 |
203 |
204 | if fAlexa:
205 | return(finalText)
206 | else:
207 | buffer = BytesIO()
208 | img.save(buffer, 'PNG')
209 | img_str = base64.b64encode(buffer.getvalue()).decode('ascii')
210 |
211 | return render_template("who_see.html", who_see = finalText, who_see_image = img_str)
212 |
213 |
214 | else:
215 | iSeeText = "No Face Detected"
216 | return (iSeeText)
217 |
218 | @app.route('/selfie')
219 | def selfie(fAlexa=False):
220 |
221 | print ("starting selfie")
222 |
223 | if fAlexa:
224 | SelfieAlert.requestId = ask_request.requestId
225 |
226 | SelfieAlert.apiEndpoint = ask_context.System.apiEndpoint
227 | SelfieAlert.apiAccessToken = 'Bearer ' + ask_context.System.apiAccessToken
228 |
229 | # count down for selfie
230 | for secs in range (SelfieAlert.countdown_secs,0,-1):
231 | selfie_txt = "Selfie will be taken in " if secs == SelfieAlert.countdown_secs else ""
232 | secs_txt = "second" if secs == 1 else "seconds"
233 | if secs != 0:
234 | response_txt = ("%s %d %s"% (selfie_txt, secs, secs_txt))
235 | PostDirective_SpeechText(SelfieAlert.requestId, SelfieAlert.apiAccessToken, response_txt)
236 | sleep(1)
237 |
238 | #' Smile!
239 | PostDirective_SpeechText(SelfieAlert.requestId, SelfieAlert.apiAccessToken, "Smile")
240 |
241 | # save frame, then aysychronously set timer to email photo later.so that Alexa timout doesn't occur
242 | SelfieAlert.image = camera.get_frame()
243 | # Make camera sound
244 | PostDirective_SpeechText(SelfieAlert.requestId, SelfieAlert.apiAccessToken, GetSound_SSML(camera_sound_url))
245 | sleep(1) # give the directive time to produce sound
246 |
247 | # take selfie in N seconds
248 | fRepeatTimer = False #want single countdown (2 secs), not repeated intervals
249 | timerSecs = 2
250 | SelfieAlert.tmr = al_StartAlertTriggers (timerSecs, SelfieAlert_EmailHandler, fRepeatTimer)
251 |
252 | return
253 | else:
254 | SelfieAlert.image = camera.get_frame()
255 | # take selfie in N seconds
256 | SelfieAlert.tmr = None
257 |
258 | response_txt = SelfieAlert_EmailHandler()
259 |
260 | return (response_txt)
261 |
262 |
263 |
264 | def SelfieAlert_EmailHandler():
265 |
266 | # Image should have been captured earlier
267 | if SelfieAlert.image != None:
268 | body = SelfieAlert.image
269 |
270 | # Save image to file
271 | with open(image_file, 'wb') as f:
272 | f.write(body)
273 | f.close()
274 |
275 | see_txt = ms_WhatDoYouSee(body)
276 | print ("see_txt=%s"% see_txt)
277 |
278 | if len(see_txt) > 0:
279 |
280 | # sending email is optional. This is good for testing, but should use secure storage
281 | # change this
282 | # not a good idea to hard code this info, but I used a test email and used as a quick POC for email. Best to get the info from a secure location.
283 | fromAddr = 'change this'
284 | toAddr = 'change this'
285 | email_pwd = 'change this or reference from a function'
286 |
287 | # send email - must define from and to address
288 | # this is not secure
289 | eml_SendEmail(fromAddr, toAddr, email_pwd, "See Talker Selfie", see_txt, image_file)
290 | response_txt = "Selfie taken and email sent successfully"
291 |
292 | img = Image.open(BytesIO(SelfieAlert.image))
293 | img.show(title=see_txt)
294 |
295 | else:
296 | response_txt = "There was a problem taking the selfie!"
297 |
298 |
299 | return (response_txt)
300 |
301 | def GetSound_SSML(sound_url):
302 | # only called once, but might be extended for more sophisticated SSML
303 | ssml_txt = (" "% sound_url)
304 | print ("ssml text: %s"% ssml_txt)
305 |
306 | # not using this now, but may be needed for more advanced SSML with a POST
307 | output_sound = {
308 | "type": "SSML",
309 | "ssml": ssml_txt
310 | #"ssml": " "
311 | }
312 |
313 | return (ssml_txt)
314 |
315 |
316 |
317 |
318 | def PostDirective_SpeechText(requestId, apiAccessToken, speech_txt):
319 |
320 | directive_hdr = {
321 | 'Content-Type': 'application/json',
322 | 'Authorization': apiAccessToken
323 | }
324 |
325 |
326 | directive_body = {
327 | "header":{
328 | "requestId":requestId
329 | },
330 | "directive":{
331 | "type":"VoicePlayer.Speak",
332 | "speech":speech_txt}
333 | }
334 |
335 | print("requestId=%s apiToken=%s"% (requestId, apiAccessToken))
336 | print("speech_txt = %s"% speech_txt)
337 |
338 | try:
339 | api_url = 'https://api.amazonalexa.com/v1/directives'
340 | json_body = json.dumps(directive_body)
341 |
342 | conn = http.client.HTTPSConnection('api.amazonalexa.com')
343 | response = conn.request("POST", '/v1/directives', json_body, directive_hdr)
344 | #response = conn.getresponse()
345 |
346 | conn.close()
347 |
348 | #response = requests.post(api_url, headers=directive_hdr, data=directive_hdr)
349 |
350 |
351 |
352 | except Exception as e:
353 | print('Error:')
354 | print(e)
355 |
356 |
357 | if __name__ == '__main__':
358 | app.run(host='0.0.0.0', threaded=True)
--------------------------------------------------------------------------------