├── src ├── aiy │ ├── __init__.py │ ├── vision │ │ ├── __init__.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── utils.py │ │ │ ├── dish_detection.py │ │ │ ├── face_detection.py │ │ │ ├── dish_classification.py │ │ │ ├── inaturalist_classification.py │ │ │ └── image_classification.py │ │ ├── proto │ │ │ └── __init__.py │ │ ├── streaming │ │ │ ├── proto │ │ │ │ ├── __init__.py │ │ │ │ ├── Makefile │ │ │ │ ├── messages.proto │ │ │ │ └── protocol.md │ │ │ ├── assets │ │ │ │ ├── messages.proto │ │ │ │ ├── index.html │ │ │ │ ├── broadway │ │ │ │ │ └── LICENSE │ │ │ │ └── ws_client.js │ │ │ └── svg.py │ │ └── _transport.py │ └── voice │ │ ├── __init__.py │ │ └── tts.py ├── examples │ ├── buzzer │ │ ├── dramatic.track │ │ ├── congratulations.track │ │ ├── laughing.track │ │ ├── sadtrombone.track │ │ ├── buzzer_tracker_demo.py │ │ ├── buzzer_demo.py │ │ └── tetris.track │ ├── vision │ │ ├── video_capture │ │ │ └── boat_classes.txt │ │ ├── object_meter │ │ │ ├── object_meter_demo.service │ │ │ ├── README.md │ │ │ ├── install_services.sh │ │ │ └── wordnet_grouping │ │ │ │ └── category_mapper.py │ │ ├── joy │ │ │ ├── joy_detection_demo.service │ │ │ └── install-services.sh │ │ ├── dish_classification.py │ │ ├── face_camera_trigger.py │ │ ├── face_detection.py │ │ ├── dish_detection.py │ │ ├── face_detection_raspivid.py │ │ ├── image_classification_camera.py │ │ ├── any_model_camera.py │ │ ├── inaturalist_classification.py │ │ ├── object_detection.py │ │ ├── face_detection_camera.py │ │ ├── image_classification.py │ │ └── mobilenet_based_classifier.py │ ├── gpiozero │ │ ├── led_chaser.py │ │ ├── led_example.py │ │ ├── simple_button_example.py │ │ ├── servo_example.py │ │ ├── bonnet_button.py │ │ └── button_example.py │ ├── button_led.py │ ├── voice │ │ └── voice_recorder.py │ └── leds_example.py └── tests │ ├── dish_classification_test.py │ ├── test_util.py │ ├── dish_detection_test.py │ ├── face_detection_test.py │ ├── images │ └── Makefile │ ├── object_detection_test.py │ ├── image_classification_test.py │ ├── inaturalist_classification_test.py │ ├── vision_examples_test.py │ └── engine_test.py ├── drivers ├── aiy │ ├── debian │ │ ├── compat │ │ ├── copyright │ │ ├── control │ │ ├── rules │ │ ├── aiy-dkms.dkms │ │ └── changelog │ ├── aiy │ │ ├── iio │ │ │ ├── Makefile │ │ │ └── adc │ │ │ │ ├── Kconfig │ │ │ │ ├── Makefile │ │ │ │ └── make.sh │ │ ├── Makefile │ │ ├── pwm │ │ │ ├── Kconfig │ │ │ ├── Makefile │ │ │ └── make.sh │ │ ├── gpio │ │ │ ├── Kconfig │ │ │ ├── Makefile │ │ │ └── make.sh │ │ └── mfd │ │ │ ├── Kconfig │ │ │ ├── Makefile │ │ │ └── make.sh │ └── Makefile ├── leds │ ├── debian │ │ ├── compat │ │ ├── leds-ktd202x-dkms.dkms │ │ ├── copyright │ │ ├── leds-ktd202x-dkms.udev │ │ ├── control │ │ ├── rules │ │ └── changelog │ ├── Makefile │ └── make.sh ├── pwm │ ├── debian │ │ ├── compat │ │ ├── modules-load.d │ │ │ └── pwm-soft.conf │ │ ├── pwm-soft-dkms.install │ │ ├── pwm-soft-dkms.dkms │ │ ├── copyright │ │ ├── changelog │ │ ├── control │ │ ├── pwm-soft-dkms.postinst │ │ ├── rules │ │ └── pwm-soft-dkms.udev │ └── Makefile ├── sound │ ├── debian │ │ ├── compat │ │ ├── aiy-voicebonnet-soundcard-dkms.install │ │ ├── ucm2 │ │ │ ├── aiy-voicebonnet.conf │ │ │ └── aiy-voicebonnet-HiFi.conf │ │ ├── copyright │ │ ├── aiy-voicebonnet-soundcard-dkms.dkms │ │ ├── control │ │ ├── rules │ │ └── changelog │ ├── Makefile │ ├── README.md │ ├── make.sh │ ├── rl6231.h │ └── rt5645.c.patch ├── vision │ ├── debian │ │ ├── compat │ │ ├── aiy-vision-dkms.udev │ │ ├── aiy-vision-dkms.dkms │ │ ├── copyright │ │ ├── control │ │ ├── rules │ │ └── changelog │ ├── Makefile │ ├── make.sh │ └── aiy-vision.h ├── overlays │ ├── voice │ │ ├── debian │ │ │ ├── compat │ │ │ ├── aiy-overlay-voice.install │ │ │ ├── rules │ │ │ ├── copyright │ │ │ ├── changelog │ │ │ ├── aiy-overlay-voice.prerm │ │ │ ├── control │ │ │ └── aiy-overlay-voice.postinst │ │ └── opt │ │ │ └── aiy │ │ │ └── overlay-voice │ │ │ ├── aiy-io-voice-overlay.dts │ │ │ ├── aiy-leds-voice-overlay.dts │ │ │ └── aiy-voicebonnet-overlay.dts │ ├── vision │ │ ├── debian │ │ │ ├── compat │ │ │ ├── aiy-overlay-vision.install │ │ │ ├── rules │ │ │ ├── copyright │ │ │ ├── changelog │ │ │ ├── aiy-overlay-vision.prerm │ │ │ ├── control │ │ │ └── aiy-overlay-vision.postinst │ │ └── opt │ │ │ └── aiy │ │ │ └── overlay-vision │ │ │ ├── aiy-io-vision-overlay.dts │ │ │ ├── aiy-leds-vision-overlay.dts │ │ │ └── aiy-visionbonnet-overlay.dts │ ├── make_dpkg.sh │ └── install.sh └── .gitignore ├── docs ├── aiy.pins.rst ├── aiy.board.rst ├── directives.txt ├── aiy.voice.audio.rst ├── _static │ ├── images │ │ ├── vision-kit.png │ │ ├── voice-kit.png │ │ └── bonnet-pins_2x.png │ └── custom.css ├── requirements.txt ├── aiy.leds.rst ├── aiy.voice.tts.rst ├── aiy.toneplayer.rst ├── aiy.trackplayer.rst ├── aiy.vision.annotator.rst ├── aiy.vision.inference.rst ├── README.md ├── index.rst ├── aiy.vision.models.rst ├── vision.md └── voice.md ├── eeprom └── voicekit_v1.eep ├── checkpoints ├── test_hello.raw ├── check_wifi.py └── check_audio.py ├── schematics └── voice_hat │ ├── voice_hat.pdf │ └── voice_hat_mic.pdf ├── .gitignore ├── stdeb.cfg ├── MANIFEST.in ├── .coveragerc ├── .travis.yml ├── scripts └── pre-commit ├── setup.py ├── CONTRIBUTING.md ├── README.md ├── Makefile └── CHANGES.md /src/aiy/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/aiy/vision/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/aiy/voice/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /drivers/aiy/debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /drivers/leds/debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /drivers/pwm/debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /src/aiy/vision/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/aiy/vision/proto/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /drivers/sound/debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /drivers/vision/debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /drivers/overlays/voice/debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /src/aiy/vision/streaming/proto/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /drivers/overlays/vision/debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /drivers/overlays/vision/debian/aiy-overlay-vision.install: -------------------------------------------------------------------------------- 1 | opt / -------------------------------------------------------------------------------- /drivers/overlays/voice/debian/aiy-overlay-voice.install: -------------------------------------------------------------------------------- 1 | opt / -------------------------------------------------------------------------------- /drivers/pwm/debian/modules-load.d/pwm-soft.conf: -------------------------------------------------------------------------------- 1 | pwm-soft 2 | -------------------------------------------------------------------------------- /src/aiy/vision/streaming/assets/messages.proto: -------------------------------------------------------------------------------- 1 | ../proto/messages.proto -------------------------------------------------------------------------------- /drivers/aiy/aiy/iio/Makefile: -------------------------------------------------------------------------------- 1 | KBUILD_CFLAGS += -I$(src)/include 2 | 3 | obj-y += adc/ 4 | -------------------------------------------------------------------------------- /docs/aiy.pins.rst: -------------------------------------------------------------------------------- 1 | aiy.pins 2 | ======== 3 | 4 | .. automodule:: aiy.pins 5 | :noindex: 6 | -------------------------------------------------------------------------------- /docs/aiy.board.rst: -------------------------------------------------------------------------------- 1 | aiy.board 2 | ========= 3 | 4 | .. automodule:: aiy.board 5 | :noindex: 6 | -------------------------------------------------------------------------------- /drivers/pwm/debian/pwm-soft-dkms.install: -------------------------------------------------------------------------------- 1 | debian/modules-load.d/pwm-soft.conf /usr/lib/modules-load.d 2 | -------------------------------------------------------------------------------- /docs/directives.txt: -------------------------------------------------------------------------------- 1 | .. |code| raw:: html 2 | 3 | 4 | 5 | .. |endcode| raw:: html 6 | 7 | -------------------------------------------------------------------------------- /drivers/vision/debian/aiy-vision-dkms.udev: -------------------------------------------------------------------------------- 1 | KERNEL=="vision_spicomm", SUBSYSTEM=="spicomm", TAG+="systemd" 2 | -------------------------------------------------------------------------------- /eeprom/voicekit_v1.eep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viraniac/aiyprojects-raspbian/HEAD/eeprom/voicekit_v1.eep -------------------------------------------------------------------------------- /checkpoints/test_hello.raw: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viraniac/aiyprojects-raspbian/HEAD/checkpoints/test_hello.raw -------------------------------------------------------------------------------- /drivers/sound/debian/aiy-voicebonnet-soundcard-dkms.install: -------------------------------------------------------------------------------- 1 | debian/ucm2/* /usr/share/alsa/ucm2/aiy-voicebonnet/ 2 | -------------------------------------------------------------------------------- /docs/aiy.voice.audio.rst: -------------------------------------------------------------------------------- 1 | aiy.voice.audio 2 | =============== 3 | 4 | .. automodule:: aiy.voice.audio 5 | :noindex: 6 | -------------------------------------------------------------------------------- /docs/_static/images/vision-kit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viraniac/aiyprojects-raspbian/HEAD/docs/_static/images/vision-kit.png -------------------------------------------------------------------------------- /docs/_static/images/voice-kit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viraniac/aiyprojects-raspbian/HEAD/docs/_static/images/voice-kit.png -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # Python packages required to build the docs 2 | sphinx 3 | sphinx_rtd_theme 4 | recommonmark 5 | Pillow 6 | -------------------------------------------------------------------------------- /schematics/voice_hat/voice_hat.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viraniac/aiyprojects-raspbian/HEAD/schematics/voice_hat/voice_hat.pdf -------------------------------------------------------------------------------- /docs/_static/images/bonnet-pins_2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viraniac/aiyprojects-raspbian/HEAD/docs/_static/images/bonnet-pins_2x.png -------------------------------------------------------------------------------- /schematics/voice_hat/voice_hat_mic.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viraniac/aiyprojects-raspbian/HEAD/schematics/voice_hat/voice_hat_mic.pdf -------------------------------------------------------------------------------- /docs/aiy.leds.rst: -------------------------------------------------------------------------------- 1 | aiy.leds 2 | ======== 3 | 4 | .. automodule:: aiy.leds 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/Makefile: -------------------------------------------------------------------------------- 1 | KBUILD_CFLAGS += -I$(src)/include 2 | 3 | obj-y += mfd/ 4 | obj-y += gpio/ 5 | obj-y += pwm/ 6 | obj-y += iio/ 7 | 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | __pycache__ 4 | *.pyc 5 | .DS_Store 6 | .debhelper 7 | docs/_build 8 | src/tests/images/*.jpg 9 | *.egg-info 10 | -------------------------------------------------------------------------------- /docs/aiy.voice.tts.rst: -------------------------------------------------------------------------------- 1 | aiy.voice.tts 2 | ============= 3 | 4 | .. automodule:: aiy.voice.tts 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /src/aiy/vision/streaming/proto/Makefile: -------------------------------------------------------------------------------- 1 | messages_pb2.py: messages.proto 2 | protoc --python_out=. messages.proto 3 | 4 | clean: 5 | rm -f messages_pb2.py 6 | -------------------------------------------------------------------------------- /docs/aiy.toneplayer.rst: -------------------------------------------------------------------------------- 1 | aiy.toneplayer 2 | ============== 3 | 4 | .. automodule:: aiy.toneplayer 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /stdeb.cfg: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | Depends3: avahi-utils, libttspico-utils, aiy-voicebonnet-soundcard-dkms, aiy-vision-dkms, aiy-models, aiy-dkms, pwm-soft-dkms, leds-ktd202x-dkms 3 | -------------------------------------------------------------------------------- /docs/aiy.trackplayer.rst: -------------------------------------------------------------------------------- 1 | aiy.trackplayer 2 | =============== 3 | 4 | .. automodule:: aiy.trackplayer 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /drivers/overlays/vision/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | # Uncomment this to turn on verbose mode. 5 | # export DH_VERBOSE=1 6 | 7 | %: 8 | dh $@ 9 | -------------------------------------------------------------------------------- /drivers/overlays/voice/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | # Uncomment this to turn on verbose mode. 5 | # export DH_VERBOSE=1 6 | 7 | %: 8 | dh $@ 9 | -------------------------------------------------------------------------------- /docs/aiy.vision.annotator.rst: -------------------------------------------------------------------------------- 1 | aiy.vision.annotator 2 | ==================== 3 | 4 | .. automodule:: aiy.vision.annotator 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /docs/aiy.vision.inference.rst: -------------------------------------------------------------------------------- 1 | aiy.vision.inference 2 | ==================== 3 | 4 | .. automodule:: aiy.vision.inference 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: -------------------------------------------------------------------------------- /drivers/sound/debian/ucm2/aiy-voicebonnet.conf: -------------------------------------------------------------------------------- 1 | Syntax 2 2 | Comment "Google AIY Voice Bonnet" 3 | SectionUseCase."HiFi" { 4 | File "/aiy-voicebonnet/aiy-voicebonnet-HiFi.conf" 5 | Comment "Default" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /drivers/pwm/debian/pwm-soft-dkms.dkms: -------------------------------------------------------------------------------- 1 | PACKAGE_VERSION="#MODULE_VERSION#" 2 | PACKAGE_NAME="pwm-soft" 3 | 4 | BUILT_MODULE_NAME[0]="pwm-soft" 5 | DEST_MODULE_LOCATION[0]="/kernel/drivers/pwm/" 6 | 7 | AUTOINSTALL="yes" 8 | -------------------------------------------------------------------------------- /drivers/leds/debian/leds-ktd202x-dkms.dkms: -------------------------------------------------------------------------------- 1 | PACKAGE_VERSION="#MODULE_VERSION#" 2 | PACKAGE_NAME="leds-ktd202x" 3 | 4 | BUILT_MODULE_NAME[0]="leds-ktd202x" 5 | DEST_MODULE_LOCATION[0]="/kernel/drivers/leds/" 6 | 7 | AUTOINSTALL="yes" 8 | -------------------------------------------------------------------------------- /drivers/vision/debian/aiy-vision-dkms.dkms: -------------------------------------------------------------------------------- 1 | PACKAGE_VERSION="#MODULE_VERSION#" 2 | PACKAGE_NAME="aiy-vision" 3 | 4 | BUILT_MODULE_NAME[0]="aiy-vision" 5 | DEST_MODULE_LOCATION[0]="/kernel/drivers/staging/myriad" 6 | 7 | AUTOINSTALL="yes" 8 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include checkpoints * 2 | recursive-include docs * 3 | recursive-include src/examples * 4 | recursive-include src/tests * 5 | 6 | include README.md 7 | include CONTRIBUTING.md 8 | include LICENSE 9 | include Makefile 10 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/iio/adc/Kconfig: -------------------------------------------------------------------------------- 1 | menuconfig CONFIG_AIY_ADC 2 | tristate "AIY ADC Support" 3 | help 4 | Say Y here if you want to use AIY ADC. 5 | 6 | To compile this driver as a module, choose M here: the 7 | module will be called aiy-adc. 8 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/pwm/Kconfig: -------------------------------------------------------------------------------- 1 | menuconfig CONFIG_PWM_AIY_IO 2 | tristate "AIY PWM Support" 3 | help 4 | Say Y here if you want to use AIY PWM. 5 | 6 | To compile this driver as a module, choose M here: the 7 | module will be called pwm-aiy-io. 8 | -------------------------------------------------------------------------------- /drivers/pwm/Makefile: -------------------------------------------------------------------------------- 1 | obj-m := pwm-soft.o 2 | KVERSION := $(shell uname -r) 3 | 4 | all: 5 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) modules 6 | 7 | clean: 8 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) clean 9 | 10 | 11 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/gpio/Kconfig: -------------------------------------------------------------------------------- 1 | menuconfig CONFIG_GPIO_AIY_IO 2 | tristate "AIY GPIO Support" 3 | help 4 | Say Y here if you want to use AIY GPIO. 5 | 6 | To compile this driver as a module, choose M here: the 7 | module will be called gpio-aiy-io. 8 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/mfd/Kconfig: -------------------------------------------------------------------------------- 1 | menuconfig CONFIG_AIY_IO_I2C 2 | tristate "AIY IO I2C Support" 3 | help 4 | Say Y here if you want to use AIY IO I2C. 5 | 6 | To compile this driver as a module, choose M here: the 7 | module will be called aiy-io-i2c. 8 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = src 4 | 5 | [report] 6 | exclude_lines = 7 | if self.debug: 8 | pragma: no cover 9 | raise NotImplementedError 10 | if __name__ == .__main__.: 11 | ignore_errors = True 12 | omit = 13 | tests/* -------------------------------------------------------------------------------- /src/examples/buzzer/dramatic.track: -------------------------------------------------------------------------------- 1 | title Dramatic Tune 2 | speed 10 3 | order 0 4 | end 5 | 6 | pattern 7 | G4 8 | 9 | noff 10 | 11 | G4 12 | 13 | noff 14 | 15 | G4 16 | 17 | noff 18 | 19 | f4 20 | 21 | noff f4 22 | 23 | stop 24 | end 25 | -------------------------------------------------------------------------------- /src/examples/vision/video_capture/boat_classes.txt: -------------------------------------------------------------------------------- 1 | catamaran 2 | container ship/containership/container vessel 3 | lifeboat 4 | speedboat 5 | paddle/boat paddle 6 | pirate/pirate ship 7 | paddlewheel/paddle wheel 8 | submarine/pigboat/sub/U-boat 9 | fireboat 10 | -------------------------------------------------------------------------------- /drivers/leds/Makefile: -------------------------------------------------------------------------------- 1 | obj-m := leds-ktd202x.o 2 | KVERSION := $(shell uname -r) 3 | 4 | all: 5 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) modules 6 | 7 | clean: 8 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) clean 9 | 10 | 11 | -------------------------------------------------------------------------------- /drivers/sound/Makefile: -------------------------------------------------------------------------------- 1 | obj-m := snd-aiy-voicebonnet.o rt5645.o rl6231.o 2 | KVERSION := $(shell uname -r) 3 | 4 | all: 5 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) modules 6 | 7 | clean: 8 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) clean 9 | 10 | -------------------------------------------------------------------------------- /src/examples/buzzer/congratulations.track: -------------------------------------------------------------------------------- 1 | title Congratulations! 2 | speed 5 3 | order 0 4 | end 5 | 6 | pattern 7 | C4 8 | 9 | 10 | D4 11 | 12 | 13 | E4 14 | 15 | 16 | G4 17 | 18 | 19 | 20 | 21 | E4 22 | 23 | 24 | G4 25 | 26 | 27 | 28 | 29 | stop 30 | end 31 | -------------------------------------------------------------------------------- /drivers/aiy/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: aiy-dkms 3 | Source: https://aiyprojects.withgoogle.com 4 | 5 | Files: * 6 | Copyright: Copyright 2017 Google, LLC 7 | License: Apache-2.0 8 | -------------------------------------------------------------------------------- /drivers/pwm/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: pwm-soft-dkms 3 | Source: https://aiyprojects.withgoogle.com 4 | 5 | Files: * 6 | Copyright: Copyright 2017 Google, LLC 7 | License: Apache-2.0 8 | -------------------------------------------------------------------------------- /drivers/leds/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: leds-ktd202x-dkms 3 | Source: https://aiyprojects.withgoogle.com 4 | 5 | Files: * 6 | Copyright: Copyright 2017 Google, LLC 7 | License: Apache-2.0 8 | -------------------------------------------------------------------------------- /drivers/vision/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: aiy-vision-dkms 3 | Source: https://aiyprojects.withgoogle.com 4 | 5 | Files: * 6 | Copyright: Copyright 2017 Google, LLC 7 | License: Apache-2.0 8 | -------------------------------------------------------------------------------- /drivers/vision/Makefile: -------------------------------------------------------------------------------- 1 | obj-m := aiy-vision.o 2 | ccflags-y := -std=gnu99 -Wno-declaration-after-statement 3 | KVERSION := $(shell uname -r) 4 | 5 | all: 6 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) modules 7 | 8 | clean: 9 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) clean 10 | -------------------------------------------------------------------------------- /drivers/overlays/voice/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: aiy-overlay-voice 3 | Source: https://aiyprojects.withgoogle.com 4 | 5 | Files: * 6 | Copyright: Copyright 2017 Google, LLC 7 | License: Apache-2.0 8 | -------------------------------------------------------------------------------- /drivers/overlays/vision/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: aiy-overlay-vision 3 | Source: https://aiyprojects.withgoogle.com 4 | 5 | Files: * 6 | Copyright: Copyright 2017 Google, LLC 7 | License: Apache-2.0 8 | -------------------------------------------------------------------------------- /drivers/sound/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: aiy-voicebonnet-soundcard-dkms 3 | Source: https://aiyprojects.withgoogle.com 4 | 5 | Files: * 6 | Copyright: Copyright 2017 Google, LLC 7 | License: Apache-2.0 8 | -------------------------------------------------------------------------------- /src/examples/vision/object_meter/object_meter_demo.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=AIY Object Meter Demo 3 | 4 | [Service] 5 | Type=simple 6 | Restart=no 7 | ExecStart=/usr/bin/python3 /home/pi/AIY-projects-python/src/examples/vision/object_meter/object_meter.py 8 | 9 | [Install] 10 | WantedBy=multi-user.target 11 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/iio/adc/Makefile: -------------------------------------------------------------------------------- 1 | obj-$(CONFIG_AIY_ADC) := aiy-adc.o 2 | ccflags-y := -std=gnu99 -Wno-declaration-after-statement 3 | KVERSION := $(shell uname -r) 4 | 5 | all: 6 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) modules 7 | 8 | clean: 9 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) clean 10 | 11 | -------------------------------------------------------------------------------- /src/examples/vision/object_meter/README.md: -------------------------------------------------------------------------------- 1 | # Image classification demo 2 | 3 | ## Physical Setup 4 | 5 | - VisionHat installed on a Raspberry Pi Zero 6 | - Servo connected to vision hat (signal - PIN_A, Vcc - POWER, Ground - GND) 7 | - Servo mounted in label display 8 | - (optional) Monitor connected to raspberry pi. 9 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/gpio/Makefile: -------------------------------------------------------------------------------- 1 | obj-$(CONFIG_GPIO_AIY_IO) := gpio-aiy-io.o 2 | ccflags-y := -std=gnu99 -Wno-declaration-after-statement 3 | KVERSION := $(shell uname -r) 4 | 5 | all: 6 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) modules 7 | 8 | clean: 9 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) clean 10 | 11 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/mfd/Makefile: -------------------------------------------------------------------------------- 1 | obj-$(CONFIG_AIY_IO_I2C) := aiy-io-i2c.o 2 | ccflags-y := -std=gnu99 -Wno-declaration-after-statement 3 | KVERSION := $(shell uname -r) 4 | 5 | all: 6 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) modules 7 | 8 | clean: 9 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) clean 10 | 11 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/pwm/Makefile: -------------------------------------------------------------------------------- 1 | obj-$(CONFIG_PWM_AIY_IO) := pwm-aiy-io.o 2 | ccflags-y := -std=gnu99 -Wno-declaration-after-statement 3 | KVERSION := $(shell uname -r) 4 | 5 | all: 6 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) modules 7 | 8 | clean: 9 | $(MAKE) -C /lib/modules/$(KVERSION)/build M=$(PWD) clean 10 | 11 | -------------------------------------------------------------------------------- /drivers/leds/debian/leds-ktd202x-dkms.udev: -------------------------------------------------------------------------------- 1 | SUBSYSTEM=="leds", ACTION=="add", RUN+="/bin/chgrp -R gpio /sys%p", RUN+="/bin/chmod -R ug+rw /sys%p", RUN+="/bin/chgrp -R gpio /sys%p/device/", RUN+="/bin/chmod -R ug+rw /sys%p/device/" 2 | SUBSYSTEM=="leds", ACTION=="change", ENV{TRIGGER}!="none", RUN+="/bin/chgrp -R gpio /sys%p", RUN+="/bin/chmod -R ug+rw /sys%p" 3 | -------------------------------------------------------------------------------- /drivers/overlays/make_dpkg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o xtrace 4 | set -o errexit 5 | 6 | readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 7 | 8 | pushd ${SCRIPT_DIR}/vision 9 | dpkg-buildpackage -b -rfakeroot -us -uc -tc 10 | popd 11 | 12 | pushd ${SCRIPT_DIR}/voice 13 | dpkg-buildpackage -b -rfakeroot -us -uc -tc 14 | popd 15 | -------------------------------------------------------------------------------- /drivers/overlays/vision/debian/changelog: -------------------------------------------------------------------------------- 1 | aiy-overlay-vision (1.0-1) stable; urgency=medium 2 | 3 | * Update debian package. 4 | 5 | -- Dmitry Kovalev Wed, 14 Nov 2018 13:56:08 -0800 6 | 7 | aiy-overlay-vision (1.0-0) stable; urgency=medium 8 | 9 | * Initial release. 10 | 11 | -- Dmitry Kovalev Mon, 04 Jun 2018 21:03:54 -0700 12 | -------------------------------------------------------------------------------- /drivers/overlays/voice/debian/changelog: -------------------------------------------------------------------------------- 1 | aiy-overlay-voice (1.0-1) stable; urgency=medium 2 | 3 | * Update debian package. 4 | 5 | -- Dmitry Kovalev Wed, 14 Nov 2018 13:56:08 -0800 6 | 7 | aiy-overlay-voice (1.0-0) stable; urgency=medium 8 | 9 | * Initial release. 10 | 11 | -- Dmitry Kovalev Mon, 04 Jun 2018 21:03:54 -0700 12 | -------------------------------------------------------------------------------- /drivers/pwm/debian/changelog: -------------------------------------------------------------------------------- 1 | pwm-soft-dkms (2.0-1) stable; urgency=medium 2 | 3 | * Update driver to support 4.19 kernel. 4 | 5 | -- Dmitry Kovalev Wed, 06 Nov 2019 16:36:24 -0800 6 | 7 | pwm-soft-dkms (1.1-2) stable; urgency=medium 8 | 9 | * Proper debian package based on dh_dkms. 10 | 11 | -- Dmitry Kovalev Thu, 30 Aug 2018 15:28:07 -0700 12 | -------------------------------------------------------------------------------- /drivers/leds/make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR=$(dirname $(realpath $0)) 3 | SPACEPARK=${DIR}/../.. 4 | pushd ${DIR} 5 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 6 | SUBDIRS=${SPACEPARK}/drivers-raspi/leds clean 7 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 8 | SUBDIRS=${SPACEPARK}/drivers-raspi/leds modules 9 | popd 10 | -------------------------------------------------------------------------------- /drivers/aiy/debian/control: -------------------------------------------------------------------------------- 1 | Source: aiy-dkms 2 | Section: misc 3 | Priority: optional 4 | Maintainer: AIY Projects 5 | Build-Depends: debhelper (>= 10), dkms 6 | Standards-Version: 3.8.1 7 | 8 | Package: aiy-dkms 9 | Architecture: all 10 | Depends: raspberrypi-kernel-headers, dkms (>= 1.95), ${misc:Depends} 11 | Description: Hardware device drivers for AIY Projects Kits. 12 | -------------------------------------------------------------------------------- /drivers/sound/README.md: -------------------------------------------------------------------------------- 1 | # What is this? 2 | 3 | This is the Linux kernel driver for the Google AIY Voice Kit v2 voice bonnet. 4 | 5 | Note: the rt6231 and rl5645 codec drivers were copied from the Linux source tree 6 | because the symbols needed in the snd-aiy-voicebonnet driver are not available 7 | in DKMS builds. 8 | 9 | These files are identical to the ones in the kernel tree at sound/soc/codecs/ 10 | -------------------------------------------------------------------------------- /drivers/overlays/voice/debian/aiy-overlay-voice.prerm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | case "$1" in 6 | remove|upgrade|deconfigure) 7 | rm -f /boot/overlays/aiy-voicebonnet.dtbo 8 | ;; 9 | 10 | failed-upgrade) 11 | ;; 12 | 13 | *) 14 | echo "prerm called with unknown argument \`${1}'" >&2 15 | exit 1 16 | ;; 17 | esac 18 | 19 | #DEBHELPER# 20 | 21 | exit 0 22 | -------------------------------------------------------------------------------- /drivers/overlays/vision/debian/aiy-overlay-vision.prerm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | case "$1" in 6 | remove|upgrade|deconfigure) 7 | rm -f /boot/overlays/aiy-visionbonnet.dtbo 8 | ;; 9 | 10 | failed-upgrade) 11 | ;; 12 | 13 | *) 14 | echo "prerm called with unknown argument \`${1}'" >&2 15 | exit 1 16 | ;; 17 | esac 18 | 19 | #DEBHELPER# 20 | 21 | exit 0 22 | -------------------------------------------------------------------------------- /drivers/pwm/debian/control: -------------------------------------------------------------------------------- 1 | Source: pwm-soft-dkms 2 | Section: misc 3 | Priority: optional 4 | Maintainer: AIY Projects 5 | Build-Depends: debhelper (>= 10), dkms 6 | Standards-Version: 3.8.1 7 | 8 | Package: pwm-soft-dkms 9 | Architecture: all 10 | Depends: raspberrypi-kernel-headers, dkms (>= 1.95), ${misc:Depends} 11 | Description: Hardware device drivers for AIY Projects Kits. 12 | -------------------------------------------------------------------------------- /drivers/leds/debian/control: -------------------------------------------------------------------------------- 1 | Source: leds-ktd202x-dkms 2 | Section: misc 3 | Priority: optional 4 | Maintainer: AIY Projects 5 | Build-Depends: debhelper (>= 10), dkms 6 | Standards-Version: 3.8.1 7 | 8 | Package: leds-ktd202x-dkms 9 | Architecture: all 10 | Depends: raspberrypi-kernel-headers, dkms (>= 1.95), ${misc:Depends} 11 | Description: KTD2026/KTD2027 drivers for AIY Vision/Voice Bonnet 12 | -------------------------------------------------------------------------------- /drivers/overlays/voice/debian/control: -------------------------------------------------------------------------------- 1 | Source: aiy-overlay-voice 2 | Section: misc 3 | Priority: optional 4 | Maintainer: AIY Projects 5 | Build-Depends: debhelper (>= 10) 6 | Standards-Version: 3.8.1 7 | 8 | Package: aiy-overlay-voice 9 | Architecture: all 10 | Depends: raspberrypi-kernel, device-tree-compiler (>= 1.4), ${misc:Depends} 11 | Description: Device tree overlay for AIY Voice Bonnet. 12 | -------------------------------------------------------------------------------- /drivers/pwm/debian/pwm-soft-dkms.postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | #DEBHELPER# 6 | 7 | case "$1" in 8 | configure) 9 | if [ -z "$2" ]; then 10 | modprobe pwm-soft || true 11 | fi 12 | ;; 13 | abort-upgrade|abort-remove|abort-deconfigure) 14 | ;; 15 | 16 | *) 17 | echo "postinst called with unknown argument \`$1'" >&2 18 | exit 0 19 | ;; 20 | esac 21 | -------------------------------------------------------------------------------- /drivers/overlays/vision/debian/control: -------------------------------------------------------------------------------- 1 | Source: aiy-overlay-vision 2 | Section: misc 3 | Priority: optional 4 | Maintainer: AIY Projects 5 | Build-Depends: debhelper (>= 10) 6 | Standards-Version: 3.8.1 7 | 8 | Package: aiy-overlay-vision 9 | Architecture: all 10 | Depends: raspberrypi-kernel, device-tree-compiler (>= 1.4), ${misc:Depends} 11 | Description: Device tree overlay for AIY Vision Bonnet. 12 | -------------------------------------------------------------------------------- /drivers/overlays/vision/opt/aiy/overlay-vision/aiy-io-vision-overlay.dts: -------------------------------------------------------------------------------- 1 | /dts-v1/; 2 | /plugin/; 3 | / { 4 | compatible = "brcm,bcm2708"; 5 | fragment@0 { 6 | target = <&i2c_arm>; 7 | __overlay__ { 8 | #address-cells = <1>; 9 | #size-cells = <0>; 10 | status = "okay"; 11 | aiy-io-i2c@51 { 12 | compatible = "google,aiy-io-i2c"; 13 | reg = <0x51>; 14 | status = "okay"; 15 | }; 16 | }; 17 | }; 18 | }; 19 | -------------------------------------------------------------------------------- /drivers/sound/debian/aiy-voicebonnet-soundcard-dkms.dkms: -------------------------------------------------------------------------------- 1 | PACKAGE_VERSION="#MODULE_VERSION#" 2 | PACKAGE_NAME="aiy-voicebonnet-soundcard" 3 | 4 | BUILT_MODULE_NAME[0]="snd-aiy-voicebonnet" 5 | DEST_MODULE_LOCATION[0]="/kernel/sound/drivers" 6 | BUILT_MODULE_NAME[1]="rt5645" 7 | DEST_MODULE_LOCATION[1]="/kernel/sound/soc/codecs" 8 | BUILT_MODULE_NAME[2]="rl6231" 9 | DEST_MODULE_LOCATION[2]="/kernel/sound/soc/codecs" 10 | 11 | AUTOINSTALL="yes" 12 | -------------------------------------------------------------------------------- /drivers/sound/make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR=$(dirname $(realpath $0)) 3 | SPACEPARK=${DIR}/../.. 4 | pushd ${DIR} 5 | 6 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 7 | SUBDIRS=${SPACEPARK}/drivers-raspi/sound clean 8 | 9 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 10 | SUBDIRS=${SPACEPARK}/drivers-raspi/sound CONFIG_GPIO_AIY_IO=m modules 11 | 12 | popd 13 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/mfd/make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR=$(dirname $(realpath $0)) 3 | SPACEPARK=${DIR}/../../../.. 4 | pushd ${DIR} 5 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 6 | SUBDIRS=${SPACEPARK}/drivers-raspi/aiy/aiy/mfd clean 7 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 8 | SUBDIRS=${SPACEPARK}/drivers-raspi/aiy/aiy/mfd CONFIG_AIY_IO_I2C=m modules 9 | popd 10 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/pwm/make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR=$(dirname $(realpath $0)) 3 | SPACEPARK=${DIR}/../../../.. 4 | pushd ${DIR} 5 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 6 | SUBDIRS=${SPACEPARK}/drivers-raspi/aiy/aiy/pwm clean 7 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 8 | SUBDIRS=${SPACEPARK}/drivers-raspi/aiy/aiy/pwm CONFIG_PWM_AIY_IO=m modules 9 | popd 10 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/gpio/make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR=$(dirname $(realpath $0)) 3 | SPACEPARK=${DIR}/../../../.. 4 | pushd ${DIR} 5 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 6 | SUBDIRS=${SPACEPARK}/drivers-raspi/aiy/aiy/gpio clean 7 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 8 | SUBDIRS=${SPACEPARK}/drivers-raspi/aiy/aiy/gpio CONFIG_GPIO_AIY_IO=m modules 9 | popd 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - 3.4 4 | 5 | before_install: 6 | - sudo apt-get update -qq 7 | - sudo apt-get install -y python3-numpy python3-scipy 8 | 9 | install: 10 | - pip3 install pyflakes coverage pycodestyle 11 | - pip3 install -r requirements.txt 12 | 13 | script: 14 | - pycodestyle --max-line-length=100 --exclude=*_pb2.py . 15 | - nosetests --with-coverage 16 | 17 | after_success: 18 | - bash <(curl -s https://codecov.io/bash) 19 | 20 | -------------------------------------------------------------------------------- /drivers/aiy/aiy/iio/adc/make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR=$(dirname $(realpath $0)) 3 | SPACEPARK=${DIR}/../../../../.. 4 | pushd ${DIR} 5 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 6 | SUBDIRS=${SPACEPARK}/drivers-raspi/aiy/aiy/iio/adc clean 7 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 8 | SUBDIRS=${SPACEPARK}/drivers-raspi/aiy/aiy/iio/adc CONFIG_AIY_ADC=m modules 9 | popd 10 | -------------------------------------------------------------------------------- /src/examples/gpiozero/led_chaser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Demonstrates on board LED support with correct polarity. 3 | 4 | Implements simple LED chaser. 5 | """ 6 | 7 | from time import sleep 8 | from gpiozero import LED 9 | from aiy.pins import (PIN_A, PIN_B, PIN_C, PIN_D) 10 | 11 | leds = (LED(PIN_A), LED(PIN_B), LED(PIN_C), LED(PIN_D)) 12 | while True: 13 | for led in leds: 14 | led.on() 15 | sleep(0.5) 16 | led.off() 17 | -------------------------------------------------------------------------------- /drivers/overlays/vision/opt/aiy/overlay-vision/aiy-leds-vision-overlay.dts: -------------------------------------------------------------------------------- 1 | // Definitions for Google AIY LEDs 2 | /dts-v1/; 3 | /plugin/; 4 | / { 5 | compatible = "brcm,bcm2708"; 6 | fragment@0 { 7 | target = <&i2c_arm>; 8 | __overlay__ { 9 | #address-cells = <1>; 10 | #size-cells = <0>; 11 | status = "okay"; 12 | ktd2026@30 { 13 | compatible = "kinetic,ktd2026"; 14 | reg = <0x30>; 15 | status = "okay"; 16 | }; 17 | }; 18 | }; 19 | }; 20 | -------------------------------------------------------------------------------- /drivers/overlays/voice/opt/aiy/overlay-voice/aiy-io-voice-overlay.dts: -------------------------------------------------------------------------------- 1 | /dts-v1/; 2 | /plugin/; 3 | / { 4 | compatible = "brcm,bcm2708"; 5 | fragment@0 { 6 | target = <&i2c_arm>; 7 | __overlay__ { 8 | #address-cells = <1>; 9 | #size-cells = <0>; 10 | status = "okay"; 11 | aiy-io-i2c@52 { 12 | compatible = "google,aiy-io-i2c"; 13 | reg = <0x52>; 14 | status = "okay"; 15 | }; 16 | }; 17 | }; 18 | }; 19 | -------------------------------------------------------------------------------- /drivers/sound/debian/control: -------------------------------------------------------------------------------- 1 | Source: aiy-voicebonnet-soundcard-dkms 2 | Section: misc 3 | Priority: optional 4 | Maintainer: AIY Projects 5 | Build-Depends: debhelper (>= 10), dkms 6 | Standards-Version: 3.8.1 7 | 8 | Package: aiy-voicebonnet-soundcard-dkms 9 | Architecture: all 10 | Depends: raspberrypi-kernel-headers, dkms (>= 1.95), ${misc:Depends} 11 | Description: AIY Voice DKMS driver. 12 | Required driver for Voice Bonnet board. 13 | -------------------------------------------------------------------------------- /drivers/vision/debian/control: -------------------------------------------------------------------------------- 1 | Source: aiy-vision-dkms 2 | Section: misc 3 | Priority: optional 4 | Maintainer: AIY Projects 5 | Build-Depends: debhelper (>= 10), dkms 6 | Standards-Version: 3.8.1 7 | 8 | Package: aiy-vision-dkms 9 | Architecture: all 10 | Depends: raspberrypi-kernel-headers, aiy-dkms, aiy-vision-firmware, dkms (>= 1.95), ${misc:Depends} 11 | Description: AIY Vision DKMS driver. 12 | Required driver for Vision Bonnet board. 13 | -------------------------------------------------------------------------------- /drivers/vision/make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR=$(dirname $(realpath $0)) 3 | SPACEPARK=${DIR}/../../../../.. 4 | pushd ${DIR} 5 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 6 | SUBDIRS=${SPACEPARK}/drivers-raspi/aiy/aiy/staging/myriad clean 7 | make -C ${SPACEPARK}/raspberrypi-linux ARCH=arm CROSS_COMPILE=arm-linux-gnueabihf- \ 8 | SUBDIRS=${SPACEPARK}/drivers-raspi/aiy/aiy/staging/myriad CONFIG_AIY_VISION=m modules 9 | popd 10 | 11 | -------------------------------------------------------------------------------- /src/examples/gpiozero/led_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Demonstrates on board LED support with correct polarity. 3 | 4 | Demo will turn on, then off the first LED on the hat. 5 | """ 6 | 7 | from time import sleep 8 | from gpiozero import LED 9 | from aiy.pins import LED_1 10 | 11 | led = LED(LED_1) 12 | # Alternate turning the LED off and on until the user terminates the example. 13 | while True: 14 | led.on() 15 | sleep(1) 16 | led.off() 17 | sleep(1) 18 | -------------------------------------------------------------------------------- /src/examples/button_led.py: -------------------------------------------------------------------------------- 1 | from aiy.board import Board, Led 2 | 3 | def main(): 4 | print('LED is ON while button is pressed (Ctrl-C for exit).') 5 | with Board() as board: 6 | while True: 7 | board.button.wait_for_press() 8 | print('ON') 9 | board.led.state = Led.ON 10 | board.button.wait_for_release() 11 | print('OFF') 12 | board.led.state = Led.OFF 13 | 14 | 15 | if __name__ == '__main__': 16 | main() 17 | -------------------------------------------------------------------------------- /drivers/overlays/voice/opt/aiy/overlay-voice/aiy-leds-voice-overlay.dts: -------------------------------------------------------------------------------- 1 | // Definitions for Google AIY LEDs 2 | /dts-v1/; 3 | /plugin/; 4 | / { 5 | compatible = "brcm,bcm2708"; 6 | fragment@0 { 7 | target = <&i2c_arm>; 8 | __overlay__ { 9 | #address-cells = <1>; 10 | #size-cells = <0>; 11 | status = "okay"; 12 | ktd2026@31 { 13 | compatible = "kinetic,ktd2026"; 14 | reg = <0x31>; 15 | status = "okay"; 16 | }; 17 | }; 18 | }; 19 | }; 20 | -------------------------------------------------------------------------------- /drivers/aiy/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | # Uncomment this to turn on verbose mode. 5 | # export DH_VERBOSE=1 6 | 7 | VERSION=$(shell dpkg-parsechangelog | grep '^Version:' | cut -d' ' -f2 | cut -d- -f1) 8 | 9 | %: 10 | dh $@ --with dkms 11 | 12 | override_dh_install: 13 | dh_install aiy Makefile usr/src/aiy-$(VERSION)/ 14 | 15 | override_dh_dkms: 16 | dh_dkms -V $(VERSION) 17 | 18 | override_dh_auto_clean: 19 | override_dh_auto_build: 20 | override_dh_auto_install: 21 | -------------------------------------------------------------------------------- /drivers/pwm/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | # Uncomment this to turn on verbose mode. 5 | # export DH_VERBOSE=1 6 | 7 | VERSION=$(shell dpkg-parsechangelog | grep '^Version:' | cut -d' ' -f2 | cut -d- -f1) 8 | 9 | %: 10 | dh $@ --with dkms 11 | 12 | override_dh_install: 13 | dh_install *.c Makefile usr/src/pwm-soft-$(VERSION)/ 14 | 15 | override_dh_dkms: 16 | dh_dkms -V $(VERSION) 17 | 18 | override_dh_auto_clean: 19 | override_dh_auto_build: 20 | override_dh_auto_install: 21 | -------------------------------------------------------------------------------- /drivers/leds/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | # Uncomment this to turn on verbose mode. 5 | # export DH_VERBOSE=1 6 | 7 | VERSION=$(shell dpkg-parsechangelog | grep '^Version:' | cut -d' ' -f2 | cut -d- -f1) 8 | 9 | %: 10 | dh $@ --with dkms 11 | 12 | override_dh_install: 13 | dh_install *.c Makefile usr/src/leds-ktd202x-$(VERSION)/ 14 | 15 | override_dh_dkms: 16 | dh_dkms -V $(VERSION) 17 | 18 | override_dh_auto_clean: 19 | override_dh_auto_build: 20 | override_dh_auto_install: 21 | -------------------------------------------------------------------------------- /drivers/pwm/debian/pwm-soft-dkms.udev: -------------------------------------------------------------------------------- 1 | DEVPATH=="/class/pwm-soft", ACTION=="add", RUN+="/bin/chgrp -R gpio /sys%p", RUN+="/bin/chmod -R ug+rw /sys%p" 2 | DEVPATH=="/class/pwm-soft", ACTION=="change", ENV{TRIGGER}!="none", RUN+="/bin/chgrp -R gpio /sys%p", RUN+="/bin/chmod -R ug+rw /sys%p" 3 | SUBSYSTEM=="pwm-soft", ACTION=="add", RUN+="/bin/chgrp -R gpio /sys%p", RUN+="/bin/chmod -R ug+rw /sys%p" 4 | SUBSYSTEM=="pwm-soft", ACTION=="change", ENV{TRIGGER}!="none", RUN+="/bin/chgrp -R gpio /sys%p", RUN+="/bin/chmod -R ug+rw /sys%p" 5 | -------------------------------------------------------------------------------- /drivers/vision/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | # Uncomment this to turn on verbose mode. 5 | # export DH_VERBOSE=1 6 | 7 | VERSION=$(shell dpkg-parsechangelog | grep '^Version:' | cut -d' ' -f2 | cut -d- -f1) 8 | 9 | %: 10 | dh $@ --with dkms 11 | 12 | override_dh_install: 13 | dh_install *.c *.h Makefile usr/src/aiy-vision-$(VERSION)/ 14 | 15 | override_dh_dkms: 16 | dh_dkms -V $(VERSION) 17 | 18 | override_dh_auto_clean: 19 | override_dh_auto_build: 20 | override_dh_auto_install: 21 | -------------------------------------------------------------------------------- /src/examples/buzzer/laughing.track: -------------------------------------------------------------------------------- 1 | title Bwahahaha! 2 | speed 4 3 | order 0 4 | end 5 | 6 | pattern 7 | E5 glis 1 5 8 | glis 1 5 9 | glis 1 5 10 | 11 | noff 12 | 13 | D5 glis 1 3 14 | glis 1 3 15 | glis 1 3 16 | noff 17 | 18 | C5 glis 1 3 19 | glis 1 3 20 | glis 1 3 21 | noff 22 | 23 | B4 glis 1 3 24 | glis 1 3 25 | glis 1 3 26 | noff 27 | 28 | A4 glis 1 3 29 | glis 1 3 30 | glis 1 3 31 | noff 32 | 33 | G4 glis 1 3 34 | glis 1 3 35 | glis 1 3 36 | stop 37 | end 38 | -------------------------------------------------------------------------------- /drivers/sound/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | # Uncomment this to turn on verbose mode. 5 | # export DH_VERBOSE=1 6 | 7 | VERSION=$(shell dpkg-parsechangelog | grep '^Version:' | cut -d' ' -f2 | cut -d- -f1) 8 | 9 | %: 10 | dh $@ --with dkms 11 | 12 | override_dh_install: 13 | dh_install *.h *.c Makefile usr/src/aiy-voicebonnet-soundcard-$(VERSION)/ 14 | 15 | override_dh_dkms: 16 | dh_dkms -V $(VERSION) 17 | 18 | override_dh_auto_clean: 19 | override_dh_auto_build: 20 | override_dh_auto_install: 21 | -------------------------------------------------------------------------------- /src/examples/buzzer/sadtrombone.track: -------------------------------------------------------------------------------- 1 | title Sad trombone 2 | speed 5 3 | order 0 4 | end 5 | 6 | pattern 7 | E3 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | noff 18 | d3 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | noff 29 | D3 glis -1 1 30 | glis 1 1 31 | glis -1 1 32 | glis 1 1 33 | glis -1 1 34 | glis 1 1 35 | glis -1 1 36 | glis 1 1 37 | glis -1 1 38 | glis 1 1 39 | glis -1 1 40 | glis 1 1 41 | glis -1 1 42 | glis 1 1 43 | glis -1 1 44 | glis 1 1 45 | stop 46 | end 47 | -------------------------------------------------------------------------------- /drivers/.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | # NOTE! Don't add files that are generated in specific 3 | # subdirectories here. Add them in the ".gitignore" file 4 | # in that subdirectory instead. 5 | # 6 | # NOTE! Please use 'git ls-files -i --exclude-standard' 7 | # command after changing this file, to see if there are 8 | # any tracked files which get ignored after the change. 9 | # 10 | # Normal rules 11 | # 12 | *.ko 13 | *.ko.cmd 14 | *.o 15 | *.o.cmd 16 | *.o.d 17 | *.mod 18 | *.mod.cmd 19 | *.mod.c 20 | *.deb 21 | *.buildinfo 22 | *.changes 23 | *.order 24 | Module.symvers 25 | -------------------------------------------------------------------------------- /src/aiy/vision/streaming/assets/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | AIY VisionKit Live Stream 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 |
13 | 14 | 15 | -------------------------------------------------------------------------------- /scripts/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # git hook to ensure code style 3 | # ln ../../scripts/pre-commit .git/hooks/ 4 | 5 | which >/dev/null autopep8 || { echo "please install autopep8"; exit 1; } 6 | files=$(git diff --name-only --staged --diff-filter=ACMRTUXB | egrep "*.py$") 7 | 8 | if test -n "$files"; then 9 | diff=$(autopep8 --max-line-length=100 --diff $files) 10 | if [[ -n "${diff}" ]]; then 11 | echo 12 | autopep8 --max-line-length=100 --diff $files 13 | echo 14 | echo "To fix run: autopep8 --max-line-length=100 -i $files" 15 | exit 1 16 | fi 17 | fi 18 | -------------------------------------------------------------------------------- /drivers/overlays/voice/debian/aiy-overlay-voice.postinst: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #!/bin/sh 4 | 5 | set -e 6 | 7 | #DEBHELPER# 8 | 9 | case "$1" in 10 | configure) 11 | if [ -z "$2" ]; then 12 | dtc -W no-unit_address_vs_reg -@ -O dtb -o /boot/overlays/aiy-voicebonnet.dtbo \ 13 | /opt/aiy/overlay-voice/aiy-voicebonnet-overlay.dts 14 | fi 15 | ;; 16 | abort-upgrade|abort-remove|abort-deconfigure) 17 | ;; 18 | 19 | *) 20 | echo "postinst called with unknown argument \`$1'" >&2 21 | exit 0 22 | ;; 23 | esac 24 | -------------------------------------------------------------------------------- /drivers/overlays/vision/debian/aiy-overlay-vision.postinst: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #!/bin/sh 4 | 5 | set -e 6 | 7 | #DEBHELPER# 8 | 9 | case "$1" in 10 | configure) 11 | if [ -z "$2" ]; then 12 | dtc -W no-unit_address_vs_reg -@ -O dtb -o /boot/overlays/aiy-visionbonnet.dtbo \ 13 | /opt/aiy/overlay-vision/aiy-visionbonnet-overlay.dts 14 | fi 15 | ;; 16 | abort-upgrade|abort-remove|abort-deconfigure) 17 | ;; 18 | 19 | *) 20 | echo "postinst called with unknown argument \`$1'" >&2 21 | exit 0 22 | ;; 23 | esac 24 | -------------------------------------------------------------------------------- /drivers/leds/debian/changelog: -------------------------------------------------------------------------------- 1 | leds-ktd202x-dkms (1.2-2) stable; urgency=medium 2 | 3 | * Fix compatibility with 6.1.x kernel. 4 | 5 | -- Gunjan Gupta Sat, 11 Mar 2023 23:39:40 +0530 6 | 7 | leds-ktd202x-dkms (1.2-1) stable; urgency=medium 8 | 9 | * Fix swapped tfall and trise register names. 10 | 11 | -- Dmitry Kovalev Wed, 09 Jan 2019 11:50:40 -0800 12 | 13 | leds-ktd202x-dkms (1.1-2) stable; urgency=medium 14 | 15 | * Proper debian package based on dh_dkms. 16 | 17 | -- Dmitry Kovalev Thu, 30 Aug 2018 15:28:07 -0700 18 | -------------------------------------------------------------------------------- /src/examples/vision/joy/joy_detection_demo.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=AIY Joy Detection Demo 3 | Requires=dev-vision_spicomm.device 4 | After=dev-vision_spicomm.device 5 | Wants=aiy-board-info.service 6 | After=aiy-board-info.service 7 | 8 | [Service] 9 | Type=simple 10 | Restart=no 11 | User=pi 12 | Environment=AIY_BOARD_NAME=AIY-Board 13 | EnvironmentFile=-/run/aiy-board-info 14 | ExecStart=/usr/bin/python3 /home/pi/AIY-projects-python/src/examples/vision/joy/joy_detection_demo.py --enable_streaming --mdns_name "${AIY_BOARD_NAME}" --blink_on_error 15 | 16 | [Install] 17 | WantedBy=multi-user.target 18 | -------------------------------------------------------------------------------- /drivers/vision/debian/changelog: -------------------------------------------------------------------------------- 1 | aiy-vision-dkms (1.2-1.1) stable; urgency=medium 2 | 3 | * Non-maintainer upload. 4 | * Fix compatibility with 6.1 kernel. 5 | 6 | -- Gunjan Gupta Mon, 13 Mar 2023 01:47:00 -0530 7 | 8 | aiy-vision-dkms (1.2-1) stable; urgency=medium 9 | 10 | * Proper order of cleanup calls in visionbonnet_destroy(). 11 | 12 | -- Dmitry Kovalev Mon, 16 Nov 2020 01:36:00 -0800 13 | 14 | aiy-vision-dkms (1.1-1) stable; urgency=medium 15 | 16 | * Proper debian package based on dh_dkms. 17 | 18 | -- Dmitry Kovalev Thu, 30 Aug 2018 15:28:07 -0700 19 | -------------------------------------------------------------------------------- /src/aiy/vision/streaming/proto/messages.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message ServerBound { 4 | oneof message { 5 | StreamControl stream_control = 1; 6 | } 7 | } 8 | 9 | message StreamControl { 10 | bool enabled = 1; 11 | } 12 | 13 | message ClientBound { 14 | oneof message { 15 | Start start = 1; 16 | Stop stop = 2; 17 | Video video = 3; 18 | Overlay overlay = 4; 19 | } 20 | uint64 timestamp_us = 10; 21 | } 22 | 23 | message Start { 24 | uint32 width = 1; 25 | uint32 height = 2; 26 | } 27 | 28 | message Stop { 29 | } 30 | 31 | message Video { 32 | bytes data = 1; 33 | } 34 | 35 | message Overlay { 36 | string svg = 1; 37 | } 38 | -------------------------------------------------------------------------------- /drivers/aiy/debian/aiy-dkms.dkms: -------------------------------------------------------------------------------- 1 | PACKAGE_VERSION="#MODULE_VERSION#" 2 | PACKAGE_NAME="aiy" 3 | CLEAN="make clean" 4 | 5 | BUILT_MODULE_NAME[0]="aiy-io-i2c" 6 | DEST_MODULE_LOCATION[0]="/kernel/drivers/mfd" 7 | BUILT_MODULE_LOCATION[0]="./aiy/mfd" 8 | 9 | BUILT_MODULE_NAME[1]="pwm-aiy-io" 10 | DEST_MODULE_LOCATION[1]="/kernel/drivers/pwm" 11 | BUILT_MODULE_LOCATION[1]="./aiy/pwm" 12 | 13 | BUILT_MODULE_NAME[2]="gpio-aiy-io" 14 | DEST_MODULE_LOCATION[2]="/kernel/drivers/gpio" 15 | BUILT_MODULE_LOCATION[2]="./aiy/gpio" 16 | 17 | BUILT_MODULE_NAME[3]="aiy-adc" 18 | DEST_MODULE_LOCATION[3]="/kernel/drivers/iio/adc" 19 | BUILT_MODULE_LOCATION[3]="./aiy/iio/adc" 20 | 21 | AUTOINSTALL="yes" 22 | -------------------------------------------------------------------------------- /drivers/overlays/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o xtrace 4 | set -o errexit 5 | 6 | readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 7 | readonly DTC_ARGS="-W no-unit_address_vs_reg -@ -O dtb" 8 | 9 | pushd ${SCRIPT_DIR}/vision 10 | dtc ${DTC_ARGS} -o /boot/overlays/aiy-visionbonnet.dtbo aiy-visionbonnet-overlay.dts 11 | dtc ${DTC_ARGS} -o /boot/overlays/aiy-leds-vision.dtbo aiy-leds-vision-overlay.dts 12 | dtc ${DTC_ARGS} -o /boot/overlays/aiy-io-vision.dtbo aiy-io-vision-overlay.dts 13 | popd 14 | 15 | pushd ${SCRIPT_DIR}/voice 16 | dtc ${DTC_ARGS} -o /boot/overlays/aiy-voicebonnet.dtbo aiy-voicebonnet-overlay.dts 17 | dtc ${DTC_ARGS} -o /boot/overlays/aiy-leds-voice.dtbo aiy-leds-voice-overlay.dts 18 | dtc ${DTC_ARGS} -o /boot/overlays/aiy-io-voice.dtbo aiy-io-voice-overlay.dts 19 | popd 20 | -------------------------------------------------------------------------------- /drivers/aiy/debian/changelog: -------------------------------------------------------------------------------- 1 | aiy-dkms (2.0-1.2) UNRELEASED; urgency=medium 2 | 3 | * Non-maintainer upload. 4 | * Update drivers to support 6.1 kernel 5 | 6 | -- Gunjan Gupta Sun, 12 Mar 2023 20:51:32 +0530 7 | 8 | aiy-dkms (2.0-1.1) UNRELEASED; urgency=medium 9 | 10 | * Non-maintainer upload. 11 | * Update drivers to support 5.15 kernel 12 | 13 | -- Gunjan Gupta Fri, 09 Dec 2022 12:56:32 +0000 14 | 15 | aiy-dkms (2.0-1) stable; urgency=medium 16 | 17 | * Update drivers to support 4.19 kernel. 18 | 19 | -- Dmitry Kovalev Wed, 06 Nov 2019 16:36:24 -0800 20 | 21 | aiy-dkms (1.1-2) stable; urgency=medium 22 | 23 | * Proper debian package based on dh_dkms. 24 | 25 | -- Dmitry Kovalev Thu, 30 Aug 2018 15:28:07 -0700 26 | -------------------------------------------------------------------------------- /drivers/vision/aiy-vision.h: -------------------------------------------------------------------------------- 1 | #ifndef _STAGING_MYRIAD_AIY_VISION_H 2 | #define _STAGING_MYRIAD_AIY_VISION_H 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #define AIY_VISION_IOCTL_RESET _IO(0x89, 1) 9 | #define AIY_VISION_IOCTL_TRANSACT _IOWR(0x89, 3, usr_transaction_t) 10 | #define AIY_VISION_IOCTL_TRANSACT_MMAP _IOWR(0x89, 4, usr_transaction_t) 11 | 12 | #define USR_FLAG_ONEWAY BIT(31) 13 | 14 | #define FLAG_ERROR BIT(0) 15 | #define FLAG_TIMEOUT BIT(1) 16 | #define FLAG_OVERFLOW BIT(2) 17 | #define FLAG_ACKED BIT(3) 18 | #define FLAG_RESPONSE BIT(4) 19 | 20 | typedef struct { 21 | uint32_t flags; 22 | uint32_t timeout_ms; 23 | uint32_t buffer_len_or_pgoff; 24 | uint32_t payload_len; 25 | } usr_transaction_t; 26 | 27 | #endif // _STAGING_MYRIAD_AIY_VISION_H 28 | -------------------------------------------------------------------------------- /src/examples/gpiozero/simple_button_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Example code that demonstrates using a standard pin along with a hat pin. 3 | 4 | The button uses a standard GPIO pin through the raspberry pi's memory mapped io, 5 | while the led uses the hat's sysfs driver. This implemenation difference is 6 | transparent to the user. 7 | 8 | The demo will light up the on board LED whenever the user presses the button. 9 | """ 10 | from gpiozero import Button 11 | from gpiozero import LED 12 | from aiy.pins import BUTTON_GPIO_PIN 13 | from aiy.pins import LED_1 14 | 15 | # Set up a gpiozero LED using the first onboard LED on the vision hat. 16 | led = LED(LED_1) 17 | # Set up a gpiozero Button using the button included with the vision hat. 18 | button = Button(BUTTON_GPIO_PIN) 19 | 20 | while True: 21 | if button.is_pressed: 22 | led.on() 23 | else: 24 | led.off() 25 | -------------------------------------------------------------------------------- /src/examples/vision/object_meter/install_services.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2018 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 18 | cp ${SCRIPT_DIR}/object_meter_demo.service /lib/systemd/system 19 | systemctl daemon-reload 20 | systemctl enable object_meter_demo.service 21 | -------------------------------------------------------------------------------- /src/examples/vision/joy/install-services.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2017 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 18 | 19 | ln -fs ${SCRIPT_DIR}/joy_detection_demo.service /lib/systemd/system/joy_detection_demo.service 20 | systemctl daemon-reload 21 | systemctl enable joy_detection_demo.service 22 | -------------------------------------------------------------------------------- /src/examples/gpiozero/servo_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Demonstrates simultaneous control of two servos on the hat. 3 | 4 | One servo uses the simple default configuration, the other servo is tuned to 5 | ensure the full range is reachable. 6 | """ 7 | 8 | from time import sleep 9 | from gpiozero import Servo 10 | from aiy.pins import PIN_A 11 | from aiy.pins import PIN_B 12 | 13 | # Create a default servo that will not be able to use quite the full range. 14 | simple_servo = Servo(PIN_A) 15 | # Create a servo with the custom values to give the full dynamic range. 16 | tuned_servo = Servo(PIN_B, min_pulse_width=.0005, max_pulse_width=.0019) 17 | 18 | # Move the Servos back and forth until the user terminates the example. 19 | while True: 20 | simple_servo.min() 21 | tuned_servo.max() 22 | sleep(1) 23 | simple_servo.mid() 24 | tuned_servo.mid() 25 | sleep(1) 26 | simple_servo.max() 27 | tuned_servo.min() 28 | sleep(1) 29 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='aiy-projects-python', 5 | version='1.4', 6 | description='AIY Python API', 7 | long_description='A set of Python APIs designed for the AIY Voice Kit and AIY Vision Kit, which help you build intelligent systems that can understand what they hear and see.', 8 | author='AIY Team', 9 | author_email='support-aiyprojects@google.com', 10 | url="https://aiyprojects.withgoogle.com/", 11 | project_urls={ 12 | 'GitHub: issues': 'https://github.com/viraniac/aiyprojects-raspbian/issues', 13 | 'GitHub: repo': 'https://github.com/viraniac/aiyprojects-raspbian', 14 | }, 15 | license='Apache 2', 16 | packages=find_packages('src'), 17 | package_dir={'': 'src'}, 18 | install_requires=[ 19 | 'gpiozero', 20 | 'protobuf>=3.6.1', 21 | 'picamera', 22 | 'Pillow', 23 | 'RPi.GPIO', 24 | ], 25 | python_requires='>=3.5.3', 26 | ) 27 | -------------------------------------------------------------------------------- /src/examples/gpiozero/bonnet_button.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Example code that demonstrates using a button connected through the hat. 3 | 4 | The button uses a hat pin through the sysfs driver illustrating the edge 5 | detection polling. 6 | 7 | The demo will light up the on board LED whenever PIN_D is drawn high. 8 | """ 9 | from signal import pause 10 | from gpiozero import Button 11 | from gpiozero import LED 12 | from aiy.pins import LED_1 13 | from aiy.pins import PIN_D 14 | 15 | 16 | # Set up a gpiozero LED using the first onboard LED on the vision hat. 17 | led = LED(LED_1) 18 | # Set up a gpiozero Button using the 4th pin on the vision hat expansion. 19 | button = Button(PIN_D) 20 | 21 | # When the button is pressed, call the led.on() function (turn the led on) 22 | button.when_pressed = led.on 23 | # When the button is released, call the led.off() function (turn the led off) 24 | button.when_released = led.off 25 | 26 | # Wait for the user to kill the example. 27 | pause() 28 | -------------------------------------------------------------------------------- /drivers/sound/rl6231.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0-only */ 2 | /* 3 | * rl6231.h - RL6231 class device shared support 4 | * 5 | * Copyright 2014 Realtek Semiconductor Corp. 6 | * 7 | * Author: Oder Chiou 8 | */ 9 | 10 | #ifndef __RL6231_H__ 11 | #define __RL6231_H__ 12 | 13 | #define RL6231_PLL_INP_MAX 50000000 14 | #define RL6231_PLL_INP_MIN 256000 15 | #define RL6231_PLL_N_MAX 0x1ff 16 | #define RL6231_PLL_K_MAX 0x1f 17 | #define RL6231_PLL_M_MAX 0xf 18 | 19 | struct rl6231_pll_code { 20 | bool m_bp; /* Indicates bypass m code or not. */ 21 | bool k_bp; /* Indicates bypass k code or not. */ 22 | int m_code; 23 | int n_code; 24 | int k_code; 25 | }; 26 | 27 | int rl6231_calc_dmic_clk(int rate); 28 | int rl6231_pll_calc(const unsigned int freq_in, 29 | const unsigned int freq_out, struct rl6231_pll_code *pll_code); 30 | int rl6231_get_clk_info(int sclk, int rate); 31 | int rl6231_get_pre_div(struct regmap *map, unsigned int reg, int sft); 32 | 33 | #endif /* __RL6231_H__ */ 34 | -------------------------------------------------------------------------------- /src/examples/gpiozero/button_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Example code that demonstrates using a standard pin along with a hat pin. 3 | 4 | The button uses a standard GPIO pin through the raspberry pi's memory mapped io, 5 | while the led uses the hat's sysfs driver. This implemenation difference is 6 | transparent to the user. 7 | 8 | The demo will light up the on board LED whenever the user presses the button. 9 | """ 10 | from signal import pause 11 | from gpiozero import Button 12 | from gpiozero import LED 13 | from aiy.pins import BUTTON_GPIO_PIN 14 | from aiy.pins import LED_1 15 | 16 | # Set up a gpiozero LED using the first onboard LED on the vision hat. 17 | led = LED(LED_1) 18 | # Set up a gpiozero Button using the button included with the vision hat. 19 | button = Button(BUTTON_GPIO_PIN) 20 | 21 | # When the button is pressed, call the led.on() function (turn the led on) 22 | button.when_pressed = led.on 23 | # When the button is released, call the led.off() function (turn the led off) 24 | button.when_released = led.off 25 | 26 | # Wait for the user to kill the example. 27 | pause() 28 | -------------------------------------------------------------------------------- /drivers/aiy/Makefile: -------------------------------------------------------------------------------- 1 | MDIR := aiy 2 | 3 | KVER ?= $(shell uname -r) 4 | KDIR ?= /lib/modules/$(KVER)/build 5 | 6 | export CONFIG_AIY_IO_I2C = m 7 | export CONFIG_GPIO_AIY_IO = m 8 | export CONFIG_PWM_AIY_IO = m 9 | export CONFIG_AIY_ADC = m 10 | export CONFIG_AIY_VISION = m 11 | 12 | ifneq ($(KERNELRELEASE),) 13 | 14 | ifeq ($(CONFIG_AIY_IO_I2C), m) 15 | KBUILD_CFLAGS += -DCONFIG_AIY_IO_I2C 16 | endif 17 | ifeq ($(CONFIG_GPIO_AIY_IO), m) 18 | KBUILD_CFLAGS += -DCONFIG_GPIO_AIY_IO 19 | endif 20 | ifeq ($(CONFIG_PWN_AIY_IO), m) 21 | KBUILD_CFLAGS += -DCONFIG_PWM_AIY_IO 22 | endif 23 | ifeq ($(CONFIG_AIY_ADC), m) 24 | KBUILD_CFLAGS += -DCONFIG_AIY_ADC 25 | endif 26 | ifeq ($(CONFIG_AIY_VISION), m) 27 | KBUILD_CFLAGS += -DCONFIG_AIY_VISION 28 | endif 29 | 30 | 31 | 32 | obj-y := $(MDIR)/ 33 | 34 | else 35 | 36 | all: modules 37 | 38 | modules: 39 | $(MAKE) -C $(KDIR) M="$(CURDIR)" modules 40 | 41 | install: modules 42 | $(MAKE) INSTALL_MOD_PATH=$(DESTDIR) INSTALL_MOD_DIR=$(MDIR) \ 43 | -C $(KDIR) M="$(CURDIR)" modules_install 44 | 45 | clean: 46 | $(MAKE) -C $(KDIR) M="$(CURDIR)" clean 47 | 48 | help: 49 | $(MAKE) -C $(KDIR) M="$(CURDIR)" help 50 | 51 | endif 52 | -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | /* Custom styles for AIY Projects reference */ 2 | 3 | /* code font more like aiyprojects.withgoogle.com */ 4 | code, 5 | .rst-content tt, 6 | .rst-content code { 7 | background: #f6f6f6; 8 | border: none; 9 | padding: .1em .3em; 10 | font-size: 80%; 11 | font-weight: bold; 12 | color: #263238; 13 | } 14 | 15 | .rst-content tt.literal, 16 | .rst-content tt.literal, 17 | .rst-content code.literal { 18 | color: inherit; 19 | } 20 | 21 | /* Force-remove bottom-margin from parameter list */ 22 | .rst-content table.field-list td ul.last { 23 | margin-bottom: 0 !important; 24 | } 25 | 26 | /* Remove bottom padding from table around parameter list */ 27 | .rst-content table.field-list td { 28 | padding-bottom: 0; 29 | vertical-align: top; 30 | } 31 | 32 | /* floating elements (images) */ 33 | .attempt-right { 34 | float: right; 35 | margin: 0 0 40px 40px; 36 | max-width: calc((100% - 40px)/2); 37 | } 38 | 39 | /* Smallish screens */ 40 | @media screen and (max-width: 1000px) { 41 | .attempt-right { 42 | display: block; 43 | float: none; 44 | margin: 16px 0; 45 | max-width: 100%; 46 | } 47 | } -------------------------------------------------------------------------------- /src/aiy/vision/models/utils.py: -------------------------------------------------------------------------------- 1 | """Set of reusable utilities to work with AIY models.""" 2 | 3 | import os 4 | 5 | 6 | def _path(filename): 7 | path = os.environ.get('VISION_BONNET_MODELS_PATH', '/opt/aiy/models') 8 | return os.path.join(path, filename) 9 | 10 | 11 | def load_compute_graph(filename): 12 | with open(_path(filename), 'rb') as f: 13 | return f.read() 14 | 15 | def load_labels(filename): 16 | def split(line): 17 | return tuple(word.strip() for word in line.split(',')) 18 | 19 | with open(_path(filename), encoding='utf-8') as f: 20 | return tuple(split(line) for line in f) 21 | 22 | def load_ssd_anchors(filename): 23 | def split(line): 24 | return tuple(float(word.strip()) for word in line.split(' ')) 25 | 26 | with open(_path(filename), encoding='utf-8') as f: 27 | return tuple(split(line) for line in f) 28 | 29 | 30 | def shape_tuple(shape): 31 | return (shape.batch, shape.height, shape.width, shape.depth) 32 | 33 | def reshape(array, width): 34 | assert len(array) % width == 0 35 | height = len(array) // width 36 | return [array[i * width:(i + 1) * width] for i in range(height)] 37 | -------------------------------------------------------------------------------- /src/examples/buzzer/buzzer_tracker_demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """A demo of the Piezo Buzzer.""" 17 | 18 | import sys 19 | 20 | from aiy.toneplayer import Note 21 | from aiy.trackplayer import NoteOff, Arpeggio, StopPlaying, TrackPlayer, TrackLoader 22 | 23 | 24 | def main(): 25 | if len(sys.argv) == 1: 26 | print("Usage: buzzer_tracker_demo.py ") 27 | return 28 | 29 | loader = TrackLoader(22, sys.argv[1], debug=True) 30 | player = loader.load() 31 | player.play() 32 | 33 | 34 | if __name__ == '__main__': 35 | main() 36 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | This directory holds the source files required to build the AIY Python reference with Sphinx. 4 | 5 | You can see the generated files at [aiyprojects.readthedocs.io](https://aiyprojects.readthedocs.io/). 6 | 7 | If you've downloaded the [aiyprojects-raspbian](https://github.com/google/aiyprojects-raspbian) 8 | repository, you can build these docs locally with the `make docs` command. Of course, this requires 9 | that you install Sphinx and other Python dependencies: 10 | 11 | # We require Python3, so if that's not your default, first start a virtual environment: 12 | python3 -m venv ~/.aiy_venv 13 | source ~/.aiy_venv/bin/activate 14 | 15 | # Move to the aiyprojects-raspbian repo root... 16 | 17 | # Install the dependencies: 18 | python3 -m pip install -r docs/requirements.txt 19 | 20 | # Build the docs: 21 | make docs 22 | 23 | The results are output in `docs/_build/html/`. 24 | 25 | **Note:** These output files should not be committed to this repository. We use readthedocs.org 26 | to generate the HTML documentation directly from GitHub—this repo holds the *source files 27 | only*, not the built files. 28 | 29 | For more information about the syntax in these RST files, see the [reStructuredText documentation]( 30 | http://www.sphinx-doc.org/en/master/usage/restructuredtext/index.html). 31 | -------------------------------------------------------------------------------- /drivers/sound/debian/changelog: -------------------------------------------------------------------------------- 1 | aiy-voicebonnet-soundcard-dkms (3.0-1.3) UNRELEASED; urgency=medium 2 | 3 | * Non-maintainer upload. 4 | * Fix UCM2 configuration to enable speaker on boot 5 | 6 | -- Gunjan Gupta Sun, 30 Mar 2023 18:22:55 +0530 7 | 8 | aiy-voicebonnet-soundcard-dkms (3.0-1.2) UNRELEASED; urgency=medium 9 | 10 | * Non-maintainer upload. 11 | * Update driver to support 6.1 kernel 12 | 13 | -- Gunjan Gupta Sun, 12 Mar 2023 21:39:55 +0530 14 | 15 | aiy-voicebonnet-soundcard-dkms (3.0-1.1) UNRELEASED; urgency=medium 16 | 17 | * Non-maintainer upload. 18 | * Update driver to support 5.15 kernel 19 | 20 | -- Gunjan Gupta Fri, 09 Dec 2022 09:53:55 +0000 21 | 22 | aiy-voicebonnet-soundcard-dkms (3.0-1) stable; urgency=medium 23 | 24 | * Update driver to support 5.4 kernel. 25 | 26 | -- Dmitry Kovalev Mon, 16 Nov 2020 01:36:00 -0800 27 | 28 | aiy-voicebonnet-soundcard-dkms (2.0-1) stable; urgency=medium 29 | 30 | * Update driver to support 4.19 kernel. 31 | 32 | -- Dmitry Kovalev Wed, 06 Nov 2019 16:36:24 -0800 33 | 34 | aiy-voicebonnet-soundcard-dkms (1.0-2) stable; urgency=medium 35 | 36 | * Proper debian package based on dh_dkms. 37 | 38 | -- Dmitry Kovalev Thu, 30 Aug 2018 15:28:07 -0700 39 | -------------------------------------------------------------------------------- /src/examples/buzzer/buzzer_demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """A demo of the Piezo Buzzer.""" 17 | 18 | import aiy.toneplayer 19 | 20 | 21 | def main(): 22 | tetris_theme = [ 23 | 'E5q', 24 | 'Be', 25 | 'C5e', 26 | 'D5e', 27 | 'E5s', 28 | 'D5s', 29 | 'C5s', 30 | 'Be', 31 | 'Bs', 32 | 'Aq', 33 | 'Ae', 34 | 'C5e', 35 | 'E5q', 36 | 'D5e', 37 | 'C5e', 38 | 'Bq', 39 | 'Be', 40 | 'C5e', 41 | 'D5q', 42 | 'E5q', 43 | 'C5q', 44 | 'Aq', 45 | 'Aq', 46 | ] 47 | 48 | player = aiy.toneplayer.TonePlayer(22) 49 | player.play(*tetris_theme) 50 | 51 | 52 | if __name__ == '__main__': 53 | main() 54 | -------------------------------------------------------------------------------- /src/examples/vision/dish_classification.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Dish classification library demo.""" 16 | 17 | import argparse 18 | from PIL import Image 19 | 20 | from aiy.vision.inference import ImageInference 21 | from aiy.vision.models import dish_classification 22 | 23 | def main(): 24 | parser = argparse.ArgumentParser() 25 | parser.add_argument('--input', '-i', dest='input', required=True) 26 | args = parser.parse_args() 27 | 28 | with ImageInference(dish_classification.model()) as inference: 29 | image = Image.open(args.input) 30 | classes = dish_classification.get_classes( 31 | inference.run(image), top_k=5, threshold=0.1) 32 | for i, (label, score) in enumerate(classes): 33 | print('Result %d: %s (prob=%f)' % (i, label, score)) 34 | 35 | 36 | if __name__ == '__main__': 37 | main() 38 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | AIY Projects API reference 2 | ========================== 3 | 4 | This is the Python API reference for the AIY Projects library, which is 5 | provided with the Vision Kit and Voice Kit projects. 6 | 7 | For more information about the Vision and Voice kits, including 8 | assembly guides and makers guides, go to 9 | `aiyprojects.withgoogle.com `_. 10 | 11 | 12 | .. toctree:: 13 | :caption: Common APIs 14 | 15 | aiy.board 16 | aiy.leds 17 | aiy.pins 18 | 19 | .. toctree:: 20 | :caption: Vision Kit APIs 21 | :maxdepth: 2 22 | 23 | Overview 24 | aiy.toneplayer 25 | aiy.trackplayer 26 | aiy.vision.annotator 27 | aiy.vision.inference 28 | aiy.vision.models 29 | 30 | .. toctree:: 31 | :caption: Voice Kit APIs 32 | :maxdepth: 2 33 | 34 | Overview 35 | aiy.voice.audio 36 | aiy.voice.tts 37 | 38 | .. toctree:: 39 | :caption: Other docs 40 | :maxdepth: 2 41 | 42 | Source code (GitHub) 43 | Change log (GitHub) 44 | Software setup (GitHub) 45 | Contribution guide (GitHub) 46 | 47 | API indices 48 | -------------- 49 | 50 | * :ref:`Full index ` 51 | * :ref:`Module index ` 52 | -------------------------------------------------------------------------------- /src/examples/vision/face_camera_trigger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Trigger PiCamera when face is detected.""" 16 | 17 | from aiy.vision.inference import CameraInference 18 | from aiy.vision.models import face_detection 19 | 20 | from picamera import PiCamera 21 | 22 | 23 | def main(): 24 | with PiCamera() as camera: 25 | # Configure camera 26 | camera.resolution = (1640, 922) # Full Frame, 16:9 (Camera v2) 27 | camera.start_preview() 28 | 29 | # Do inference on VisionBonnet 30 | with CameraInference(face_detection.model()) as inference: 31 | for result in inference.run(): 32 | if len(face_detection.get_faces(result)) >= 1: 33 | camera.capture('faces.jpg') 34 | break 35 | 36 | # Stop preview 37 | camera.stop_preview() 38 | 39 | 40 | if __name__ == '__main__': 41 | main() 42 | -------------------------------------------------------------------------------- /src/tests/dish_classification_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import unittest 15 | 16 | from aiy.vision.inference import ImageInference 17 | from aiy.vision.models import dish_classification 18 | from .test_util import TestImage 19 | 20 | 21 | class DishClassificationTest(unittest.TestCase): 22 | 23 | def testHotdog(self): 24 | with TestImage('hotdog.jpg') as image: 25 | with ImageInference(dish_classification.model()) as inference: 26 | classes = dish_classification.get_classes(inference.run(image)) 27 | label, score = classes[0] 28 | self.assertEqual('Hot dog', label) 29 | self.assertAlmostEqual(score, 0.744, delta=0.001) 30 | 31 | label, score = classes[1] 32 | self.assertEqual('Lobster roll', label) 33 | self.assertAlmostEqual(score, 0.119, delta=0.001) 34 | 35 | 36 | if __name__ == '__main__': 37 | unittest.main() 38 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We'd love to accept your patches and contributions to this project. 4 | There are just a few small guidelines you need to follow. 5 | 6 | ## Scope of Contributions 7 | 8 | This project consists of the support libraries (audio, gRPC, etc) required for 9 | AIY Projects, as well as simple examples to experiment with and build upon. 10 | 11 | Please limit pull requests to bug fixes or improvements to code or documentation 12 | clarity. If you've added new examples and you'd like to publish your fork for 13 | others to use, you can post on [hackster.io] or other channels. 14 | 15 | ## Contributor License Agreement 16 | 17 | Contributions to this project must be accompanied by a Contributor License 18 | Agreement. You (or your employer) retain the copyright to your contribution, 19 | this simply gives us permission to use and redistribute your contributions as 20 | part of the project. Head over to to see 21 | your current agreements on file or to sign a new one. 22 | 23 | You generally only need to submit a CLA once, so if you've already submitted one 24 | (even if it was for a different project), you probably don't need to do it 25 | again. 26 | 27 | ## Code reviews 28 | 29 | All submissions, including submissions by project members, require review. We 30 | use GitHub pull requests for this purpose. Consult [GitHub Help] for more 31 | information on using pull requests. 32 | 33 | [hackster.io]: https://www.hackster.io/ 34 | [GitHub Help]: https://help.github.com/articles/about-pull-requests/ -------------------------------------------------------------------------------- /src/tests/test_util.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import contextlib 15 | import os 16 | import unittest 17 | 18 | from PIL import Image 19 | 20 | def test_image_path(name): 21 | p = os.path.join(os.path.dirname(__file__), 'images', name) 22 | return os.path.abspath(p) 23 | 24 | 25 | @contextlib.contextmanager 26 | def TestImageFile(name): 27 | with open(test_image_path(name), 'rb') as f: 28 | yield f 29 | 30 | 31 | @contextlib.contextmanager 32 | def TestImage(name): 33 | with TestImageFile(name) as f: 34 | image = Image.open(f) 35 | try: 36 | yield image 37 | finally: 38 | image.close() 39 | 40 | 41 | def define_test_case(scope, test_class, *test_args): 42 | def init(self, *args, **kwargs): 43 | unittest.TestCase.__init__(self, *args, **kwargs) 44 | test_class.__init__(self, *test_args) 45 | name = 'Test%s' % str(test_args) 46 | scope[name] = type(name, (test_class, unittest.TestCase), {'__init__': init}) 47 | -------------------------------------------------------------------------------- /docs/aiy.vision.models.rst: -------------------------------------------------------------------------------- 1 | aiy.vision.models 2 | ================= 3 | 4 | A collection of modules that perform ML inferences with specific types of 5 | image classification and object detection models. 6 | 7 | Each of these modules has a corresponding sample app in 8 | :github:`src/examples/vision`. 9 | Also see the instructions to `run the models with the Vision Kit 10 | `_. 11 | 12 | 13 | aiy.vision.models.dish\_classification 14 | -------------------------------------- 15 | 16 | .. automodule:: aiy.vision.models.dish_classification 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | 21 | aiy.vision.models.dish\_detection 22 | --------------------------------- 23 | 24 | .. automodule:: aiy.vision.models.dish_detection 25 | :members: 26 | :undoc-members: 27 | :show-inheritance: 28 | 29 | aiy.vision.models.face\_detection 30 | --------------------------------- 31 | 32 | .. automodule:: aiy.vision.models.face_detection 33 | :members: 34 | :undoc-members: 35 | :show-inheritance: 36 | 37 | aiy.vision.models.image\_classification 38 | --------------------------------------- 39 | 40 | .. automodule:: aiy.vision.models.image_classification 41 | :members: 42 | :undoc-members: 43 | :show-inheritance: 44 | 45 | aiy.vision.models.inaturalist\_classification 46 | --------------------------------------------- 47 | 48 | .. automodule:: aiy.vision.models.inaturalist_classification 49 | :members: 50 | :undoc-members: 51 | :show-inheritance: 52 | 53 | aiy.vision.models.object\_detection 54 | ----------------------------------- 55 | 56 | .. automodule:: aiy.vision.models.object_detection 57 | :members: 58 | :undoc-members: 59 | :show-inheritance: 60 | -------------------------------------------------------------------------------- /src/tests/dish_detection_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import unittest 15 | 16 | from aiy.vision.inference import ImageInference 17 | from aiy.vision.models import dish_detection 18 | from .test_util import TestImage 19 | 20 | 21 | class DishDetectionTest(unittest.TestCase): 22 | 23 | def testHotdog(self): 24 | with TestImage('hotdog.jpg') as image: 25 | with ImageInference(dish_detection.model()) as inference: 26 | dishes = dish_detection.get_dishes(inference.run(image), top_k=3, threshold=0.1) 27 | self.assertEqual(1, len(dishes)) 28 | dish = dishes[0] 29 | 30 | self.assertEqual((417.0, 51.0, 2438.0, 2388.0), dish.bounding_box) 31 | self.assertEqual(2, len(dish.sorted_scores)) 32 | 33 | label, score = dish.sorted_scores[0] 34 | self.assertEqual('Hot dog', label) 35 | self.assertAlmostEqual(0.223, score, delta=0.001) 36 | 37 | label, score = dish.sorted_scores[1] 38 | self.assertEqual('Bento', label) 39 | self.assertAlmostEqual(0.152, score, delta=0.001) 40 | 41 | if __name__ == '__main__': 42 | unittest.main() 43 | -------------------------------------------------------------------------------- /src/aiy/vision/streaming/assets/broadway/LICENSE: -------------------------------------------------------------------------------- 1 | Broadway 2 | https://github.com/mbebenita/Broadway 3 | 4 | Copyright (c) 2011, Project Authors (see AUTHORS file) 5 | All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 11 | * Neither the names of the Project Authors nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | 15 | -- 16 | 17 | The 3-clause BSD above applies to all code except for code originating 18 | from the Android project (the .cpp files in Avc/). Those files are under 19 | the Android project's Apache 2.0 license. 20 | -------------------------------------------------------------------------------- /src/examples/vision/face_detection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Face detection library demo. 16 | 17 | - Takes an input image and tries to detect faces. 18 | - Draws bounding boxes around detected objects. 19 | - Saves an image with bounding boxes around detected objects. 20 | """ 21 | import argparse 22 | 23 | from PIL import Image, ImageDraw 24 | 25 | from aiy.vision.inference import ImageInference 26 | from aiy.vision.models import face_detection 27 | 28 | def main(): 29 | parser = argparse.ArgumentParser() 30 | parser.add_argument('--input', '-i', dest='input', required=True) 31 | parser.add_argument('--output', '-o', dest='output') 32 | args = parser.parse_args() 33 | 34 | with ImageInference(face_detection.model()) as inference: 35 | image = Image.open(args.input) 36 | draw = ImageDraw.Draw(image) 37 | faces = face_detection.get_faces(inference.run(image)) 38 | for i, face in enumerate(faces): 39 | print('Face #%d: %s' % (i, face)) 40 | x, y, width, height = face.bounding_box 41 | draw.rectangle((x, y, x + width, y + height), outline='red') 42 | if args.output: 43 | image.save(args.output) 44 | 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /src/examples/vision/dish_detection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Dish detector library demo. 16 | 17 | - Takes an input image and tries to detect dish types. 18 | - Draws bounding boxes around detected dishes. 19 | - Saves an image with bounding boxes around detected dishes. 20 | """ 21 | import argparse 22 | 23 | from PIL import Image, ImageDraw 24 | 25 | from aiy.vision.inference import ImageInference 26 | from aiy.vision.models import dish_detection 27 | 28 | def main(): 29 | parser = argparse.ArgumentParser() 30 | parser.add_argument('--input', '-i', dest='input', required=True) 31 | parser.add_argument('--output', '-o', dest='output') 32 | args = parser.parse_args() 33 | 34 | with ImageInference(dish_detection.model()) as inference: 35 | image = Image.open(args.input) 36 | draw = ImageDraw.Draw(image) 37 | dishes = dish_detection.get_dishes(inference.run(image)) 38 | for i, dish in enumerate(dishes): 39 | print('Dish #%d: %s' % (i, dish)) 40 | x, y, width, height = dish.bounding_box 41 | draw.rectangle((x, y, x + width, y + height), outline='red') 42 | if args.output: 43 | image.save(args.output) 44 | 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /src/examples/voice/voice_recorder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import argparse 17 | import time 18 | import threading 19 | 20 | from aiy.board import Board 21 | from aiy.voice.audio import AudioFormat, play_wav, record_file, Recorder 22 | 23 | def main(): 24 | parser = argparse.ArgumentParser() 25 | parser.add_argument('--filename', '-f', default='recording.wav') 26 | args = parser.parse_args() 27 | 28 | with Board() as board: 29 | print('Press button to start recording.') 30 | board.button.wait_for_press() 31 | 32 | done = threading.Event() 33 | board.button.when_pressed = done.set 34 | 35 | def wait(): 36 | start = time.monotonic() 37 | while not done.is_set(): 38 | duration = time.monotonic() - start 39 | print('Recording: %.02f seconds [Press button to stop]' % duration) 40 | time.sleep(0.5) 41 | 42 | record_file(AudioFormat.CD, filename=args.filename, wait=wait, filetype='wav') 43 | print('Press button to play recorded sound.') 44 | board.button.wait_for_press() 45 | 46 | print('Playing...') 47 | play_wav(args.filename) 48 | print('Done.') 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /drivers/overlays/vision/opt/aiy/overlay-vision/aiy-visionbonnet-overlay.dts: -------------------------------------------------------------------------------- 1 | // Definitions for Google visionbonnet v1 overlay 2 | /dts-v1/; 3 | /plugin/; 4 | 5 | / { 6 | compatible = "brcm,bcm2708"; 7 | 8 | fragment@0 { 9 | target = <&spi0>; 10 | __overlay__ { 11 | status = "okay"; 12 | }; 13 | }; 14 | 15 | fragment@1 { 16 | target = <&spidev0>; 17 | __overlay__ { 18 | status = "disabled"; 19 | }; 20 | }; 21 | 22 | fragment@2 { 23 | target = <&gpio>; 24 | __overlay__ { 25 | aiyvision_pins: aiyvision_pins { 26 | brcm,pins = <6 13 26 8>; 27 | brcm,function = <0 0 0 1>; 28 | brcm,pull = <0 2 0 2>; 29 | }; 30 | }; 31 | }; 32 | 33 | fragment@3 { 34 | target = <&spi0_cs_pins>; 35 | __overlay__ { 36 | brcm,pins = <>; 37 | }; 38 | }; 39 | 40 | fragment@4 { 41 | target = <&i2c_arm>; 42 | __overlay__ { 43 | #address-cells = <1>; 44 | #size-cells = <0>; 45 | status = "okay"; 46 | aiy_io: aiy-io-i2c@51 { 47 | compatible = "google,aiy-io-i2c"; 48 | reg = <0x51>; 49 | type = "vision"; 50 | status = "okay"; 51 | aiy_gpio: aiy-gpio { 52 | compatible = "google,gpio-aiy-io"; 53 | gpio-controller; 54 | #gpio-cells = <2>; 55 | }; 56 | }; 57 | ktd2026@30 { 58 | compatible = "kinetic,ktd2026"; 59 | reg = <0x30>; 60 | status = "okay"; 61 | }; 62 | }; 63 | }; 64 | 65 | fragment@5 { 66 | target = <&spi0>; 67 | __overlay__ { 68 | #address-cells = <1>; 69 | #size-cells = <0>; 70 | aiyvision@0 { 71 | compatible = "google,visionbonnet"; 72 | pinctrl-names = "default"; 73 | pinctrl-0 = <&aiyvision_pins>; 74 | spi-max-frequency = <8000000>; 75 | vision-gpios = <&gpio 13 0>, 76 | <&gpio 6 0>, 77 | <&gpio 26 0>, 78 | <&gpio 8 0>; 79 | aiy-gpios = <&aiy_gpio 7 0>; 80 | reg = <0>; 81 | status = "okay"; 82 | }; 83 | }; 84 | }; 85 | }; 86 | -------------------------------------------------------------------------------- /drivers/overlays/voice/opt/aiy/overlay-voice/aiy-voicebonnet-overlay.dts: -------------------------------------------------------------------------------- 1 | // Definitions for Google voiceHAT v1 soundcard overlay 2 | /dts-v1/; 3 | /plugin/; 4 | 5 | / { 6 | compatible = "brcm,bcm2708"; 7 | 8 | fragment@0 { 9 | target = <&i2s>; 10 | __overlay__ { 11 | status = "okay"; 12 | }; 13 | }; 14 | 15 | fragment@1 { 16 | target = <&i2c_arm>; 17 | __overlay__ { 18 | #address-cells = <1>; 19 | #size-cells = <0>; 20 | status = "okay"; 21 | 22 | aiy-io-i2c@52 { 23 | compatible = "google,aiy-io-i2c"; 24 | reg = <0x52>; 25 | type = "voice"; 26 | status = "okay"; 27 | 28 | aiy_gpio: aiy-gpio { 29 | compatible = "google,gpio-aiy-io"; 30 | gpio-controller; 31 | #gpio-cells = <2>; 32 | }; 33 | }; 34 | 35 | rt5645: rt5645@1a { 36 | compatible = "realtek,rt5645"; 37 | reg = <0x1a>; 38 | realtek,dmic1-data-pin = <0>; 39 | realtek,dmic2-data-pin = <0>; 40 | realtek,jd-mode = <0>; 41 | realtek,jd-invert; 42 | interrupt-parent = <&gpio>; 43 | interrupts = <24 3>; 44 | hp-detect-gpios = <&gpio 24 0>; 45 | }; 46 | 47 | ktd2026@31 { 48 | compatible = "kinetic,ktd2026"; 49 | reg = <0x31>; 50 | status = "okay"; 51 | }; 52 | 53 | }; 54 | }; 55 | 56 | fragment@2 { 57 | target = <&gpio>; 58 | __overlay__ { 59 | pinctrl-names = "default"; 60 | pinctrl-0 = <&hp_detect_pins>; 61 | 62 | hp_detect_pins: hp_detect_pins { 63 | brcm,pins = <24>; 64 | brcm,function = <0>; /* GPIO in */ 65 | brcm,pull = <0>; /* disable pull-down/up */ 66 | }; 67 | }; 68 | }; 69 | 70 | fragment@3 { 71 | target = <&sound>; 72 | __overlay__ { 73 | compatible = "google,aiy-voicebonnet"; 74 | i2s-controller = <&i2s>; 75 | aiy-voicebonnet,audio-codec = <&rt5645>; 76 | google,model = "aiy-voicebonnet"; 77 | mic-switch-gpios = <&aiy_gpio 0 0>; 78 | status = "okay"; 79 | }; 80 | }; 81 | }; 82 | -------------------------------------------------------------------------------- /src/aiy/vision/models/dish_detection.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """API for Dish Detection.""" 15 | 16 | from collections import namedtuple 17 | 18 | from aiy.vision.inference import ModelDescriptor 19 | from aiy.vision.models import utils 20 | 21 | 22 | _COMPUTE_GRAPH_NAME = 'dish_detection.binaryproto' 23 | _CLASSES = utils.load_labels('mobilenet_v1_192res_1.0_seefood_labels.txt') 24 | 25 | # sorted_scores: sorted list of (label, score) tuples. 26 | # bounding_box: (x, y, width, height) tuple. 27 | Dish = namedtuple('Dish', ('sorted_scores', 'bounding_box')) 28 | 29 | 30 | def model(): 31 | return ModelDescriptor( 32 | name='DishDetection', 33 | input_shape=(1, 0, 0, 3), 34 | input_normalizer=(0, 0), 35 | compute_graph=utils.load_compute_graph(_COMPUTE_GRAPH_NAME)) 36 | 37 | 38 | def _get_sorted_scores(scores, top_k, threshold): 39 | pairs = [('/'.join(_CLASSES[i]), prob) for i, prob in enumerate(scores) if prob > threshold] 40 | pairs = sorted(pairs, key=lambda pair: pair[1], reverse=True) 41 | return pairs[0:top_k] 42 | 43 | 44 | def get_dishes(result, top_k=3, threshold=0.1): 45 | """Returns list of Dish objects decoded from the inference result.""" 46 | assert len(result.tensors) == 2 47 | bboxes = utils.reshape(result.tensors['bounding_boxes'].data, 4) 48 | dish_scores = utils.reshape(result.tensors['dish_scores'].data, len(_CLASSES)) 49 | assert len(bboxes) == len(dish_scores) 50 | 51 | return [Dish(_get_sorted_scores(scores, top_k, threshold), tuple(bbox)) 52 | for scores, bbox in zip(dish_scores, bboxes)] 53 | -------------------------------------------------------------------------------- /src/examples/vision/object_meter/wordnet_grouping/category_mapper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2018 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Utility for grouping ImageNet classifications under super-categories. 16 | 17 | Super categories are defined by the mapping_data.py file which uses labels from 18 | nodes above the grouped leaves in the wordnet for defining super-categories. 19 | """ 20 | 21 | from .mapping_data import CATEGORIES 22 | from .mapping_data import MAPPINGS 23 | 24 | 25 | def get_category(word): 26 | return MAPPINGS.get(word) 27 | 28 | 29 | def get_categories(): 30 | return CATEGORIES 31 | 32 | 33 | def get_word_index(word): 34 | category = get_category(word) 35 | if category is None: 36 | return -1 37 | return get_categories().index(category) 38 | 39 | 40 | def get_category_index(category): 41 | try: 42 | return get_categories().index(category) 43 | except ValueError: 44 | return -1 45 | 46 | 47 | def _example_usage(): 48 | """Example usage for the category mapper utility.""" 49 | print('~'.join(get_categories())) 50 | print(get_category('hay')) 51 | print(get_category('ballpoint/ballpoint pen/ballpen/Biro')) 52 | print(get_category('beer bottle')) 53 | print(get_category('NASDFOIAAS')) 54 | print(get_word_index('beer bottle')) 55 | print(get_word_index('NASDFLJ')) 56 | for cat in get_categories(): 57 | print('%d : %s' % (get_category_index(cat), cat)) 58 | 59 | cat = 'Other' 60 | print('%d : %s' % (get_category_index(cat), cat)) 61 | 62 | 63 | if __name__ == '__main__': 64 | _example_usage() 65 | -------------------------------------------------------------------------------- /src/aiy/vision/models/face_detection.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """API for Face Detection.""" 15 | 16 | from collections import namedtuple 17 | 18 | from aiy.vision.inference import ModelDescriptor 19 | from aiy.vision.models import utils 20 | 21 | 22 | _COMPUTE_GRAPH_NAME = 'face_detection.binaryproto' 23 | 24 | # face_score: float, face confidence score from 0.0 to 1.0. 25 | # joy_score: float, face joy score from 0.0 to 1.0. 26 | # bounding_box: (x, y, width, height) tuple. 27 | Face = namedtuple('Face', ('face_score', 'joy_score', 'bounding_box')) 28 | 29 | 30 | def model(): 31 | # Face detection model has special implementation in VisionBonnet firmware. 32 | # input_shape, input_normalizer, and compute_graph params have no effect. 33 | return ModelDescriptor( 34 | name='FaceDetection', 35 | input_shape=(1, 0, 0, 3), 36 | input_normalizer=(0, 0), 37 | compute_graph=utils.load_compute_graph(_COMPUTE_GRAPH_NAME)) 38 | 39 | 40 | def get_faces(result): 41 | """Returns list of Face objects decoded from the inference result.""" 42 | assert len(result.tensors) == 3 43 | # TODO(dkovalev): check tensor shapes 44 | bboxes = utils.reshape(result.tensors['bounding_boxes'].data, 4) 45 | face_scores = tuple(result.tensors['face_scores'].data) 46 | joy_scores = tuple(result.tensors['joy_scores'].data) 47 | assert len(bboxes) == len(joy_scores) 48 | assert len(bboxes) == len(face_scores) 49 | return [ 50 | Face(face_score, joy_score, tuple(bbox)) 51 | for face_score, joy_score, bbox in zip(face_scores, joy_scores, bboxes) 52 | ] 53 | -------------------------------------------------------------------------------- /src/examples/vision/face_detection_raspivid.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2017 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """Camera inference face detection demo code. 17 | 18 | Runs continuous face detection on the VisionBonnet and prints the number of 19 | detected faces. 20 | 21 | Example: 22 | face_detection_raspivid.py --num_frames 10 23 | """ 24 | import argparse 25 | import subprocess 26 | 27 | from contextlib import contextmanager 28 | from aiy.vision.inference import CameraInference 29 | from aiy.vision.models import face_detection 30 | 31 | def avg_joy_score(faces): 32 | if faces: 33 | return sum(face.joy_score for face in faces) / len(faces) 34 | return 0.0 35 | 36 | def raspivid_cmd(sensor_mode): 37 | return ('raspivid', '--mode', str(sensor_mode), '--timeout', '0', '--nopreview') 38 | 39 | @contextmanager 40 | def Process(cmd): 41 | process = subprocess.Popen(cmd) 42 | try: 43 | yield 44 | finally: 45 | process.terminate() 46 | process.wait() 47 | 48 | def main(): 49 | parser = argparse.ArgumentParser('Face detection using raspivid.') 50 | parser.add_argument('--num_frames', '-n', type=int, default=None, 51 | help='Sets the number of frames to run for, otherwise runs forever.') 52 | args = parser.parse_args() 53 | 54 | with Process(raspivid_cmd(sensor_mode=4)), \ 55 | CameraInference(face_detection.model()) as inference: 56 | for result in inference.run(args.num_frames): 57 | faces = face_detection.get_faces(result) 58 | print('#%05d (%5.2f fps): num_faces=%d, avg_joy_score=%.2f' % 59 | (inference.count, inference.rate, len(faces), avg_joy_score(faces))) 60 | 61 | if __name__ == '__main__': 62 | main() 63 | -------------------------------------------------------------------------------- /src/aiy/vision/models/dish_classification.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """API for Dish Classification.""" 15 | 16 | from aiy.vision.inference import ModelDescriptor 17 | from aiy.vision.models import utils 18 | 19 | _COMPUTE_GRAPH_NAME = 'mobilenet_v1_192res_1.0_seefood.binaryproto' 20 | _CLASSES = utils.load_labels('mobilenet_v1_192res_1.0_seefood_labels.txt') 21 | 22 | def model(): 23 | return ModelDescriptor( 24 | name='dish_classification', 25 | input_shape=(1, 192, 192, 3), 26 | input_normalizer=(128.0, 128.0), 27 | compute_graph=utils.load_compute_graph(_COMPUTE_GRAPH_NAME)) 28 | 29 | 30 | def _get_probs(result): 31 | assert len(result.tensors) == 1 32 | tensor = result.tensors['MobilenetV1/Predictions/Softmax'] 33 | assert utils.shape_tuple(tensor.shape) == (1, 1, 1, 2024) 34 | return tuple(tensor.data) 35 | 36 | 37 | def get_classes(result, top_k=None, threshold=0.0): 38 | """Converts dish classification model output to list of detected objects. 39 | 40 | Args: 41 | result: output tensor from dish classification model. 42 | top_k: int; max number of objects to return. 43 | threshold: float; min probability of each returned object. 44 | 45 | Returns: 46 | A list of (class_name: string, probability: float) pairs ordered by 47 | probability from highest to lowest. The number of pairs is not greater than 48 | top_k. Each probability is greater than threshold. For 49 | example: 50 | 51 | [('Ramen', 0.981934) 52 | ('Yaka mein, 0.005497)] 53 | """ 54 | probs = _get_probs(result) 55 | pairs = [pair for pair in enumerate(probs) if pair[1] > threshold] 56 | pairs = sorted(pairs, key=lambda pair: pair[1], reverse=True) 57 | pairs = pairs[0:top_k] 58 | return [('/'.join(_CLASSES[index]), prob) for index, prob in pairs] 59 | -------------------------------------------------------------------------------- /src/examples/buzzer/tetris.track: -------------------------------------------------------------------------------- 1 | title Arpeggiated Tetris Theme 2 | speed 4 3 | order 0 4 | end 5 | 6 | pattern 7 | E5 arpg E4 8 | arpg E4 9 | arpg E4 10 | arpg E4 11 | arpg A4 12 | arpg A4 13 | arpg A4 14 | arpg A4 15 | arpg A4 16 | B4 arpg E4 17 | arpg E4 18 | arpg E4 19 | arpg E4 20 | arpg E4 21 | C5 arpg A4 22 | arpg A4 23 | arpg A4 24 | arpg A4 25 | arpg A4 26 | D5 arpg F4 27 | arpg F4 28 | arpg F4 29 | arpg F4 30 | arpg F4 31 | E5 arpg G4 32 | arpg G4 33 | D5 arpg G4 34 | arpg G4 35 | C5 arpg F4 36 | arpg F4 37 | arpg F4 38 | arpg F4 39 | arpg F4 40 | B4 arpg G4 41 | arpg G4 42 | arpg G4 43 | arpg G4 44 | arpg G4 45 | A4 arpg E4 46 | arpg E4 47 | arpg E4 48 | arpg E4 49 | arpg A4 50 | arpg A4 51 | arpg A4 52 | arpg A4 53 | arpg A4 54 | A4 arpg E4 55 | arpg E4 56 | arpg E4 57 | arpg E4 58 | arpg E4 59 | C5 arpg A4 60 | arpg A4 61 | arpg A4 62 | arpg A4 63 | arpg A4 64 | E5 arpg G4 65 | arpg G4 66 | arpg G4 67 | arpg G4 68 | arpg G4 69 | arpg F4 70 | arpg F4 71 | arpg F4 72 | arpg F4 73 | D5 arpg F4 74 | arpg F4 75 | arpg F4 76 | arpg F4 77 | arpg F4 78 | C5 arpg E4 79 | arpg E4 80 | arpg E4 81 | arpg E4 82 | arpg E4 83 | B4 arpg D4 84 | arpg D4 85 | arpg D4 86 | arpg D4 87 | arpg D4 88 | arpg D4 89 | arpg D4 90 | arpg D4 91 | arpg D4 92 | B4 arpg E4 93 | arpg E4 94 | arpg E4 95 | arpg E4 96 | arpg E4 97 | C5 arpg A4 98 | arpg A4 99 | arpg A4 100 | arpg A4 101 | arpg A4 102 | D5 arpg E4 103 | arpg E4 104 | arpg E4 105 | arpg E4 106 | arpg E4 107 | arpg A4 108 | arpg A4 109 | arpg A4 110 | arpg A4 111 | E5 arpg G4 112 | arpg G4 113 | arpg G4 114 | arpg G4 115 | arpg G4 116 | arpg F4 117 | arpg F4 118 | arpg F4 119 | arpg F4 120 | C5 arpg G4 121 | arpg G4 122 | arpg G4 123 | arpg G4 124 | arpg G4 125 | arpg F4 126 | arpg F4 127 | arpg F4 128 | arpg F4 129 | A4 arpg C4 130 | arpg C4 131 | arpg C4 132 | arpg C4 133 | arpg C4 134 | arpg C4 135 | arpg C4 136 | noff 137 | 138 | A4 arpg C4 139 | arpg C4 140 | arpg C4 141 | arpg C4 142 | arpg C4 143 | arpg C4 144 | arpg C4 145 | arpg C4 146 | arpg C4 147 | stop 148 | end 149 | -------------------------------------------------------------------------------- /src/tests/face_detection_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import unittest 15 | 16 | from aiy.vision.inference import ImageInference 17 | from aiy.vision.models import face_detection 18 | from .test_util import TestImage 19 | 20 | 21 | class FaceDetectionTest(unittest.TestCase): 22 | 23 | def testFaceDetection(self): 24 | with TestImage('faces.jpg') as image: 25 | with ImageInference(face_detection.model()) as inference: 26 | faces = face_detection.get_faces(inference.run(image)) 27 | self.assertEqual(2, len(faces)) 28 | 29 | face0 = faces[0] 30 | self.assertAlmostEqual(face0.face_score, 1.0, delta=0.001) 31 | self.assertAlmostEqual(face0.joy_score, 0.969, delta=0.001) 32 | self.assertEqual((812.0, 44.0, 1000.0, 1000.0), face0.bounding_box) 33 | 34 | face1 = faces[1] 35 | self.assertAlmostEqual(face1.face_score, 0.884, delta=0.001) 36 | self.assertAlmostEqual(face1.joy_score, 0.073, delta=0.001) 37 | self.assertEqual((748.0, 1063.0, 496.0, 496.0), face1.bounding_box) 38 | 39 | def testFaceDetectionWithParams(self): 40 | with TestImage('faces.jpg') as image: 41 | with ImageInference(face_detection.model()) as inference: 42 | params = {'max_face_size': 500} 43 | faces = face_detection.get_faces(inference.run(image, params)) 44 | self.assertEqual(1, len(faces)) 45 | 46 | face0 = faces[0] 47 | self.assertAlmostEqual(face0.face_score, 0.884, delta=0.001) 48 | self.assertAlmostEqual(face0.joy_score, 0.073, delta=0.001) 49 | self.assertEqual((748.0, 1063.0, 496.0, 496.0), face0.bounding_box) 50 | 51 | if __name__ == '__main__': 52 | unittest.main() 53 | -------------------------------------------------------------------------------- /src/examples/vision/image_classification_camera.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2017 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """Camera image classification demo code. 17 | 18 | Runs continuous image classification on camera frames and prints detected object 19 | classes. 20 | 21 | Example: 22 | image_classification_camera.py --num_frames 10 23 | """ 24 | import argparse 25 | import contextlib 26 | 27 | from aiy.vision.inference import CameraInference 28 | from aiy.vision.models import image_classification 29 | from picamera import PiCamera 30 | 31 | def classes_info(classes): 32 | return ', '.join('%s (%.2f)' % pair for pair in classes) 33 | 34 | @contextlib.contextmanager 35 | def CameraPreview(camera, enabled): 36 | if enabled: 37 | camera.start_preview() 38 | try: 39 | yield 40 | finally: 41 | if enabled: 42 | camera.stop_preview() 43 | 44 | def main(): 45 | parser = argparse.ArgumentParser('Image classification camera inference example.') 46 | parser.add_argument('--num_frames', '-n', type=int, default=None, 47 | help='Sets the number of frames to run for, otherwise runs forever.') 48 | parser.add_argument('--num_objects', '-c', type=int, default=3, 49 | help='Sets the number of object interences to print.') 50 | parser.add_argument('--nopreview', dest='preview', action='store_false', default=True, 51 | help='Enable camera preview') 52 | args = parser.parse_args() 53 | 54 | with PiCamera(sensor_mode=4, framerate=30) as camera, \ 55 | CameraPreview(camera, enabled=args.preview), \ 56 | CameraInference(image_classification.model()) as inference: 57 | for result in inference.run(args.num_frames): 58 | classes = image_classification.get_classes(result, top_k=args.num_objects) 59 | print(classes_info(classes)) 60 | if classes: 61 | camera.annotate_text = '%s (%.2f)' % classes[0] 62 | 63 | if __name__ == '__main__': 64 | main() 65 | -------------------------------------------------------------------------------- /src/tests/images/Makefile: -------------------------------------------------------------------------------- 1 | # Open Images Dataset V4 2 | all: cat.jpg dog.jpg hotdog.jpg faces.jpg sparrow.jpg bee.jpg lily.jpg 3 | 4 | define download_image 5 | $(eval $1_TMP := $(shell mktemp /tmp/$1.XXXXXX)) 6 | wget -O $($1_TMP) $2 7 | echo $3 $($1_TMP) | md5sum --check - 8 | mv $($1_TMP) $1 9 | endef 10 | 11 | # License: https://creativecommons.org/licenses/by/2.0/ 12 | # URL: https://storage.googleapis.com/openimages/web/visualizer/index.html?set=train&c=%2Fm%2F01yrx&id=818c36d0b603e096 13 | cat.jpg: 14 | $(call download_image,$@,"https://farm7.staticflickr.com/3712/11270364836_8d1182be97_o.jpg","f08083ddb111a9d5bd6e00d7c04aa97f") 15 | 16 | # License: https://creativecommons.org/licenses/by/2.0/ 17 | # URL: https://storage.googleapis.com/openimages/web/visualizer/index.html?set=train&c=%2Fm%2F0bt9lr&id=b2e6af69191eb3bf 18 | dog.jpg: 19 | $(call download_image,$@,"https://c8.staticflickr.com/8/7580/15865399370_3857b60669_o.jpg","416afbfd4bd09758955cdffa2e62a58e") 20 | 21 | # License: https://creativecommons.org/licenses/by/2.0/ 22 | # URL: https://storage.googleapis.com/openimages/web/visualizer/index.html?set=train&c=%2Fm%2F01b9xk&id=4d7133d6aabede44 23 | hotdog.jpg: 24 | $(call download_image,$@,"https://farm5.staticflickr.com/7284/16301478750_b036c617cb_o.jpg","cc466191c0844c8da25bbbf63b66bd15") 25 | 26 | # License: https://creativecommons.org/licenses/by/2.0/ 27 | # URL: https://storage.googleapis.com/openimages/web/visualizer/index.html?set=train&c=%2Fm%2F0dzct&id=769bb77259bd85d2 28 | faces.jpg: 29 | $(call download_image,$@,"https://farm3.staticflickr.com/4109/5031905069_cd67b182e3_o.jpg","e23cef02664a28c9c8523a7889b6ff9b") 30 | 31 | # License: https://creativecommons.org/licenses/by/2.0/ 32 | # https://storage.googleapis.com/openimages/web/visualizer/index.html?set=train&c=%2Fm%2F0h23m&id=d7b9e42392fc7998 33 | sparrow.jpg: 34 | $(call download_image,$@,"https://c1.staticflickr.com/1/542/19730052293_4e9bede5d0_o.jpg","3c0a157b55daaf87dbab7145072226f0") 35 | 36 | # License: https://creativecommons.org/licenses/by/2.0/ 37 | # https://storage.googleapis.com/openimages/web/visualizer/index.html?set=train&c=%2Fm%2F01h3n&id=618074321d59d0d8 38 | bee.jpg: 39 | $(call download_image,$@,"https://c1.staticflickr.com/3/2911/13993945720_95a938e5cf_o.jpg","13b855c8f32727929393ed0eb2a89525") 40 | 41 | # License: https://creativecommons.org/licenses/by/2.0/ 42 | # https://storage.googleapis.com/openimages/web/visualizer/index.html?set=train&c=%2Fm%2F0jqgx&id=7bf5c5c8b0f4f293 43 | lily.jpg: 44 | $(call download_image,$@,"https://farm4.staticflickr.com/4110/5099896296_2e2617a0a8_o.jpg","b3b94395f631f2f5ded9299bc46c39a4") 45 | 46 | clean: 47 | rm -f *.jpg 48 | -------------------------------------------------------------------------------- /src/aiy/vision/_transport.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Transport to communicate with VisionBonnet board.""" 16 | 17 | import logging 18 | import os 19 | import socket 20 | import struct 21 | 22 | from . import _spicomm 23 | 24 | class _SpiTransport: 25 | """Communicate with VisionBonnet over SPI bus.""" 26 | 27 | def __init__(self): 28 | self._spicomm = _spicomm.Spicomm() 29 | 30 | def send(self, request, timeout=None): 31 | return self._spicomm.transact(request, timeout=timeout) 32 | 33 | def close(self): 34 | self._spicomm.close() 35 | 36 | 37 | def _socket_recvall(s, size): 38 | buf = b'' 39 | while size: 40 | newbuf = s.recv(size) 41 | if not newbuf: 42 | return None 43 | buf += newbuf 44 | size -= len(newbuf) 45 | return buf 46 | 47 | 48 | def _socket_receive_message(s): 49 | buf = _socket_recvall(s, 4) # 4 bytes 50 | if not buf: 51 | return None 52 | size = struct.unpack('!I', buf)[0] 53 | return _socket_recvall(s, size) 54 | 55 | 56 | def _socket_send_message(s, msg): 57 | s.sendall(struct.pack('!I', len(msg))) # 4 bytes 58 | s.sendall(msg) # len(msg) bytes 59 | 60 | 61 | class _SocketTransport: 62 | """Communicate with VisionBonnet over socket.""" 63 | 64 | def __init__(self): 65 | """Open connection to the bonnet.""" 66 | self._client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 67 | 68 | host = os.environ.get('VISION_BONNET_HOST', '172.28.28.10') 69 | port = int(os.environ.get('VISION_BONNET_PORT', '35000')) 70 | self._client.connect((host, port)) 71 | 72 | def send(self, request, timeout=None): 73 | _socket_send_message(self._client, request) 74 | return _socket_receive_message(self._client) 75 | 76 | def close(self): 77 | self._client.close() 78 | 79 | 80 | def _is_arm(): 81 | return os.uname()[4].startswith('arm') 82 | 83 | 84 | def make_transport(): 85 | if _is_arm(): 86 | return _SpiTransport() 87 | return _SocketTransport() 88 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AIY Projects 2 | 3 | This repository contains an easy-to-use Python API for the [AIY Vision Kit][aiy-vision] 4 | and [AIY Voice Kit][aiy-voice]. The code for all AIY kits is in the `aiyprojects` branch, 5 | and is included in images starting with `aiyprojects-2017-12-18.img`. 6 | The previous `voicekit` branch contains code just for the Voice Kit, and the 7 | `master` branch contains the original deprecated `Voice Recognizer` demo. 8 | 9 | ## Documentation 10 | 11 | If you're just getting started with the Vision or Voice kit, see the 12 | assembly guide and other maker guides at [aiyprojects.withgoogle.com]. 13 | 14 | If you just need the Python API reference, see [aiyprojects.readthedocs.io]. 15 | Also have a look at the [example code][aiy-github-examples]. 16 | 17 | If you want to flash the latest AIY system image or install AIY packages on an existing 18 | Raspbian system, read the [system updates guide][HACKING.md]. 19 | 20 | ## Releases 21 | 22 | * [SD card image downloads][downloads] 23 | * [Change log][changelog] 24 | 25 | You can also build an SD card image yourself using [AIY Projects Tools][aiy-projects-tools]. 26 | 27 | ## Bugs & Support 28 | 29 | If you've found a bug, please [review the known issues and report a new one][aiy-github-issues]. 30 | 31 | If you've fixed a bug yourself, please send us a pull request! 32 | For details, read [CONTRIBUTING.md]. 33 | 34 | If you're having trouble assembling your kit or running the demos, try the following links: 35 | 36 | * [AIY Help docs][help-docs] 37 | * [AIY Forums][aiy-forums] 38 | * [AIY Stack Overflow][aiy-stack-overflow] 39 | * [AIY GitHub Issues][aiy-github-issues] 40 | * support-aiyprojects@google.com 41 | 42 | ## 43 | 44 |

45 | 46 |

47 | 48 | [HACKING.md]: HACKING.md 49 | [CONTRIBUTING.md]: CONTRIBUTING.md 50 | [downloads]: https://github.com/google/aiyprojects-raspbian/releases 51 | [changelog]: CHANGES.md 52 | 53 | [aiy-projects-tools]: https://github.com/google/aiyprojects-raspbian-tools 54 | [aiyprojects.withgoogle.com]: https://aiyprojects.withgoogle.com 55 | [aiyprojects.readthedocs.io]: https://aiyprojects.readthedocs.io 56 | [aiy-vision]: https://aiyprojects.withgoogle.com/vision/ 57 | [aiy-voice]: https://aiyprojects.withgoogle.com/voice/ 58 | 59 | [help-docs]: https://aiyprojects.withgoogle.com/help 60 | [aiy-forums]: https://www.raspberrypi.org/forums/viewforum.php?f=114 61 | [aiy-stack-overflow]: https://stackoverflow.com/questions/tagged/google-aiy 62 | [aiy-github-issues]: https://github.com/viraniac/aiyprojects-raspbian/issues 63 | [aiy-github-examples]: https://github.com/viraniac/aiyprojects-raspbian/tree/aiyprojects/src/examples 64 | -------------------------------------------------------------------------------- /src/examples/vision/any_model_camera.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2017 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """Script to load and run model on Vision Bonnet. 17 | 18 | The primary purpose of this script is to make sure a compiled model can run on 19 | Vision Bonnet. It does not try to interpret the output tensor. 20 | 21 | Example: 22 | ~/AIY-projects-python/src/examples/vision/any_model_camera.py \ 23 | --model_path ~/models/mobilenet_ssd_256res_0.125_person_cat_dog.binaryproto \ 24 | --input_height 256 \ 25 | --input_width 256 26 | """ 27 | import argparse 28 | 29 | from picamera import PiCamera 30 | 31 | from aiy.vision.inference import CameraInference, ModelDescriptor 32 | from aiy.vision.models import utils 33 | 34 | def tensors_info(tensors): 35 | return ', '.join('%s [%d elements]' % (name, len(tensor.data)) 36 | for name, tensor in tensors.items()) 37 | 38 | def main(): 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument('--model_name', default='test_model', help='Model identifier.') 41 | parser.add_argument('--model_path', required=True, help='Path to model file.') 42 | parser.add_argument('--input_height', type=int, required=True, help='Input height.') 43 | parser.add_argument('--input_width', type=int, required=True, help='Input width.') 44 | parser.add_argument('--input_depth', type=int, default=3, help='Input depth.') 45 | parser.add_argument('--input_mean', type=float, default=128.0, help='Input mean.') 46 | parser.add_argument('--input_std', type=float, default=128.0, help='Input std.') 47 | args = parser.parse_args() 48 | 49 | model = ModelDescriptor( 50 | name=args.model_name, 51 | input_shape=(1, args.input_height, args.input_width, args.input_depth), 52 | input_normalizer=(args.input_mean, args.input_std), 53 | compute_graph=utils.load_compute_graph(args.model_path)) 54 | 55 | with PiCamera(sensor_mode=4, framerate=30): 56 | with CameraInference(model) as inference: 57 | for result in inference.run(): 58 | print('#%05d (%5.2f fps): %s' % 59 | (inference.count, inference.rate, tensors_info(result.tensors))) 60 | 61 | 62 | if __name__ == '__main__': 63 | main() 64 | -------------------------------------------------------------------------------- /checkpoints/check_wifi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2017 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Check that Wi-Fi is working.""" 18 | 19 | import socket 20 | import subprocess 21 | import traceback 22 | 23 | WPA_CONF_PATH = '/etc/wpa_supplicant/wpa_supplicant.conf' 24 | GOOGLE_SERVER_ADDRESS = ('speech.googleapis.com', 443) 25 | 26 | ERROR_NOT_CONFIGURED=''' 27 | Please click the Wi-Fi icon at the top right to set up a Wi-Fi network. 28 | ''' 29 | 30 | ERROR_NOT_CONNECTED=''' 31 | Please click the Wi-Fi icon at the top right to check your Wi-Fi settings. 32 | ''' 33 | 34 | ERROR_GOOGLE_SERVER=''' 35 | Failed to reach Google servers. Please check that your Wi-Fi network is 36 | connected to the Internet. 37 | ''' 38 | 39 | 40 | def error(message): 41 | print(message.strip()) 42 | 43 | 44 | def check_wifi_is_configured(): 45 | """Check wpa_supplicant.conf has at least one network configured.""" 46 | output = subprocess.check_output(['sudo', 'cat', WPA_CONF_PATH]).decode('utf-8') 47 | if 'network=' not in output: 48 | error(ERROR_NOT_CONFIGURED) 49 | return False 50 | return True 51 | 52 | 53 | def check_wifi_is_connected(): 54 | """Check wlan0 has an IP address.""" 55 | output = subprocess.check_output(['ifconfig', 'wlan0']).decode('utf-8') 56 | if 'inet ' not in output: 57 | error(ERROR_NOT_CONNECTED) 58 | return False 59 | return True 60 | 61 | 62 | def check_can_reach_google_server(): 63 | """Check the API server is reachable on port 443.""" 64 | try: 65 | with socket.create_connection(GOOGLE_SERVER_ADDRESS, timeout=10): 66 | pass 67 | return True 68 | except: 69 | error(ERROR_GOOGLE_SERVER) 70 | return False 71 | 72 | 73 | def main(): 74 | print('Checking the Wi-Fi connection...') 75 | 76 | if not check_wifi_is_configured(): 77 | return 78 | 79 | if not check_wifi_is_connected(): 80 | return 81 | 82 | if not check_can_reach_google_server(): 83 | return 84 | 85 | print('Wi-Fi seems to be working!') 86 | 87 | 88 | if __name__ == '__main__': 89 | try: 90 | main() 91 | except: 92 | traceback.print_exc() 93 | finally: 94 | input('Press Enter to close...') 95 | -------------------------------------------------------------------------------- /drivers/sound/rt5645.c.patch: -------------------------------------------------------------------------------- 1 | --- rt5645.c.orig 2023-03-12 21:22:33.105728429 +0530 2 | +++ rt5645.c 2023-03-12 21:24:11.951409594 +0530 3 | @@ -3153,6 +3153,10 @@ 4 | if (jack_insert) { 5 | regmap_write(rt5645->regmap, RT5645_CHARGE_PUMP, 0x0e06); 6 | 7 | + // Update some bits in MX-91 to prevent accidental 2.17V on the HPO_L 8 | + // output which screws up the jack detection code. 9 | + regmap_update_bits(rt5645->regmap, RT5645_CHARGE_PUMP, 3 << 11, 3 << 11); 10 | + 11 | /* for jack type detect */ 12 | snd_soc_dapm_force_enable_pin(dapm, "LDO2"); 13 | snd_soc_dapm_force_enable_pin(dapm, "Mic Det Power"); 14 | @@ -3172,8 +3176,12 @@ 15 | regmap_write(rt5645->regmap, RT5645_JD_CTRL3, 0x00f0); 16 | regmap_update_bits(rt5645->regmap, RT5645_IN1_CTRL2, 17 | RT5645_CBJ_MN_JD, RT5645_CBJ_MN_JD); 18 | - regmap_update_bits(rt5645->regmap, RT5645_IN1_CTRL1, 19 | - RT5645_CBJ_BST1_EN, RT5645_CBJ_BST1_EN); 20 | + // On the AIY voice bonnet, we do not wish to disable 21 | + // the IN1 port when the jack is connected. 22 | + if (rt5645->pdata.jd_mode != 0) { 23 | + regmap_update_bits(rt5645->regmap, RT5645_IN1_CTRL1, 24 | + RT5645_CBJ_BST1_EN, RT5645_CBJ_BST1_EN); 25 | + } 26 | msleep(100); 27 | regmap_update_bits(rt5645->regmap, RT5645_IN1_CTRL2, 28 | RT5645_CBJ_MN_JD, 0); 29 | @@ -3251,8 +3259,15 @@ 30 | RT5645_GP1_PIN_IRQ, RT5645_GP1_PIN_IRQ); 31 | regmap_update_bits(rt5645->regmap, RT5645_GEN_CTRL1, 32 | RT5645_DIG_GATE_CTRL, RT5645_DIG_GATE_CTRL); 33 | + rt5645_irq(0, rt5645); 34 | + } else if (rt5645->codec_type == CODEC_TYPE_RT5645) { 35 | + regmap_update_bits(rt5645->regmap, RT5645_IN1_CTRL2, 36 | + RT5645_CBJ_DET_MODE, RT5645_CBJ_DET_MODE); 37 | + regmap_update_bits(rt5645->regmap, RT5645_GEN_CTRL2, 0x1 << 4, 0x1 << 4); 38 | + regmap_update_bits(rt5645->regmap, RT5645_GEN_CTRL3, 0x1 << 8, 0x1 << 8); 39 | + regmap_update_bits(rt5645->regmap, RT5645_IN1_CTRL1, 0x1 << 5, 0x1 << 5); 40 | + regmap_update_bits(rt5645->regmap, RT5645_IN1_CTRL3, RT5645_CBJ_TIE_G_L, 0); 41 | } 42 | - rt5645_irq(0, rt5645); 43 | 44 | return 0; 45 | } 46 | @@ -3834,7 +3849,8 @@ 47 | if (device_property_present(dev, "realtek,in2-differential") || 48 | device_property_present(dev, "realtek,dmic1-data-pin") || 49 | device_property_present(dev, "realtek,dmic2-data-pin") || 50 | - device_property_present(dev, "realtek,jd-mode")) 51 | + device_property_present(dev, "realtek,jd-mode") || 52 | + device_property_present(dev, "realtek,jd-invert")) 53 | return true; 54 | 55 | return false; 56 | @@ -3850,7 +3866,7 @@ 57 | "realtek,dmic2-data-pin", &rt5645->pdata.dmic2_data_pin); 58 | device_property_read_u32(dev, 59 | "realtek,jd-mode", &rt5645->pdata.jd_mode); 60 | - 61 | + rt5645->pdata.inv_jd1_1 = device_property_read_bool(dev, "realtek,jd-invert"); 62 | return 0; 63 | } 64 | 65 | -------------------------------------------------------------------------------- /src/tests/object_detection_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import unittest 15 | 16 | from aiy.vision.inference import ImageInference 17 | from aiy.vision.models import object_detection as od 18 | from .test_util import define_test_case, TestImage 19 | 20 | def crop_center(image): 21 | width, height = image.size 22 | size = min(width, height) 23 | x, y = (width - size) / 2, (height - size) / 2 24 | return image.crop((x, y, x + size, y + size)), (x, y) 25 | 26 | 27 | class ObjectDetectionTest: 28 | THRESHOLD = 0.3 29 | 30 | def __init__(self, image_file, sparse): 31 | self.image_file = image_file 32 | self.sparse = sparse 33 | self.check = {'dog.jpg': self.check_dog, 'cat.jpg': self.check_cat}[image_file] 34 | 35 | def check_dog(self, objects): 36 | self.assertEqual(1, len(objects)) 37 | self.assertEqual(od.Object.DOG, objects[0].kind) 38 | self.assertAlmostEqual(0.914, objects[0].score, delta=0.001) 39 | self.assertEqual((52, 116, 570, 485), objects[0].bounding_box) 40 | 41 | 42 | def check_cat(self, objects): 43 | self.assertEqual(1, len(objects)) 44 | self.assertEqual(od.Object.CAT, objects[0].kind) 45 | self.assertAlmostEqual(0.672, objects[0].score, delta=0.001) 46 | self.assertEqual((575, 586, 2187, 1758), objects[0].bounding_box) 47 | 48 | def test_detection(self): 49 | with TestImage(self.image_file) as image: 50 | image_center, offset = crop_center(image) 51 | with ImageInference(od.model()) as inference: 52 | if self.sparse: 53 | sparse_configs = od.sparse_configs(threshold=self.THRESHOLD) 54 | result = inference.run(image_center, sparse_configs=sparse_configs) 55 | objects = od.get_objects_sparse(result, offset) 56 | else: 57 | result = inference.run(image_center) 58 | objects = od.get_objects(result, self.THRESHOLD, offset) 59 | self.check(objects) 60 | 61 | define_test_case(globals(), ObjectDetectionTest, 'dog.jpg', False) 62 | define_test_case(globals(), ObjectDetectionTest, 'dog.jpg', True) 63 | define_test_case(globals(), ObjectDetectionTest, 'cat.jpg', False) 64 | define_test_case(globals(), ObjectDetectionTest, 'cat.jpg', True) 65 | 66 | if __name__ == '__main__': 67 | unittest.main(verbosity=2) 68 | -------------------------------------------------------------------------------- /src/examples/vision/inaturalist_classification.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Image classification library demo.""" 16 | 17 | import argparse 18 | 19 | from PIL import Image 20 | 21 | from aiy.vision.inference import ImageInference 22 | from aiy.vision.models import inaturalist_classification 23 | 24 | 25 | def main(): 26 | parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) 27 | parser.add_argument('--input', '-i', required=True, 28 | help='Input image file.') 29 | parser.add_argument('--threshold', '-t', type=float, default=0.1, 30 | help='Classification probability threshold.') 31 | parser.add_argument('--top_k', '-n', type=int, default=5, 32 | help='Max number of returned classes.') 33 | parser.add_argument('--sparse', '-s', action='store_true', default=False, 34 | help='Use sparse tensors.') 35 | parser.add_argument('--model', '-m', choices=('plants', 'insects', 'birds'), required=True, 36 | help='Model to run.') 37 | args = parser.parse_args() 38 | 39 | model_type = {'plants': inaturalist_classification.PLANTS, 40 | 'insects': inaturalist_classification.INSECTS, 41 | 'birds': inaturalist_classification.BIRDS}[args.model] 42 | 43 | with ImageInference(inaturalist_classification.model(model_type)) as inference: 44 | image = Image.open(args.input) 45 | 46 | if args.sparse: 47 | configs = inaturalist_classification.sparse_configs(top_k=args.top_k, 48 | threshold=args.threshold, 49 | model_type=model_type) 50 | result = inference.run(image, sparse_configs=configs) 51 | classes = inaturalist_classification.get_classes_sparse(result) 52 | else: 53 | result = inference.run(image) 54 | classes = inaturalist_classification.get_classes(result, 55 | top_k=args.top_k, 56 | threshold=args.threshold) 57 | 58 | for i, (label, score) in enumerate(classes): 59 | print('Result %d: %s (prob=%f)' % (i, label, score)) 60 | 61 | 62 | if __name__ == '__main__': 63 | main() 64 | -------------------------------------------------------------------------------- /src/examples/vision/object_detection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Object detection library demo. 16 | 17 | - Takes an input image and tries to detect person, dog, or cat. 18 | - Draws bounding boxes around detected objects. 19 | - Saves an image with bounding boxes around detected objects. 20 | """ 21 | import argparse 22 | 23 | from PIL import Image, ImageDraw 24 | 25 | from aiy.vision.inference import ImageInference 26 | from aiy.vision.models import object_detection 27 | 28 | 29 | def crop_center(image): 30 | width, height = image.size 31 | size = min(width, height) 32 | x, y = (width - size) / 2, (height - size) / 2 33 | return image.crop((x, y, x + size, y + size)), (x, y) 34 | 35 | 36 | def main(): 37 | parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) 38 | parser.add_argument('--input', '-i', dest='input', required=True, 39 | help='Input image file.') 40 | parser.add_argument('--output', '-o', dest='output', 41 | help='Output image file with bounding boxes.') 42 | parser.add_argument('--sparse', '-s', action='store_true', default=False, 43 | help='Use sparse tensors.') 44 | parser.add_argument('--threshold', '-t', type=float, default=0.3, 45 | help='Detection probability threshold.') 46 | args = parser.parse_args() 47 | 48 | with ImageInference(object_detection.model()) as inference: 49 | image = Image.open(args.input) 50 | image_center, offset = crop_center(image) 51 | 52 | if args.sparse: 53 | result = inference.run(image_center, 54 | sparse_configs=object_detection.sparse_configs(args.threshold)) 55 | objects = object_detection.get_objects_sparse(result, offset) 56 | else: 57 | result = inference.run(image_center) 58 | objects = object_detection.get_objects(result, args.threshold, offset) 59 | 60 | for i, obj in enumerate(objects): 61 | print('Object #%d: %s' % (i, obj)) 62 | 63 | if args.output: 64 | draw = ImageDraw.Draw(image) 65 | for i, obj in enumerate(objects): 66 | x, y, width, height = obj.bounding_box 67 | draw.rectangle((x, y, x + width, y + height), outline='red') 68 | image.save(args.output) 69 | 70 | 71 | if __name__ == '__main__': 72 | main() 73 | -------------------------------------------------------------------------------- /drivers/sound/debian/ucm2/aiy-voicebonnet-HiFi.conf: -------------------------------------------------------------------------------- 1 | SectionVerb { 2 | # ALSA PCM 3 | Value { 4 | TQ "HiFi" 5 | 6 | # ALSA PCM device for HiFi 7 | PlaybackPCM "hw:${CardId}" 8 | CapturePCM "hw:${CardId}" 9 | } 10 | 11 | EnableSequence [ 12 | cset "name='DAC1 Playback Volume' 80,80" 13 | 14 | # Output Configuration 15 | cset "name='DAC1 MIXL DAC1 Switch' on" 16 | cset "name='DAC1 MIXR DAC1 Switch' on" 17 | cset "name='Stereo DAC MIXL DAC L1 Switch' on" 18 | cset "name='Stereo DAC MIXR DAC R1 Switch' on" 19 | 20 | cset "name='HPOVOL MIXL DAC1 Switch' on" 21 | cset "name='HPOVOL MIXR DAC1 Switch' on" 22 | cset "name='HPOVOL L Switch' on" 23 | cset "name='HPOVOL R Switch' on" 24 | cset "name='HPO MIX HPVOL Switch' on" 25 | cset "name='Headphone Switch' on" 26 | cset "name='Headphone Playback Volume' 75" 27 | 28 | cset "name='SPK MIXL DAC L1 Switch' on" 29 | cset "name='SPK MIXR DAC R1 Switch' on" 30 | cset "name='SPKVOL L Switch' on" 31 | cset "name='SPKVOL R Switch' on" 32 | cset "name='SPOL MIX SPKVOL L Switch' on" 33 | cset "name='SPOR MIX SPKVOL R Switch' on" 34 | cset "name='Speaker Switch' on" 35 | cset "name='Speaker Channel Switch' on" 36 | cset "name='Speaker Playback Volume' 75" 37 | 38 | # Input Configuration 39 | cset "name='ADC Capture Volume' 65,65" 40 | cset "name='ADC Boost Capture Volume' 1,1" 41 | cset "name='RECMIXL BST1 Switch' on" 42 | cset "name='RECMIXR BST2 Switch' on" 43 | # cset "name='Stereo ADC1 Mux' 'ADC'" 44 | cset "name='Sto1 ADC MIXL ADC1 Switch' on" 45 | cset "name='Sto1 ADC MIXR ADC1 Switch' on" 46 | cset "name='Int Mic Switch' on" 47 | ] 48 | 49 | DisableSequence [ 50 | ] 51 | } 52 | 53 | SectionDevice."Speaker" { 54 | Comment "Speaker" 55 | 56 | Value { 57 | PlaybackChannels "2" 58 | } 59 | 60 | ConflictingDevice [ 61 | "Headphone" 62 | ] 63 | 64 | EnableSequence [ 65 | cset "name='Speaker Channel Switch' on" 66 | cset "name='Speaker Switch' on" 67 | ] 68 | 69 | DisableSequence [ 70 | cset "name='Speaker Channel Switch' off" 71 | cset "name='Speaker Switch' off" 72 | ] 73 | } 74 | 75 | SectionDevice."Headphone" { 76 | Comment "Headphones" 77 | 78 | Value { 79 | PlaybackChannels "2" 80 | JackControl "Headphone Jack" 81 | JackDev "/dev/input/by-path/platform-soc:sound-event" 82 | } 83 | 84 | ConflictingDevice [ 85 | "Speaker" 86 | ] 87 | 88 | EnableSequence [ 89 | cset "name='Speaker Channel Switch' off" 90 | cset "name='Speaker Switch' off" 91 | 92 | cset "name='Headphone Channel Switch' on" 93 | cset "name='Headphone Switch' on" 94 | ] 95 | 96 | DisableSequence [ 97 | cset "name='Headphone Channel Switch' off" 98 | cset "name='Headphone Switch' off" 99 | 100 | cset "name='Speaker Channel Switch' on" 101 | cset "name='Speaker Switch' on" 102 | ] 103 | } 104 | 105 | SectionDevice."Mic" { 106 | Comment "Internal Microphone" 107 | 108 | Value { 109 | CaptureChannels "2" 110 | CapturePriority "150" 111 | } 112 | 113 | EnableSequence [ 114 | ] 115 | 116 | DisableSequence [ 117 | ] 118 | } 119 | 120 | -------------------------------------------------------------------------------- /src/tests/image_classification_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import unittest 15 | 16 | from aiy.vision.inference import ImageInference 17 | from aiy.vision.models import image_classification as ic 18 | from .test_util import define_test_case, TestImage 19 | 20 | class ImageClassificationTest: 21 | TOP_K = 20 22 | THRESHOLD = 0.0 23 | 24 | def __init__(self, model_type, sparse): 25 | self.image_file = 'dog.jpg' 26 | self.model_type = model_type 27 | self.sparse = sparse 28 | self.check = {ic.MOBILENET: self.check_dog_mobilenet, 29 | ic.SQUEEZENET: self.check_dog_squeezenet}[model_type] 30 | 31 | def check_dog_mobilenet(self, classes): 32 | label, score = classes[0] 33 | self.assertEqual('boxer', label) 34 | self.assertAlmostEqual(score, 0.684, delta=0.001) 35 | 36 | label, score = classes[1] 37 | self.assertEqual('bull mastiff', label) 38 | self.assertAlmostEqual(score, 0.222, delta=0.001) 39 | 40 | def check_dog_squeezenet(self, classes): 41 | label, score = classes[0] 42 | self.assertEqual('pug/pug-dog', label) 43 | self.assertAlmostEqual(score, 0.271, delta=0.001) 44 | 45 | label, score = classes[1] 46 | self.assertEqual('bull mastiff', label) 47 | self.assertAlmostEqual(score, 0.141, delta=0.001) 48 | 49 | def test_classification(self): 50 | with TestImage(self.image_file) as image: 51 | with ImageInference(ic.model(self.model_type)) as inference: 52 | if self.sparse: 53 | sparse_configs = ic.sparse_configs(top_k=self.TOP_K, 54 | threshold=self.THRESHOLD, 55 | model_type=self.model_type) 56 | result = inference.run(image, sparse_configs=sparse_configs) 57 | classes = ic.get_classes_sparse(result) 58 | else: 59 | result = inference.run(image) 60 | classes = ic.get_classes(result, top_k=self.TOP_K, threshold=self.THRESHOLD) 61 | 62 | self.check(classes) 63 | 64 | define_test_case(globals(), ImageClassificationTest, ic.MOBILENET, False) 65 | define_test_case(globals(), ImageClassificationTest, ic.MOBILENET, True) 66 | define_test_case(globals(), ImageClassificationTest, ic.SQUEEZENET, False) 67 | define_test_case(globals(), ImageClassificationTest, ic.SQUEEZENET, True) 68 | 69 | if __name__ == '__main__': 70 | unittest.main(verbosity=2) 71 | -------------------------------------------------------------------------------- /src/aiy/vision/streaming/svg.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from collections import OrderedDict 15 | 16 | 17 | def rgb(color): 18 | return 'rgb(%s, %s, %s)' % color 19 | 20 | 21 | class Tag: 22 | NAME = None 23 | REQUIRED_ATTRS = () 24 | 25 | def __init__(self, **kwargs): 26 | self._attrs = OrderedDict() 27 | 28 | for attr in self.REQUIRED_ATTRS: 29 | if attr not in kwargs: 30 | raise ValueError('Missing attribute "%s" from tag <%s/>' % (attr, self.NAME)) 31 | 32 | for key, value in kwargs.items(): 33 | self._attrs[key.replace('_', '-')] = value 34 | 35 | @property 36 | def value(self): 37 | return None 38 | 39 | def __str__(self): 40 | sattrs = ' '.join('%s="%s"' % (name, value) for name, value in self._attrs.items()) 41 | if sattrs: 42 | sattrs = ' ' + sattrs 43 | v = self.value 44 | if v is None: 45 | return '<%s%s/>' % (self.NAME, sattrs) 46 | 47 | return '<%s%s>%s' % (self.NAME, sattrs, v, self.NAME) 48 | 49 | 50 | class TagContainer(Tag): 51 | def __init__(self, **kwargs): 52 | super().__init__(**kwargs) 53 | self._children = [] 54 | 55 | def add(self, child): 56 | self._children.append(child) 57 | return child 58 | 59 | @property 60 | def value(self): 61 | return ''.join(str(child) for child in self._children) 62 | 63 | class Svg(TagContainer): 64 | NAME = 'svg' 65 | 66 | def __init__(self, **kwargs): 67 | super().__init__(**{'xmlns':'http://www.w3.org/2000/svg', **kwargs}) 68 | 69 | 70 | class Group(TagContainer): 71 | NAME = 'g' 72 | 73 | 74 | class Line(Tag): 75 | NAME = 'line' 76 | REQUIRED_ATTRS = ('x1', 'y1', 'x2', 'y2') 77 | 78 | 79 | class Rect(Tag): 80 | NAME = 'rect' 81 | REQUIRED_ATTRS = ('x', 'y', 'width', 'height') 82 | 83 | 84 | class Circle(Tag): 85 | NAME = 'circle' 86 | REQUIRED_ATTRS = ('cx', 'cy', 'r') 87 | 88 | 89 | class Ellipse(Tag): 90 | NAME = 'ellipse' 91 | REQUIRED_ATTRS = ('cx', 'cy', 'rx', 'ry') 92 | 93 | 94 | class Text(Tag): 95 | NAME = 'text' 96 | REQUIRED_ATTRS = ('x', 'y') 97 | 98 | def __init__(self, text, **kwargs): 99 | super().__init__(**kwargs) 100 | self._text = text 101 | 102 | @property 103 | def value(self): 104 | return self._text 105 | 106 | class Path(Tag): 107 | NAME = 'path' 108 | REQUIRED_ATTRS = ('d',) 109 | -------------------------------------------------------------------------------- /src/examples/vision/face_detection_camera.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Camera inference face detection demo code. 16 | 17 | Runs continuous face detection on the VisionBonnet and prints the number of 18 | detected faces. 19 | 20 | Example: 21 | face_detection_camera.py --num_frames 10 22 | """ 23 | import argparse 24 | 25 | from picamera import PiCamera 26 | 27 | from aiy.vision.inference import CameraInference 28 | from aiy.vision.models import face_detection 29 | from aiy.vision.annotator import Annotator 30 | 31 | 32 | def avg_joy_score(faces): 33 | if faces: 34 | return sum(face.joy_score for face in faces) / len(faces) 35 | return 0.0 36 | 37 | def main(): 38 | """Face detection camera inference example.""" 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument('--num_frames', '-n', type=int, dest='num_frames', default=None, 41 | help='Sets the number of frames to run for, otherwise runs forever.') 42 | args = parser.parse_args() 43 | 44 | # Forced sensor mode, 1640x1232, full FoV. See: 45 | # https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes 46 | # This is the resolution inference run on. 47 | with PiCamera(sensor_mode=4, resolution=(1640, 1232), framerate=30) as camera: 48 | camera.start_preview() 49 | 50 | # Annotator renders in software so use a smaller size and scale results 51 | # for increased performace. 52 | annotator = Annotator(camera, dimensions=(320, 240)) 53 | scale_x = 320 / 1640 54 | scale_y = 240 / 1232 55 | 56 | # Incoming boxes are of the form (x, y, width, height). Scale and 57 | # transform to the form (x1, y1, x2, y2). 58 | def transform(bounding_box): 59 | x, y, width, height = bounding_box 60 | return (scale_x * x, scale_y * y, scale_x * (x + width), 61 | scale_y * (y + height)) 62 | 63 | with CameraInference(face_detection.model()) as inference: 64 | for result in inference.run(args.num_frames): 65 | faces = face_detection.get_faces(result) 66 | annotator.clear() 67 | for face in faces: 68 | annotator.bounding_box(transform(face.bounding_box), fill=0) 69 | annotator.update() 70 | 71 | print('#%05d (%5.2f fps): num_faces=%d, avg_joy_score=%.2f' % 72 | (inference.count, inference.rate, len(faces), avg_joy_score(faces))) 73 | 74 | camera.stop_preview() 75 | 76 | 77 | if __name__ == '__main__': 78 | main() 79 | -------------------------------------------------------------------------------- /src/examples/vision/image_classification.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2017 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Image classification library demo.""" 16 | 17 | import argparse 18 | 19 | from PIL import Image 20 | 21 | from aiy.vision.inference import ImageInference 22 | from aiy.vision.models import image_classification 23 | 24 | 25 | def main(): 26 | parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) 27 | parser.add_argument('--input', '-i', required=True, 28 | help='Input image file.') 29 | parser.add_argument('--threshold', '-t', type=float, default=0.1, 30 | help='Classification probability threshold.') 31 | parser.add_argument('--top_k', '-n', type=int, default=5, 32 | help='Max number of returned classes.') 33 | parser.add_argument('--sparse', '-s', action='store_true', default=False, 34 | help='Use sparse tensors.') 35 | parser.add_argument('--model', '-m', choices=('squeezenet', 'mobilenet'), default='mobilenet', 36 | help='Model to run.') 37 | args = parser.parse_args() 38 | 39 | # There are two models available for image classification task: 40 | # 1) MobileNet based (image_classification.MOBILENET), which has 59.9% top-1 41 | # accuracy on ImageNet; 42 | # 2) SqueezeNet based (image_classification.SQUEEZENET), which has 45.3% top-1 43 | # accuracy on ImageNet; 44 | model_type = {'squeezenet': image_classification.SQUEEZENET, 45 | 'mobilenet': image_classification.MOBILENET}[args.model] 46 | 47 | with ImageInference(image_classification.model(model_type)) as inference: 48 | image = Image.open(args.input) 49 | 50 | if args.sparse: 51 | configs = image_classification.sparse_configs(top_k=args.top_k, 52 | threshold=args.threshold, 53 | model_type=model_type) 54 | result = inference.run(image, sparse_configs=configs) 55 | classes = image_classification.get_classes_sparse(result) 56 | else: 57 | result = inference.run(image) 58 | classes = image_classification.get_classes(result, 59 | top_k=args.top_k, 60 | threshold=args.threshold) 61 | 62 | for i, (label, score) in enumerate(classes): 63 | print('Result %d: %s (prob=%f)' % (i, label, score)) 64 | 65 | 66 | if __name__ == '__main__': 67 | main() 68 | -------------------------------------------------------------------------------- /src/aiy/vision/streaming/proto/protocol.md: -------------------------------------------------------------------------------- 1 | # Streaming protocol specification 2 | 3 | Server streams data to clients. There is only one server and there can be zero 4 | or more connected clients. 5 | 6 | ## Messages from Server to Client (ClientBound) 7 | 8 | Each server streaming session must start with `ClientBound{start}` and 9 | stop with `ClientBound{stop}` messages. Interleaved `ClientBound{video}` 10 | and `ClientBound{overlay}` are allowed in between: 11 | 12 | ``` 13 | -> ClientBound{start={width=, height=}} 14 | 15 | -> ClientBound{video} or ClientBound{overlay} 16 | 17 | -> ClientBound{stop} 18 | ``` 19 | 20 | Each video message contains one or more H264 encoded NAL units. Partial NAL 21 | units are not allowed. The first NAL unit must be SPS (Sequence Parameter Set), 22 | the second one is IDR. Concatenation of all video messages should produce a 23 | valid H264 bitstream. All subsequent SPS NAL units must contain the same 24 | information as the first one. 25 | 26 | Overlay messages are allowed at any time. You can think that there are two 27 | logical streams during the session: video stream and stream of overlays. 28 | Each overlay message contains SVG image which is drawn on top of the video. 29 | 30 | ## Messages from Client to Server (ServerBound) 31 | 32 | Client can control the server by sending `ServerBound{stream_control}` messages. 33 | There are only two allowed: `ServerBound{stream_control={enabled=true}}` to 34 | start streaming and `ServerBound{stream_control={enabled=false}}` to stop 35 | streaming. Server could ignore these messages if it is already in the requested 36 | state. 37 | 38 | ``` 39 | <- ServerBound{stream_control={enabled=true}} 40 | 41 | 42 | <- ServerBound{stream_control={enabled=false}} 43 | 44 | ``` 45 | 46 | ## Example 47 | 48 | From the Server's point of view ordered by absolute time: 49 | 50 | ``` 51 | <- ServerBound{stream_control={enabled=true}} 52 | -> ClientBound{start={width=720, height=480}} 53 | -> ClientBound{overlay={svg=} 54 | -> ClientBound{video={}} # First NAL unit in video stream 55 | -> ClientBound{overlay={svg=} 56 | -> ClientBound{video={}} # Second NAL unit in video stream 57 | -> ClientBound{video={}} 58 | -> ClientBound{video={}} 59 | -> ClientBound{overlay={svg=} 60 | -> ClientBound{video={}} 61 | -> ClientBound{video={}} 62 | -> ClientBound{video={}} 63 | -> ClientBound{video={}} 64 | -> ClientBound{video={}} 65 | -> ClientBound{video={}} 66 | -> ClientBound{video={}} 67 | -> ClientBound{overlay={svg=} 68 | -> ClientBound{video={}} # The same as the first SPS NAL unit. 69 | -> ClientBound{video={}} 70 | -> ClientBound{video={}} 71 | -> ClientBound{video={}} 72 | -> ClientBound{video={}} 73 | -> ClientBound{video={}} 74 | -> ClientBound{overlay={svg=} 75 | <- ServerBound{stream_control={enabled=false}} 76 | -> ClientBound{video={}} 77 | -> ClientBound{video={}} 78 | -> ClientBound{stop} 79 | ``` 80 | 81 | ## References 82 | 83 | [Protocol Buffers](https://developers.google.com/protocol-buffers/) 84 | [H264 Specification](https://www.itu.int/rec/T-REC-H.264-201704-I) 85 | [SVG Specification](https://www.w3.org/TR/SVG11/) 86 | -------------------------------------------------------------------------------- /src/aiy/voice/tts.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | An API that performs text-to-speech. 17 | 18 | You can also use this to perform text-to-speech from the command line:: 19 | 20 | python ~/AIY-projects-python/src/aiy/voice/tts.py "hello world" 21 | 22 | """ 23 | 24 | import argparse 25 | import os 26 | import subprocess 27 | import tempfile 28 | 29 | RUN_DIR = '/run/user/%d' % os.getuid() 30 | 31 | def say(text, lang='en-US', volume=60, pitch=130, speed=100, device='default'): 32 | """ 33 | Speaks the provided text. 34 | 35 | Args: 36 | text: The text you want to speak. 37 | lang: The language to use. Supported languages are: 38 | en-US, en-GB, de-DE, es-ES, fr-FR, it-IT. 39 | volume: Volume level for the converted audio. The normal volume level is 40 | 100. Valid volume levels are between 0 (no audible output) and 500 (increasing the 41 | volume by a factor of 5). Values higher than 100 might result in degraded signal 42 | quality due to saturation effects (clipping) and is not recommended. To instead adjust 43 | the volume output of your device, enter ``alsamixer`` at the command line. 44 | pitch: The pitch level for the voice. The normal pitch level is 100, the allowed values lie 45 | between 50 (one octave lower) and 200 (one octave higher). 46 | speed: The speed of the voice. The normal speed level is 100, the allowed values lie 47 | between 20 (slowing down by a factor of 5) and 500 (speeding up by a factor of 5). 48 | device: The PCM device name. Leave as ``default`` to use the default ALSA soundcard. 49 | """ 50 | data = "%s" % \ 51 | (volume, pitch, speed, text) 52 | with tempfile.NamedTemporaryFile(suffix='.wav', dir=RUN_DIR) as f: 53 | cmd = 'pico2wave --wave %s --lang %s "%s" && aplay -q -D %s %s' % \ 54 | (f.name, lang, data, device, f.name) 55 | subprocess.check_call(cmd, shell=True) 56 | 57 | 58 | def _main(): 59 | parser = argparse.ArgumentParser(description='Text To Speech (pico2wave)') 60 | parser.add_argument('--lang', default='en-US') 61 | parser.add_argument('--volume', type=int, default=60) 62 | parser.add_argument('--pitch', type=int, default=130) 63 | parser.add_argument('--speed', type=int, default=100) 64 | parser.add_argument('--device', default='default') 65 | parser.add_argument('text', help='path to disk image file ') 66 | args = parser.parse_args() 67 | say(args.text, lang=args.lang, volume=args.volume, pitch=args.pitch, speed=args.speed, 68 | device=args.device) 69 | 70 | 71 | if __name__ == '__main__': 72 | _main() 73 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MAKEFILE_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) 2 | PYTHON=python3 3 | 4 | .PHONY: help 5 | help: 6 | @echo "make help - Show all make targets" 7 | @echo "make test-vision-images - Download vision test images" 8 | @echo "make test-vision-driver - Run vision driver tests" 9 | @echo "make test-vision-latency - Run vision latency tests" 10 | @echo "make test-vision-models - Run vision model tests" 11 | @echo "make test-vision-examples - Run vision example tests" 12 | @echo "make test-vision - Run all vision tests" 13 | @echo "make docs - Generate documentation" 14 | @echo "make docs-clean - Remove generated documentation" 15 | @echo "make docs-open - Open generated documentation" 16 | @echo "make joy-demo-start - Start JoyDemo service" 17 | @echo "make joy-demo-stop - Stop JoyDemo service" 18 | @echo "make joy-demo-restart - Restart JoyDemo service" 19 | @echo "make joy-demo-enable - Enable JoyDemo service" 20 | @echo "make joy-demo-disable - Disable JoyDemo service" 21 | @echo "make joy-demo-log - Print JoyDemo service log" 22 | @echo "make lint - Run python code linter" 23 | @echo "make pep8-diff - Show incorrect code formatting" 24 | @echo "make clean - Remove generated files" 25 | 26 | .PHONY: test-vision-images \ 27 | test-vision-driver \ 28 | test-vision-latency \ 29 | test-vision-models \ 30 | test-vision-examples \ 31 | test-vision 32 | 33 | test-vision-images: 34 | $(MAKE) -C src/tests/images 35 | 36 | VISION_DRIVER_TESTS:=src/tests/spicomm_test.py 37 | VISION_LATENCY_TESTS:=src/tests/camera_inference_latency_test.py 38 | VISION_EXAMPLE_TESTS:=src/tests/vision_examples_test.py 39 | VISION_MODEL_TESTS:=\ 40 | src/tests/engine_test.py \ 41 | src/tests/dish_classification_test.py \ 42 | src/tests/dish_detection_test.py \ 43 | src/tests/face_detection_test.py \ 44 | src/tests/image_classification_test.py \ 45 | src/tests/object_detection_test.py \ 46 | src/tests/inaturalist_classification_test.py 47 | 48 | test-vision-driver: 49 | $(PYTHON) -m unittest -v $(VISION_DRIVER_TESTS) 50 | 51 | test-vision-latency: 52 | $(PYTHON) -m unittest -v $(VISION_LATENCY_TESTS) 53 | 54 | test-vision-models: test-vision-images 55 | $(PYTHON) -m unittest -v $(VISION_MODEL_TESTS) 56 | 57 | test-vision-examples: test-vision-images 58 | $(PYTHON) -m unittest -v $(VISION_EXAMPLE_TESTS) 59 | 60 | test-vision: test-vision-images 61 | $(PYTHON) -m unittest -v \ 62 | $(VISION_DRIVER_TESTS) \ 63 | $(VISION_LATENCY_TESTS) \ 64 | $(VISION_MODEL_TESTS) \ 65 | $(VISION_EXAMPLE_TESTS) 66 | 67 | .PHONY: docs docs-clean docs-open 68 | docs: 69 | sphinx-build -b html $(MAKEFILE_DIR)/docs $(MAKEFILE_DIR)/docs/_build/html 70 | 71 | docs-clean: 72 | rm -rf $(MAKEFILE_DIR)/docs/_build 73 | 74 | docs-open: 75 | $(PYTHON) -m webbrowser -t "file://$(MAKEFILE_DIR)/docs/_build/html/index.html" 76 | 77 | .PHONY: joy-demo-log 78 | 79 | joy-demo-%: 80 | sudo systemctl $* joy_detection_demo.service 81 | 82 | joy-demo-log: 83 | sudo journalctl -u joy_detection_demo.service -b -f 84 | 85 | .PHONY: lint pep8-diff 86 | lint: 87 | find src -iname "*.py" | grep -v pb2 | xargs $(PYTHON) -m pylint --rcfile .pylintrc 88 | 89 | pep8-diff: 90 | $(PYTHON) -m autopep8 --max-line-length=100 --diff `find src -iname "*.py" | grep -v pb2` 91 | 92 | .PHONY: clean 93 | clean: 94 | rm -f $(MAKEFILE_DIR)/src/tests/images/*.jpg 95 | -------------------------------------------------------------------------------- /docs/vision.md: -------------------------------------------------------------------------------- 1 | # Vision Kit overview 2 | 3 | 5 | 6 | The AIY Vision Kit is a do-it-yourself intelligent camera built with a 7 | Raspberry Pi and the Vision Bonnet. 8 | 9 | After you assemble the kit and run the included demos, 10 | you can extend the kit with your own software and hardware. 11 | 12 | Also see the [Vision Kit assembly guide]( 13 | https://aiyprojects.withgoogle.com/vision/). 14 | 15 | ## Software 16 | 17 | To execute ML models and perform other actions with the Vision Kit, the 18 | system image includes the Python library with the following modules: 19 | 20 | * [`aiy.toneplayer`](aiy.toneplayer.html): 21 | A simple melodic music player for the piezo buzzer. 22 | * [`aiy.trackplayer`](aiy.trackplayer.html): 23 | A tracker-based music player for the piezo buzzer. 24 | * [`aiy.vision.annotator`](aiy.vision.annotator.html): 25 | An annotation library that draws overlays on the Raspberry Pi’s camera preview. 26 | * [`aiy.vision.inference`](aiy.vision.inference.html): 27 | An inference engine that communicates with the Vision Bonnet from the Raspberry 28 | Pi side. 29 | * [`aiy.vision.models`](aiy.vision.models.html): 30 | A collection of modules that perform ML inferences with specific types of image 31 | classification and object detection models. 32 | * [`aiy.board`](aiy.board.html): 33 | APIs to use the button that’s attached to the Vision Bonnet’s button connector. 34 | * [`aiy.leds`](aiy.leds.html): 35 | APIs to control certain LEDs, such as the LEDs in the button and the privacy 36 | LED. 37 | * [`aiy.pins`](aiy.pins.html): 38 | Pin definitions for the bonnet's extra GPIO pins, for use with gpiozero. 39 | 40 | 41 | ## Vision Bonnet 42 | 43 | ### Hardware 44 | 45 | * SOC: `Myriad 2450` 46 | * MCU: `ATSAMD09D14` [I²C address: `0x51`] 47 | * LED Driver: `KTD2027A` [I²C address: `0x30`] 48 | * Crypto (optional): `ATECC608A` [I²C address: `0x60`] 49 | * IMU: `BMI160` 50 | 51 | ### Drivers 52 | 53 | * MCU driver: `modinfo aiy-io-i2c` 54 | * MCU PWM driver: `modinfo pwm-aiy-io` 55 | * MCU GPIO driver: `modinfo gpio-aiy-io` 56 | * MCU ADC driver: `modinfo aiy-adc` 57 | * LED driver: `modinfo leds-ktd202x` 58 | * Software PWM driver for buzzer: `modinfo pwm-soft` 59 | * Myriad driver: `modinfo aiy-vision` 60 | 61 | To reset MCU: 62 | ``` 63 | echo 1 | sudo tee /sys/bus/i2c/devices/1-0051/reset 64 | ``` 65 | 66 | To get MCU status message (including firmware version) and last error code: 67 | ``` 68 | cat /sys/bus/i2c/devices/1-0051/{status_message,error_code} 69 | ``` 70 | 71 | ### Pinout (40-pin header) 72 | 73 | ``` 74 | 3.3V --> 1 2 <-- 5V 75 | I2C_SDA --> 3 4 <-- 5V 76 | I2C_SCL --> 5 6 <-- GND 77 | 7 8 78 | GND --> 9 10 79 | 11 12 80 | 13 14 <-- GND 81 | (GPIO_22) BUZZER_GPIO --> 15 16 <-- BUTTON_GPIO (GPIO_23) 82 | 3.3V --> 17 18 83 | SPI_MOSI --> 19 20 <-- GND 84 | SPI_MISO --> 21 22 85 | SPI_SCLK --> 23 24 <-- SPI_CE_MRD 86 | GND --> 25 26 87 | ID_SDA --> 27 28 <-- ID_SCL 88 | 29 30 <-- GND 89 | PI_TO_MRD_IRQ --> 31 32 90 | MRD_TO_PI_IRQ --> 33 34 <-- GND 91 | 35 36 92 | MRD_UNUSED --> 37 38 93 | GND --> 39 40 94 | ``` 95 | 96 | Also see the [Vision Bonnet on pinout.xyz](https://pinout.xyz/pinout/aiy_vision_bonnet). 97 | 98 | ## Troubleshooting 99 | 100 | See the [Vision Kit help](https://aiyprojects.withgoogle.com/help#vision-kit). 101 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Change log 2 | 3 | This page describes the changes in each release. 4 | 5 | To update your kit, see the [system updates guide][system-updates]. 6 | All system images can be downloaded from the [GitHub releases page][github-releases]. 7 | 8 | ## AIY Kits Release 2021-04-02 9 | 10 | Compatible with Voice HAT, Voice Bonnet, and Vision Bonnet. 11 | 12 | Based on the [Raspberry Pi OS with desktop][raspberry-pi-os] from March 4th 2021. 13 | 14 | **Fixes** 15 | 16 | * Sound driver compilation errors on the latest Raspberry Pi OS 17 | * Upgrade Colab TF version to 1.13.1 18 | * Driver source code added to this repository 19 | 20 | ## AIY Kits Release 2020-11-20 21 | 22 | Compatible with Voice HAT, Voice Bonnet, and Vision Bonnet. 23 | 24 | Based on the [Raspberry Pi OS with desktop][raspberry-pi-os] from August 20th 2020. 25 | 26 | **Fixes** 27 | 28 | * Driver compilation errors on the latest Raspberry Pi OS 29 | * HACKING.md instructions for Vision Bonnet and Voice HAT 30 | 31 | ## AIY Kits Release 2019-11-13 32 | 33 | Compatible with Voice HAT, Voice Bonnet, and Vision Bonnet. 34 | 35 | Based on the [Raspbian Buster with desktop][raspbian]. 36 | 37 | **Fixes** 38 | 39 | * Fix driver compilation errors on the latest Raspbian 40 | * Fix HACKING.md instructions 41 | * Fix gpiozero integration 42 | 43 | **Improvements** 44 | 45 | * Debian packages are now hosted at https://packages.cloud.google.com/ 46 | * Original Raspbian image is modified in the minimal way (no unnecessary packages) 47 | 48 | ## AIY Kits Release 2018-11-16 49 | 50 | Compatible with Voice HAT, Voice Bonnet, and Vision Bonnet. 51 | 52 | **Fixes** 53 | 54 | * Fix tts engine click sound 55 | * Fix `assistant_grpc_demo.service`: add DISPLAY environment variable and proper 56 | systemd dependencies 57 | * Fix various linter findings: python3 compatibility, wrong variable names, etc. 58 | 59 | **Improvements** 60 | 61 | * New `board.py` to access button and LED on all boards 62 | * New audio API in `voice/audio.py` 63 | * Direct python support for iNaturalist models 64 | * Load anchors/labels directly from text files 65 | * Add `get_inference_state()` and `resest()` to Vision Bonnet protocol 66 | * Add Voice HAT schematic in `docs` folder 67 | * Add sparse representation for output tensors to increase data transfer rate 68 | * SVG-image overlay for video streaming (experimental) 69 | 70 | ## AIY Kits Release 2018-08-03 71 | 72 | Compatible with Voice HAT, Voice Bonnet, and Vision Bonnet. 73 | 74 | **Fixes** 75 | 76 | * Fix PulseAudio infinite loop with Voice Bonnet 77 | * Fix PulseAudio volume control 78 | * Fix gpiozero LED on/off bug 79 | * Fix local USB networking on macOS, no driver required 80 | * Fix check_audio.py 81 | 82 | **Improvements** 83 | 84 | * Add Makefile for common shortcuts 85 | * Add vision unit tests for all models and examples 86 | * Add video streaming support (experimental) 87 | * Add Google Cloud IoT support (experimental) 88 | * Add more documentation (pinouts, drivers, troubleshooting, etc.) 89 | * Add new code examples and update existing ones 90 | * Add CHANGES.md to track release changes 91 | * Remove unnecessary files (e.g. ALSA configs) 92 | * Update vision driver to support mmap syscall 93 | * Update sound driver to support latest Raspbian image 94 | * Update HACKING.md 95 | 96 | ## AIY Kits Release 2018-04-13 97 | 98 | Compatible with Voice HAT, Voice Bonnet, and Vision Bonnet. 99 | 100 | ## AIY Kits Release 2018-02-21 101 | 102 | Compatible with Voice HAT, Voice Bonnet, and Vision Bonnet. 103 | 104 | ## AIY Kits Release 2017-12-18 105 | 106 | Compatible with Voice HAT and Vision Bonnet. 107 | 108 | ## VoiceKit Classic Image 2017-09-11 109 | 110 | Compatible with Voice HAT. 111 | 112 | [github-releases]: https://github.com/google/aiyprojects-raspbian/releases 113 | [system-updates]: https://github.com/google/aiyprojects-raspbian/blob/aiyprojects/HACKING.md 114 | [raspbian]: https://www.raspberrypi.org/downloads/raspbian/ 115 | [raspberry-pi-os]: https://www.raspberrypi.org/software/operating-systems/ 116 | -------------------------------------------------------------------------------- /src/tests/inaturalist_classification_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import unittest 15 | 16 | from aiy.vision.inference import ImageInference 17 | from aiy.vision.models import inaturalist_classification as m 18 | from .test_util import define_test_case, TestImage 19 | 20 | LILY_CLASSES = ( 21 | ('Lilium canadense (Canada lily)', 0.60107), 22 | ("Lilium superbum (Turk's-Cap lily)", 0.07489), 23 | ('Lilium parvum (Sierra Tiger Lily)', 0.05954), 24 | ('Lilium philadelphicum (Wood Lily)', 0.01994), 25 | ('Calochortus venustus (Butterfly Mariposa Lily)', 0.01755) 26 | ) 27 | 28 | BEE_CLASSES = ( 29 | ('Apis mellifera (Honey Bee)', 0.65234), 30 | ('Ancistrocerus gazella (European Tube Wasp)', 0.00761), 31 | ('Vespula vulgaris (Common Yellowjacket)', 0.00698), 32 | ('Vespula germanica (German Wasp)', 0.00618), 33 | ('Eristalis tenax (Drone Fly)', 0.00440) 34 | ) 35 | 36 | SPARROW_CLASSES = ( 37 | ('Passer domesticus (House Sparrow)', 0.91943), 38 | ('Passer italiae (Italian Sparrow)', 0.01279), 39 | ('Passer montanus (Eurasian Tree Sparrow)', 0.00241), 40 | ('Sylvia communis (Greater Whitethroat)', 0.00198), 41 | ('Spiza americana (Dickcissel)', 0.00166), 42 | ) 43 | 44 | class InaturalistClassificationTest: 45 | TOP_K = 5 46 | THRESHOLD = 0.0 47 | 48 | def __init__(self, model_type, sparse): 49 | self.model_type = model_type 50 | self.sparse = sparse 51 | 52 | self.image_file = {m.PLANTS: 'lily.jpg', 53 | m.INSECTS: 'bee.jpg', 54 | m.BIRDS: 'sparrow.jpg'}[model_type] 55 | 56 | self.classes = {m.PLANTS: LILY_CLASSES, 57 | m.INSECTS: BEE_CLASSES, 58 | m.BIRDS: SPARROW_CLASSES}[model_type] 59 | 60 | def check(self, classes): 61 | self.assertEqual(len(classes), len(self.classes)) 62 | for (label1, prob1), (label2, prob2) in zip(classes, self.classes): 63 | self.assertEqual(label1, label2) 64 | self.assertAlmostEqual(prob1, prob2, delta=0.0001) 65 | 66 | def test_classification(self): 67 | with TestImage(self.image_file) as image: 68 | with ImageInference(m.model(self.model_type)) as inference: 69 | if self.sparse: 70 | configs = m.sparse_configs(top_k=self.TOP_K, 71 | threshold=self.THRESHOLD, 72 | model_type=self.model_type) 73 | result = inference.run(image, sparse_configs=configs) 74 | classes = m.get_classes_sparse(result) 75 | else: 76 | result = inference.run(image) 77 | classes = m.get_classes(result, top_k=self.TOP_K, threshold=self.THRESHOLD) 78 | 79 | self.check(classes) 80 | 81 | 82 | define_test_case(globals(), InaturalistClassificationTest, m.PLANTS, False) 83 | define_test_case(globals(), InaturalistClassificationTest, m.PLANTS, True) 84 | define_test_case(globals(), InaturalistClassificationTest, m.INSECTS, False) 85 | define_test_case(globals(), InaturalistClassificationTest, m.INSECTS, True) 86 | define_test_case(globals(), InaturalistClassificationTest, m.BIRDS, False) 87 | define_test_case(globals(), InaturalistClassificationTest, m.BIRDS, True) 88 | 89 | 90 | if __name__ == '__main__': 91 | unittest.main(verbosity=2) 92 | -------------------------------------------------------------------------------- /src/aiy/vision/models/inaturalist_classification.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | API for detecting plants, insects, and birds from the iNaturalist dataset. 17 | """ 18 | 19 | from collections import namedtuple 20 | 21 | from aiy.vision.inference import ModelDescriptor, ThresholdingConfig 22 | from aiy.vision.models import utils 23 | 24 | PLANTS = 'inaturalist_plants' 25 | INSECTS = 'inaturalist_insects' 26 | BIRDS = 'inaturalist_birds' 27 | 28 | class Model(namedtuple('Model', ('labels', 29 | 'compute_graph_file', 30 | 'input_shape', 31 | 'input_normalizer', 32 | 'output_name'))): 33 | def compute_graph(self): 34 | return utils.load_compute_graph(self.compute_graph_file) 35 | 36 | _MODELS = { 37 | PLANTS: Model(labels=utils.load_labels('mobilenet_v2_192res_1.0_inat_plant_labels.txt'), 38 | compute_graph_file='mobilenet_v2_192res_1.0_inat_plant.binaryproto', 39 | input_shape=(1, 192, 192, 3), 40 | input_normalizer=(128.0, 128.0), 41 | output_name='prediction'), 42 | INSECTS: Model(labels=utils.load_labels('mobilenet_v2_192res_1.0_inat_insect_labels.txt'), 43 | compute_graph_file='mobilenet_v2_192res_1.0_inat_insect.binaryproto', 44 | input_shape=(1, 192, 192, 3), 45 | input_normalizer=(128.0, 128.0), 46 | output_name='prediction'), 47 | BIRDS: Model(labels=utils.load_labels('mobilenet_v2_192res_1.0_inat_bird_labels.txt'), 48 | compute_graph_file='mobilenet_v2_192res_1.0_inat_bird.binaryproto', 49 | input_shape=(1, 192, 192, 3), 50 | input_normalizer=(128.0, 128.0), 51 | output_name='prediction'), 52 | } 53 | 54 | 55 | def sparse_configs(model_type, top_k=None, threshold=0.0): 56 | this_model = _MODELS[model_type] 57 | num_labels = len(this_model.labels) 58 | return { 59 | this_model.output_name: ThresholdingConfig(logical_shape=[num_labels], 60 | threshold=threshold, 61 | top_k=num_labels if top_k is None else top_k, 62 | to_ignore=[]) 63 | } 64 | 65 | 66 | def model(model_type): 67 | this_model = _MODELS[model_type] 68 | return ModelDescriptor(name=model_type, 69 | input_shape=this_model.input_shape, 70 | input_normalizer=this_model.input_normalizer, 71 | compute_graph=this_model.compute_graph()) 72 | 73 | 74 | def get_classes(result, top_k=None, threshold=0.0): 75 | assert len(result.tensors) == 1 76 | 77 | this_model = _MODELS[result.model_name] 78 | labels = this_model.labels 79 | 80 | tensor = result.tensors[this_model.output_name] 81 | probs, shape = tensor.data, tensor.shape 82 | assert shape.depth == len(labels) 83 | pairs = [pair for pair in enumerate(probs) if pair[1] > threshold] 84 | pairs = sorted(pairs, key=lambda pair: pair[1], reverse=True) 85 | pairs = pairs[0:top_k] 86 | return [('/'.join(labels[index]), prob) for index, prob in pairs] 87 | 88 | 89 | def get_classes_sparse(result): 90 | assert len(result.tensors) == 1 91 | 92 | this_model = _MODELS[result.model_name] 93 | labels = this_model.labels 94 | 95 | tensor = result.tensors[this_model.output_name] 96 | indices, probs = tuple(tensor.indices), tuple(tensor.data) 97 | pairs = [(index.values[0], prob) for index, prob in zip(indices, probs)] 98 | pairs = sorted(pairs, key=lambda pair: pair[1], reverse=True) 99 | return [('/'.join(labels[index]), prob) for index, prob in pairs] 100 | -------------------------------------------------------------------------------- /src/aiy/vision/streaming/assets/ws_client.js: -------------------------------------------------------------------------------- 1 | function createPlayer(width, height, streamControl) { 2 | var player = new Player({ 3 | useWorker: true, 4 | workerFile: "broadway/Decoder.js", 5 | reuseMemory: true, 6 | webgl: "auto", 7 | size: { 8 | width: width, 9 | height: height, 10 | } 11 | }); 12 | 13 | var frameCount = 0 14 | player.onPictureDecoded = function(data) { 15 | if (frameCount == 0) { 16 | console.log("First frame decoded"); 17 | } 18 | frameCount++; 19 | }; 20 | 21 | var container = document.getElementById("container"); 22 | 23 | var cropDiv = document.createElement("div"); 24 | cropDiv.style.overflow = "hidden"; 25 | cropDiv.style.position = "absolute"; 26 | cropDiv.style.width = width + "px"; 27 | cropDiv.style.height = height + "px"; 28 | cropDiv.appendChild(player.canvas); 29 | container.appendChild(cropDiv); 30 | 31 | var canvas = document.createElement("canvas"); 32 | canvas.id = "overlay" 33 | canvas.style.position = "absolute"; 34 | canvas.width = width; 35 | canvas.height = height; 36 | container.appendChild(canvas); 37 | 38 | var top = (height + 10) + "px"; 39 | 40 | var startButton = document.createElement("button"); 41 | startButton.style.position = "relative"; 42 | startButton.style.top = top; 43 | startButton.style.marginRight = "10px" 44 | startButton.innerHTML = "Start" 45 | startButton.onclick = function() { 46 | console.log('Start clicked!') 47 | streamControl(true); 48 | } 49 | container.appendChild(startButton); 50 | 51 | var stopButton = document.createElement("button"); 52 | stopButton.style.position = "relative"; 53 | stopButton.style.top = top; 54 | stopButton.style.marginRight = "10px" 55 | stopButton.innerHTML = "Stop" 56 | stopButton.onclick = function() { 57 | console.log('Stop clicked!') 58 | streamControl(false); 59 | } 60 | container.appendChild(stopButton); 61 | 62 | var licenseLink = document.createElement("a"); 63 | licenseLink.appendChild(document.createTextNode("Open source licenses")); 64 | licenseLink.title = "LICENSE"; 65 | licenseLink.href = "broadway/LICENSE"; 66 | licenseLink.target= "_blank"; 67 | licenseLink.style.position = "relative"; 68 | licenseLink.style.top = top; 69 | container.appendChild(licenseLink); 70 | 71 | return player 72 | } 73 | 74 | window.onload = function() { 75 | protobuf.load("messages.proto", function(err, root) { 76 | if (err) 77 | throw err; 78 | 79 | var ClientBound = root.lookupType("ClientBound"); 80 | var ServerBound = root.lookupType("ServerBound") 81 | 82 | function streamControl(enabled) { 83 | serverBound = ServerBound.create({streamControl: {enabled:enabled}}); 84 | socket.send(ServerBound.encode(serverBound).finish()); 85 | } 86 | 87 | var player = null; 88 | var socket = new WebSocket("ws://" + window.location.host + "/stream"); 89 | socket.binaryType = "arraybuffer"; 90 | 91 | socket.onopen = function(event) { 92 | console.log("Socket connected."); 93 | streamControl(true); 94 | }; 95 | 96 | socket.onclose = function(event) { 97 | console.log("Socket closed."); 98 | }; 99 | 100 | socket.onmessage = function(event) { 101 | var clientBound = ClientBound.decode(new Uint8Array(event.data)) 102 | switch (clientBound.message) { 103 | case 'start': 104 | console.log('Starting...') 105 | start = clientBound.start; 106 | if (player == null) { 107 | console.log('Starting...') 108 | player = createPlayer(start.width, start.height, streamControl); 109 | console.log("Started: " + start.width + "x" + start.height); 110 | } 111 | break; 112 | case 'video': 113 | player.decode(clientBound.video.data); 114 | break; 115 | case 'overlay': 116 | var canvas = document.getElementById("overlay"); 117 | var ctx = canvas.getContext("2d"); 118 | var img = new Image(); 119 | img.onload = function() { 120 | ctx.clearRect(0, 0, canvas.width, canvas.height); 121 | ctx.drawImage(img, 0, 0, canvas.width, canvas.height); 122 | } 123 | img.src = "data:image/svg+xml;charset=utf-8," + clientBound.overlay.svg; 124 | break; 125 | case 'stop': 126 | console.log("Stopped."); 127 | break; 128 | } 129 | }; 130 | }); 131 | }; 132 | -------------------------------------------------------------------------------- /src/aiy/vision/models/image_classification.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """API for Image Classification tasks.""" 15 | 16 | from aiy.vision.inference import ModelDescriptor, ThresholdingConfig 17 | from aiy.vision.models import utils 18 | 19 | # There are two models in our repository that can do image classification. One 20 | # based on MobileNet model structure, the other based on SqueezeNet model 21 | # structure. 22 | # 23 | # MobileNet based model has 59.9% top-1 accuracy on ImageNet. 24 | # SqueezeNet based model has 45.3% top-1 accuracy on ImageNet. 25 | MOBILENET = 'image_classification_mobilenet' 26 | SQUEEZENET = 'image_classification_squeezenet' 27 | 28 | _COMPUTE_GRAPH_NAME_MAP = { 29 | MOBILENET: 'mobilenet_v1_160res_0.5_imagenet.binaryproto', 30 | SQUEEZENET: 'squeezenet_160res_5x5_0.75.binaryproto', 31 | } 32 | 33 | _OUTPUT_TENSOR_NAME_MAP = { 34 | MOBILENET: 'MobilenetV1/Predictions/Softmax', 35 | SQUEEZENET: 'Prediction', 36 | } 37 | 38 | _CLASSES = utils.load_labels('mobilenet_v1_160res_0.5_imagenet_labels.txt') 39 | 40 | def sparse_configs(top_k=len(_CLASSES), threshold=0.0, model_type=MOBILENET): 41 | name = _OUTPUT_TENSOR_NAME_MAP[model_type] 42 | return { 43 | name: ThresholdingConfig(logical_shape=[len(_CLASSES)], 44 | threshold=threshold, 45 | top_k=top_k, 46 | to_ignore=[]) 47 | } 48 | 49 | def model(model_type=MOBILENET): 50 | return ModelDescriptor( 51 | name=model_type, 52 | input_shape=(1, 160, 160, 3), 53 | input_normalizer=(128.0, 128.0), 54 | compute_graph=utils.load_compute_graph(_COMPUTE_GRAPH_NAME_MAP[model_type])) 55 | 56 | 57 | def _get_probs(result): 58 | assert len(result.tensors) == 1 59 | tensor = result.tensors[_OUTPUT_TENSOR_NAME_MAP[result.model_name]] 60 | assert utils.shape_tuple(tensor.shape) == (1, 1, 1, len(_CLASSES)) 61 | return tuple(tensor.data) 62 | 63 | 64 | def get_classes(result, top_k=None, threshold=0.0): 65 | """Converts image classification model output to list of detected objects. 66 | 67 | Args: 68 | result: output tensor from image classification model. 69 | top_k: int; max number of objects to return. 70 | threshold: float; min probability of each returned object. 71 | 72 | Returns: 73 | A list of (class_name: string, probability: float) pairs ordered by 74 | probability from highest to lowest. The number of pairs is not greater than 75 | top_k. Each probability is greater than threshold. For 76 | example: 77 | 78 | [('Egyptian cat', 0.767578) 79 | ('tiger cat, 0.163574) 80 | ('lynx/catamount', 0.039795)] 81 | """ 82 | probs = _get_probs(result) 83 | pairs = [pair for pair in enumerate(probs) if pair[1] > threshold] 84 | pairs = sorted(pairs, key=lambda pair: pair[1], reverse=True) 85 | pairs = pairs[0:top_k] 86 | return [('/'.join(_CLASSES[index]), prob) for index, prob in pairs] 87 | 88 | 89 | def _get_pairs(result): 90 | assert len(result.tensors) == 1 91 | tensor = result.tensors[_OUTPUT_TENSOR_NAME_MAP[result.model_name]] 92 | indices = tuple(tensor.indices) 93 | data = tuple(tensor.data) 94 | return [(index.values[0], prob) for index, prob in zip(indices, data)] 95 | 96 | 97 | def get_classes_sparse(result): 98 | """Converts sparse image classification model output to list of detected objects. 99 | 100 | Args: 101 | result: sparse output tensor from image classification model. 102 | 103 | Returns: 104 | A list of (class_name: string, probability: float) pairs ordered by 105 | probability from highest to lowest. 106 | For example: 107 | 108 | [('Egyptian cat', 0.767578) 109 | ('tiger cat, 0.163574) 110 | """ 111 | pairs = _get_pairs(result) 112 | pairs = sorted(pairs, key=lambda pair: pair[1], reverse=True) 113 | return [('/'.join(_CLASSES[index]), prob) for index, prob in pairs] 114 | -------------------------------------------------------------------------------- /checkpoints/check_audio.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2017 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | """Checks that the AIY sound card is working.""" 18 | 19 | import os 20 | import tempfile 21 | import time 22 | import traceback 23 | 24 | from aiy.voice.audio import AudioFormat, play_wav, record_file 25 | 26 | AIY_CARDS = { 27 | 'sndrpigooglevoi': 'Voice HAT (v1)', 28 | 'aiyvoicebonnet': 'Voice Bonnet (v2)' 29 | } 30 | 31 | TEST_SOUND_PATH = '/usr/share/sounds/alsa/Front_Center.wav' 32 | 33 | RECORD_DURATION_SECONDS = 3 34 | 35 | ERROR_NO_SOUND_CARDS = ''' 36 | You do not have any sound cards installed. Please check that AIY sound card is 37 | properly connected. 38 | 39 | For some Voice HATs (not Voice Bonnets!) you need to add the following line 40 | to /boot/config.txt: 41 | 42 | dtoverlay=googlevoicehat-soundcard 43 | 44 | To do that simply run from a separate terminal: 45 | 46 | echo "dtoverlay=googlevoicehat-soundcard" | sudo tee -a /boot/config.txt 47 | 48 | ''' 49 | 50 | ERROR_NO_AIY_SOUND_CARDS = ''' 51 | You have sound cards installed but you do not have any AIY ones. Please check 52 | that AIY sound card is properly connected. 53 | ''' 54 | 55 | ERROR_NOT_A_FIRST_SOUND_CARD = ''' 56 | Your AIY sound card is not a first sound device. The voice recognizer may be 57 | unable to find it. Please try removing other sound drivers. 58 | ''' 59 | 60 | ERROR_NO_SPEAKER_SOUND = ''' 61 | There may be a problem with your speaker. Check that it is connected properly. 62 | ''' 63 | 64 | ERROR_NO_RECORDED_SOUND = ''' 65 | There may be a problem with your microphone. Check that it is connected 66 | properly. 67 | ''' 68 | 69 | def ask(prompt): 70 | answer = input('%s (y/n) ' % prompt).lower() 71 | while answer not in ('y', 'n'): 72 | answer = input('Please enter y or n: ') 73 | return answer == 'y' 74 | 75 | def error(message): 76 | print(message.strip()) 77 | 78 | def find_sound_cards(max_count=16): 79 | cards = [] 80 | for i in range(max_count): 81 | path = '/proc/asound/card%d/id' % i 82 | if not os.path.exists(path): 83 | break 84 | with open(path) as f: 85 | cards.append(f.read().strip()) 86 | return cards 87 | 88 | 89 | def check_sound_card_present(): 90 | cards = find_sound_cards() 91 | if not cards: 92 | error(ERROR_NO_SOUND_CARDS) 93 | return False 94 | 95 | aiy_cards = set.intersection(set(cards), AIY_CARDS.keys()) 96 | if len(aiy_cards) != 1: 97 | error(ERROR_NO_AIY_SOUND_CARDS) 98 | return False 99 | 100 | for card in aiy_cards: 101 | index = cards.index(card) 102 | print('You have %s installed at index %d!' % (AIY_CARDS[card], index)) 103 | if index != 0: 104 | error(ERROR_NOT_A_FIRST_SOUND_CARD) 105 | return False 106 | 107 | return True 108 | 109 | def check_speaker_works(): 110 | print('Playing a test sound...') 111 | play_wav(TEST_SOUND_PATH) 112 | 113 | if not ask('Did you hear the test sound?'): 114 | error(ERROR_NO_SPEAKER_SOUND) 115 | return False 116 | 117 | return True 118 | 119 | def check_microphone_works(): 120 | with tempfile.NamedTemporaryFile() as f: 121 | input('When you are ready, press Enter and say "Testing, 1 2 3"...') 122 | print('Recording for %d seconds...' % RECORD_DURATION_SECONDS) 123 | 124 | record_file(AudioFormat.CD, filename=f.name, filetype='wav', 125 | wait=lambda: time.sleep(RECORD_DURATION_SECONDS)) 126 | print('Playing back recorded audio...') 127 | play_wav(f.name) 128 | 129 | if not ask('Did you hear your own voice?'): 130 | error(ERROR_NO_RECORDED_SOUND) 131 | return False 132 | 133 | return True 134 | 135 | def main(): 136 | if not check_sound_card_present(): 137 | return 138 | 139 | if not check_speaker_works(): 140 | return 141 | 142 | if not check_microphone_works(): 143 | return 144 | 145 | print('AIY sound card seems to be working!') 146 | 147 | if __name__ == '__main__': 148 | try: 149 | main() 150 | except: 151 | traceback.print_exc() 152 | finally: 153 | input('Press Enter to close...') 154 | -------------------------------------------------------------------------------- /docs/voice.md: -------------------------------------------------------------------------------- 1 | # Voice Kit overview 2 | 3 | 4 | 5 | The AIY Voice Kit is a do-it-yourself intelligent speaker built with a 6 | Raspberry Pi and the Voice Bonnet (or Voice HAT if using the V1 Voice Kit). 7 | 8 | After you assemble the kit and run the included demos, 9 | you can extend the kit with your own software and hardware. 10 | 11 | Also see the [Voice Kit assembly guide](https://aiyprojects.withgoogle.com/voice/). 12 | 13 | ## Software 14 | 15 | To convert speech to text, and perform other actions with the Voice Kit, the 16 | system image includes Python library with the 17 | following modules: 18 | 19 | * [`aiy.voice.audio`](aiy.voice.audio.html): 20 | APIs to record and play audio files. 21 | * [`aiy.voice.tts`](aiy.voice.tts.html): 22 | An API that performs text-to-speech. 23 | * [`aiy.board`](aiy.board.html): 24 | APIs to use the button that’s attached to the Voice Bonnet’s button connector. 25 | * [`aiy.leds`](aiy.leds.html): 26 | APIs to control certain LEDs, such as the LEDs in the button and the privacy 27 | LED. 28 | * [`aiy.pins`](aiy.pins.html): 29 | Pin definitions for the bonnet's extra GPIO pins, for use with gpiozero. 30 | 31 | ## Voice Bonnet (Voice Kit V2) 32 | 33 | ### Hardware 34 | 35 | * Audio Codec: `ALC5645` [I²C address: `0x1A`] 36 | * MCU: `ATSAMD09D14` [I²C address: `0x52`] 37 | * LED Driver: `KTD2027B` [I²C address: `0x31`] 38 | * Crypto (optional): `ATECC608A` [I²C address: `0x62`] 39 | * Microphone: `SPH1642HT5H-1` x 2 40 | 41 | ### Drivers 42 | 43 | * MCU driver: `modinfo aiy-io-i2c` 44 | * MCU PWM driver: `modinfo pwm-aiy-io` 45 | * MCU GPIO driver: `modinfo gpio-aiy-io` 46 | * MCU ADC driver: `modinfo aiy-adc` 47 | * LED driver: `modinfo leds-ktd202x` 48 | * Software PWM driver for buzzer: `modinfo pwm-soft` 49 | * Sound drivers: `modinfo rl6231 rt5645 snd_aiy_voicebonnet` 50 | 51 | ### Pinout (40-pin header) 52 | 53 | ``` 54 | 3.3V --> 1 2 <-- 5V 55 | 3 4 <-- 5V 56 | 5 6 <-- GND 57 | 7 8 58 | GND --> 9 10 59 | 11 12 <-- I2S_BCLK 60 | 13 14 <-- GND 61 | 15 16 <-- BUTTON_GPIO (GPIO_23) 62 | 3.3V --> 17 18 63 | 19 20 <-- GND 64 | 21 22 <-- LED_GPIO (GPIO_25) 65 | 23 24 66 | GND --> 25 26 67 | ID_SDA --> 27 28 <-- ID_SCL 68 | 29 30 <-- GND 69 | 31 32 70 | 33 34 <-- GND 71 | I2S_LRCLK --> 35 36 <-- AMP_ENABLE 72 | 37 38 <-- I2S_DIN 73 | GND --> 39 40 <-- I2S_DOUT 74 | ``` 75 | 76 | Also see the [Voice Bonnet on pinout.xyz](https://pinout.xyz/pinout/aiy_voice_bonnet). 77 | 78 | 79 | ## Voice HAT (Voice Kit V1) 80 | 81 | ### Hardware 82 | 83 | * Audio Amplifier: `MAX98357A` 84 | * Microphone: `ICS-43432` x 2 85 | 86 | ### Schematics 87 | 88 | * [Main Board](https://github.com/google/aiyprojects-raspbian/blob/aiyprojects/schematics/voice_hat/voice_hat.pdf) 89 | * [Microphone Board](https://github.com/google/aiyprojects-raspbian/blob/aiyprojects/schematics/voice_hat/voice_hat_mic.pdf) 90 | 91 | ### Drivers 92 | 93 | * [googlevoicehat-codec.c](https://github.com/raspberrypi/linux/blob/rpi-4.14.y/sound/soc/bcm/googlevoicehat-codec.c) 94 | * [googlevoicehat-soundcard.c](https://github.com/raspberrypi/linux/blob/rpi-4.14.y/sound/soc/bcm/googlevoicehat-soundcard.c) 95 | * [googlevoicehat-soundcard-overlay.dts](https://github.com/raspberrypi/linux/blob/rpi-4.14.y/arch/arm/boot/dts/overlays/googlevoicehat-soundcard-overlay.dts) 96 | 97 | Manual overlay load: 98 | ``` 99 | sudo dtoverlay googlevoicehat-soundcard 100 | ``` 101 | 102 | Load overlay on each boot: 103 | ``` 104 | echo "dtoverlay=googlevoicehat-soundcard" | sudo tee -a /boot/config.txt 105 | ``` 106 | 107 | ### Pinout (40-pin header) 108 | 109 | ``` 110 | 3.3V --> 1 2 <-- 5V 111 | I2C_SDA --> 3 4 <-- 5V 112 | I2C_SCL --> 5 6 <-- GND 113 | 7 8 114 | GND --> 9 10 115 | 11 12 <-- I2S_BCLK 116 | 13 14 <-- GND 117 | 15 16 <-- BUTTON_GPIO (GPIO_23) 118 | 3.3V --> 17 18 119 | 19 20 <-- GND 120 | 21 22 121 | 23 24 122 | GND --> 25 26 123 | ID_SDA --> 27 28 <-- ID_SCL 124 | 29 30 <-- GND 125 | 31 32 126 | 33 34 <-- GND 127 | I2S_LRCLK --> 35 36 128 | 37 38 <-- I2S_DIN 129 | GND --> 39 40 <-- I2S_DOUT 130 | ``` 131 | 132 | Also see the [Voice HAT on pinout.xyz](https://pinout.xyz/pinout/voice_hat). 133 | 134 | ## Troubleshooting 135 | 136 | See the [Voice Kit help](https://aiyprojects.withgoogle.com/help#voice-kit). 137 | -------------------------------------------------------------------------------- /src/tests/vision_examples_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import signal 3 | import subprocess 4 | import sys 5 | import time 6 | import unittest 7 | 8 | from aiy.vision.inference import InferenceEngine 9 | 10 | from .test_util import test_image_path 11 | 12 | ENV = { 13 | 'PYTHONUNBUFFERED': '1', 14 | 'PYTHONPATH': os.path.join(os.path.dirname(__file__), '..'), 15 | } 16 | 17 | def model_path(name): 18 | return os.path.join('/home/pi/models', name) 19 | 20 | def example_path(name): 21 | p = os.path.join(os.path.dirname(__file__), '..', 'examples', 'vision', name) 22 | return os.path.abspath(p) 23 | 24 | def wait_terminated(process, timeout, poll_interval=1.0): 25 | start = time.monotonic() 26 | while time.monotonic() - start < timeout: 27 | code = process.poll() 28 | if code is not None: 29 | return code 30 | time.sleep(poll_interval) 31 | return None 32 | 33 | class VisionExamplesTest(unittest.TestCase): 34 | def setUp(self): 35 | with InferenceEngine() as engine: 36 | engine.reset() 37 | 38 | def execute(self, args, timeout): 39 | file, *rest = args 40 | cmd = [sys.executable, example_path(file)] + rest 41 | print(cmd) 42 | process = subprocess.Popen(cmd, shell=False, env=ENV) 43 | code = wait_terminated(process, timeout) 44 | if code is not None: 45 | self.assertEqual(0, code) 46 | return 47 | 48 | print('Interrupting process (Control-C)') 49 | os.kill(process.pid, signal.SIGINT) 50 | if wait_terminated(process, timeout=5.0) is None: 51 | print('Terminating process') 52 | os.kill(process.pid, signal.SIGTERM) 53 | if wait_terminated(process, timeout=5.0) is None: 54 | print('Killing process') 55 | os.kill(process.pid, signal.SIGKILL) 56 | 57 | process.wait() 58 | self.fail('Process did not finish in time: %s' % timeout) 59 | 60 | def test_dish_classification(self): 61 | image = test_image_path('hotdog.jpg') 62 | self.execute(['dish_classification.py', '--input', image], timeout=60.0) 63 | 64 | def test_dish_detection(self): 65 | image = test_image_path('hotdog.jpg') 66 | self.execute(['dish_detection.py', '--input', image], timeout=65.0) 67 | 68 | def test_face_detection(self): 69 | image = test_image_path('faces.jpg') 70 | self.execute(['face_detection.py', '--input', image], timeout=45.0) 71 | 72 | def test_face_detection_camera(self): 73 | self.execute(['face_detection_camera.py', '--num_frames', '100'], timeout=45.0) 74 | 75 | def test_face_detection_raspivid(self): 76 | self.execute(['face_detection_raspivid.py', '--num_frames', '100'], timeout=45.0) 77 | 78 | def test_image_classification_mobilenet(self): 79 | image = test_image_path('dog.jpg') 80 | self.execute(['image_classification.py', '--model', 'mobilenet', '--input', image], 81 | timeout=45.0) 82 | 83 | def test_image_classification_squeezenet(self): 84 | image = test_image_path('dog.jpg') 85 | self.execute(['image_classification.py', '--model', 'squeezenet', '--input', image], 86 | timeout=45.0) 87 | 88 | def test_image_classification_camera(self): 89 | self.execute(['image_classification_camera.py', '--num_frames', '100'], timeout=45.0) 90 | 91 | def test_inaturalist_classification_plants(self): 92 | image = test_image_path('lily.jpg') 93 | self.execute(['inaturalist_classification.py', '--model', 'plants', '--input', image], 94 | timeout=45.0) 95 | 96 | def test_inaturalist_classification_insects(self): 97 | image = test_image_path('bee.jpg') 98 | self.execute(['inaturalist_classification.py', '--model', 'insects', '--input', image], 99 | timeout=45.0) 100 | 101 | def test_inaturalist_classification_birds(self): 102 | image = test_image_path('sparrow.jpg') 103 | self.execute(['inaturalist_classification.py', '--model', 'birds', '--input', image], 104 | timeout=45.0) 105 | 106 | def test_mobilenet_based_classifier(self): 107 | self.execute(['mobilenet_based_classifier.py', 108 | '--model_path', model_path('mobilenet_v2_192res_1.0_inat_plant.binaryproto'), 109 | '--label_path', model_path('mobilenet_v2_192res_1.0_inat_plant_labels.txt'), 110 | '--input_height', '192', 111 | '--input_width', '192', 112 | '--input_layer', 'map/TensorArrayStack/TensorArrayGatherV3', 113 | '--output_layer', 'prediction', 114 | '--num_frames', '100', 115 | '--preview'], timeout=45.0) 116 | 117 | def test_object_detection(self): 118 | image = test_image_path('cat.jpg') 119 | self.execute(['object_detection.py', '--input', image], timeout=45.0) 120 | 121 | if __name__ == '__main__': 122 | unittest.main() 123 | -------------------------------------------------------------------------------- /src/examples/leds_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Copyright 2018 Google Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import math 16 | import time 17 | 18 | from aiy.leds import (Leds, Pattern, PrivacyLed, RgbLeds, Color) 19 | 20 | def main(): 21 | with Leds() as leds: 22 | print('RGB: Solid RED for 1 second') 23 | leds.update(Leds.rgb_on(Color.RED)) 24 | time.sleep(1) 25 | 26 | print('RGB: Solid GREEN for 1 second') 27 | leds.update(Leds.rgb_on(Color.GREEN)) 28 | time.sleep(1) 29 | 30 | print('RGB: Solid YELLOW for 1 second') 31 | leds.update(Leds.rgb_on(Color.YELLOW)) 32 | time.sleep(1) 33 | 34 | print('RGB: Solid BLUE for 1 second') 35 | leds.update(Leds.rgb_on(Color.BLUE)) 36 | time.sleep(1) 37 | 38 | print('RGB: Solid PURPLE for 1 second') 39 | leds.update(Leds.rgb_on(Color.PURPLE)) 40 | time.sleep(1) 41 | 42 | print('RGB: Solid CYAN for 1 second') 43 | leds.update(Leds.rgb_on(Color.CYAN)) 44 | time.sleep(1) 45 | 46 | print('RGB: Solid WHITE for 1 second') 47 | leds.update(Leds.rgb_on(Color.WHITE)) 48 | time.sleep(1) 49 | 50 | print('RGB: Off for 1 second') 51 | leds.update(Leds.rgb_off()) 52 | time.sleep(1) 53 | 54 | for _ in range(3): 55 | print('Privacy: On (brightness=default)') 56 | leds.update(Leds.privacy_on()) 57 | time.sleep(1) 58 | print('Privacy: Off') 59 | leds.update(Leds.privacy_off()) 60 | time.sleep(1) 61 | 62 | for _ in range(3): 63 | print('Privacy: On (brightness=5)') 64 | leds.update(Leds.privacy_on(5)) 65 | time.sleep(1) 66 | print('Privacy: Off') 67 | leds.update(Leds.privacy_off()) 68 | time.sleep(1) 69 | 70 | print('Set blink pattern: period=500ms (2Hz)') 71 | leds.pattern = Pattern.blink(500) 72 | 73 | print('RGB: Blink RED for 5 seconds') 74 | leds.update(Leds.rgb_pattern(Color.RED)) 75 | time.sleep(5) 76 | 77 | print('RGB: Blink GREEN for 5 seconds') 78 | leds.update(Leds.rgb_pattern(Color.GREEN)) 79 | time.sleep(5) 80 | 81 | print('RGB: Blink BLUE for 5 seconds') 82 | leds.update(Leds.rgb_pattern(Color.BLUE)) 83 | time.sleep(5) 84 | 85 | print('Set breathe pattern: period=1000ms (1Hz)') 86 | leds.pattern = Pattern.breathe(1000) 87 | 88 | print('RGB: Breathe RED for 5 seconds') 89 | leds.update(Leds.rgb_pattern(Color.RED)) 90 | time.sleep(5) 91 | 92 | print('RGB: Breathe GREEN for 5 seconds') 93 | leds.update(Leds.rgb_pattern(Color.GREEN)) 94 | time.sleep(5) 95 | 96 | print('RGB: Breathe BLUE for 5 seconds') 97 | leds.update(Leds.rgb_pattern(Color.BLUE)) 98 | time.sleep(5) 99 | 100 | print('RGB: Increase RED brightness for 3.2 seconds') 101 | for i in range(32): 102 | leds.update(Leds.rgb_on((8 * i, 0, 0))) 103 | time.sleep(0.1) 104 | 105 | print('RGB: Decrease RED brightness for 3.2 seconds') 106 | for i in reversed(range(32)): 107 | leds.update(Leds.rgb_on((8 * i, 0, 0))) 108 | time.sleep(0.1) 109 | 110 | print('RGB: Blend between GREEN and BLUE for 3.2 seconds') 111 | for i in range(32): 112 | color = Color.blend(Color.BLUE, Color.GREEN, i / 32) 113 | leds.update(Leds.rgb_on(color)) 114 | time.sleep(0.1) 115 | 116 | print('RGB: Off for 1 second') 117 | leds.update(Leds.rgb_off()) 118 | time.sleep(1) 119 | 120 | print('Privacy: On for 2 seconds') 121 | with PrivacyLed(leds): 122 | time.sleep(2) 123 | 124 | print('RGB: Solid GREEN for 2 seconds') 125 | with RgbLeds(leds, Leds.rgb_on(Color.GREEN)): 126 | time.sleep(2) 127 | 128 | print('Custom configuration for 5 seconds') 129 | leds.update({ 130 | 1: Leds.Channel(Leds.Channel.PATTERN, 128), # Red channel 131 | 2: Leds.Channel(Leds.Channel.OFF, 0), # Green channel 132 | 3: Leds.Channel(Leds.Channel.ON, 128), # Blue channel 133 | 4: Leds.Channel(Leds.Channel.PATTERN, 64), # Privacy channel 134 | }) 135 | time.sleep(5) 136 | 137 | print('Done') 138 | 139 | if __name__ == '__main__': 140 | main() 141 | -------------------------------------------------------------------------------- /src/tests/engine_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from picamera import PiCamera 4 | 5 | from aiy.vision.inference import InferenceEngine, InferenceException, \ 6 | ImageInference, CameraInference 7 | from aiy.vision.models import face_detection as fd 8 | 9 | from .test_util import TestImage, TestImageFile 10 | 11 | class InferenceEngineTest(unittest.TestCase): 12 | def setUp(self): 13 | with InferenceEngine() as engine: 14 | engine.reset() 15 | 16 | def test_firmware_info(self): 17 | with InferenceEngine() as engine: 18 | for _ in range(50): 19 | info = engine.get_firmware_info() 20 | self.assertTrue(hasattr(info, 'major')) 21 | self.assertTrue(hasattr(info, 'minor')) 22 | 23 | def test_system_info(self): 24 | with InferenceEngine() as engine: 25 | for _ in range(50): 26 | info = engine.get_system_info() 27 | self.assertTrue(hasattr(info, 'uptime_seconds')) 28 | self.assertTrue(hasattr(info, 'temperature_celsius')) 29 | 30 | def test_load_unload(self): 31 | with InferenceEngine() as engine: 32 | state = engine.get_inference_state() 33 | self.assertFalse(state.loaded_models) 34 | self.assertFalse(state.processing_models) 35 | 36 | model_name = engine.load_model(fd.model()) 37 | state = engine.get_inference_state() 38 | self.assertEqual(set(state.loaded_models), {model_name}) 39 | self.assertFalse(state.processing_models) 40 | 41 | with self.assertRaises(InferenceException): 42 | engine.unload_model('invalid_model_name') 43 | 44 | engine.unload_model(model_name) 45 | state = engine.get_inference_state() 46 | self.assertFalse(state.loaded_models) 47 | self.assertFalse(state.processing_models) 48 | 49 | def test_inference_state(self): 50 | with InferenceEngine() as engine: 51 | state = engine.get_inference_state() 52 | self.assertFalse(state.loaded_models) 53 | self.assertFalse(state.processing_models) 54 | 55 | model_name = engine.load_model(fd.model()) 56 | state = engine.get_inference_state() 57 | self.assertEqual(set(state.loaded_models), {model_name}) 58 | self.assertFalse(state.processing_models) 59 | 60 | engine.reset() 61 | state = engine.get_inference_state() 62 | self.assertFalse(state.loaded_models) 63 | self.assertFalse(state.processing_models) 64 | 65 | model_name = engine.load_model(fd.model()) 66 | 67 | with PiCamera(sensor_mode=4): 68 | engine.start_camera_inference(model_name) 69 | state = engine.get_inference_state() 70 | self.assertEqual(set(state.loaded_models), {model_name}) 71 | self.assertEqual(set(state.processing_models), {model_name}) 72 | 73 | engine.reset() 74 | state = engine.get_inference_state() 75 | self.assertFalse(state.loaded_models) 76 | self.assertFalse(state.processing_models) 77 | 78 | def test_camera_state(self): 79 | MODES = {1: (1920, 1080), 80 | 2: (3280, 2464), 81 | 3: (3280, 2464), 82 | 4: (1640, 1232), 83 | 5: (1640, 922), 84 | 6: (1280, 720), 85 | 7: (640, 480)} 86 | with InferenceEngine() as engine: 87 | for mode, (width, height) in MODES.items(): 88 | state = engine.get_camera_state() 89 | self.assertFalse(state.running) 90 | 91 | with PiCamera(sensor_mode=mode): 92 | state = engine.get_camera_state() 93 | self.assertTrue(state.running) 94 | self.assertEqual(state.width, width) 95 | self.assertEqual(state.height, height) 96 | info = engine.get_system_info() 97 | 98 | state = engine.get_camera_state() 99 | self.assertFalse(state.running) 100 | 101 | def test_image_inference_raw(self): 102 | with ImageInference(fd.model()) as inference, TestImage('faces.jpg') as image: 103 | fd.get_faces(inference.run(image)) 104 | 105 | def test_image_inference_jpeg(self): 106 | with ImageInference(fd.model()) as inference, TestImageFile('faces.jpg') as f: 107 | fd.get_faces(inference.run(f.read())) 108 | 109 | def test_camera_inference(self): 110 | with PiCamera(sensor_mode=4): 111 | with CameraInference(fd.model()) as inference: 112 | state = inference.engine.get_inference_state() 113 | self.assertEqual(len(state.loaded_models), 1) 114 | self.assertEqual(len(state.processing_models), 1) 115 | 116 | results = [fd.get_faces(result) for result in inference.run(10)] 117 | self.assertEqual(len(results), 10) 118 | 119 | if __name__ == '__main__': 120 | unittest.main() 121 | -------------------------------------------------------------------------------- /src/examples/vision/mobilenet_based_classifier.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2017 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """Script to run generic MobileNet based classification model.""" 17 | import argparse 18 | 19 | from picamera import PiCamera, Color 20 | 21 | from aiy.vision import inference 22 | from aiy.vision.models import utils 23 | 24 | 25 | def read_labels(label_path): 26 | with open(label_path) as label_file: 27 | return [label.strip() for label in label_file.readlines()] 28 | 29 | 30 | def get_message(result, threshold, top_k): 31 | if result: 32 | return 'Detecting:\n %s' % '\n'.join(result) 33 | 34 | return 'Nothing detected when threshold=%.2f, top_k=%d' % (threshold, top_k) 35 | 36 | 37 | def process(result, labels, tensor_name, threshold, top_k): 38 | """Processes inference result and returns labels sorted by confidence.""" 39 | # MobileNet based classification model returns one result vector. 40 | assert len(result.tensors) == 1 41 | tensor = result.tensors[tensor_name] 42 | probs, shape = tensor.data, tensor.shape 43 | assert shape.depth == len(labels) 44 | pairs = [pair for pair in enumerate(probs) if pair[1] > threshold] 45 | pairs = sorted(pairs, key=lambda pair: pair[1], reverse=True) 46 | pairs = pairs[0:top_k] 47 | return [' %s (%.2f)' % (labels[index], prob) for index, prob in pairs] 48 | 49 | 50 | def main(): 51 | parser = argparse.ArgumentParser() 52 | parser.add_argument('--model_path', required=True, 53 | help='Path to converted model file that can run on VisionKit.') 54 | parser.add_argument('--label_path', required=True, 55 | help='Path to label file that corresponds to the model.') 56 | parser.add_argument('--input_height', type=int, required=True, help='Input height.') 57 | parser.add_argument('--input_width', type=int, required=True, help='Input width.') 58 | parser.add_argument('--input_layer', required=True, help='Name of input layer.') 59 | parser.add_argument('--output_layer', required=True, help='Name of output layer.') 60 | parser.add_argument('--num_frames', type=int, default=None, 61 | help='Sets the number of frames to run for, otherwise runs forever.') 62 | parser.add_argument('--input_mean', type=float, default=128.0, help='Input mean.') 63 | parser.add_argument('--input_std', type=float, default=128.0, help='Input std.') 64 | parser.add_argument('--input_depth', type=int, default=3, help='Input depth.') 65 | parser.add_argument('--threshold', type=float, default=0.1, 66 | help='Threshold for classification score (from output tensor).') 67 | parser.add_argument('--top_k', type=int, default=3, help='Keep at most top_k labels.') 68 | parser.add_argument('--preview', action='store_true', default=False, 69 | help='Enables camera preview in addition to printing result to terminal.') 70 | parser.add_argument('--show_fps', action='store_true', default=False, 71 | help='Shows end to end FPS.') 72 | args = parser.parse_args() 73 | 74 | model = inference.ModelDescriptor( 75 | name='mobilenet_based_classifier', 76 | input_shape=(1, args.input_height, args.input_width, args.input_depth), 77 | input_normalizer=(args.input_mean, args.input_std), 78 | compute_graph=utils.load_compute_graph(args.model_path)) 79 | labels = read_labels(args.label_path) 80 | 81 | with PiCamera(sensor_mode=4, resolution=(1640, 1232), framerate=30) as camera: 82 | if args.preview: 83 | camera.start_preview() 84 | 85 | with inference.CameraInference(model) as camera_inference: 86 | for result in camera_inference.run(args.num_frames): 87 | processed_result = process(result, labels, args.output_layer, 88 | args.threshold, args.top_k) 89 | message = get_message(processed_result, args.threshold, args.top_k) 90 | if args.show_fps: 91 | message += '\nWith %.1f FPS.' % camera_inference.rate 92 | print(message) 93 | 94 | if args.preview: 95 | camera.annotate_foreground = Color('black') 96 | camera.annotate_background = Color('white') 97 | # PiCamera text annotation only supports ascii. 98 | camera.annotate_text = '\n %s' % message.encode( 99 | 'ascii', 'backslashreplace').decode('ascii') 100 | 101 | if args.preview: 102 | camera.stop_preview() 103 | 104 | 105 | if __name__ == '__main__': 106 | main() 107 | --------------------------------------------------------------------------------