├── .gitignore ├── .kokoro ├── common.cfg ├── continuous.cfg ├── deploy-build-test.sh ├── periodic.cfg ├── presubmit.cfg └── trampoline.sh ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SPEECH-TRANSLATION.md ├── app ├── .gitignore ├── build.gradle ├── proguard-rules.pro └── src │ ├── androidTest │ ├── assets │ │ ├── speech-recording-16khz.b64 │ │ ├── speech-recording-16khz.wav │ │ ├── speech-recording-24khz.b64 │ │ └── speech-recording-24khz.wav │ └── java │ │ └── com │ │ └── google │ │ └── cloud │ │ └── solutions │ │ └── flexenv │ │ └── common │ │ ├── Base64EncodingHelperAndroidTest.java │ │ └── SpeechTranslationHelperAndroidTest.java │ └── main │ ├── AndroidManifest.xml │ ├── java │ └── com │ │ └── google │ │ └── cloud │ │ └── solutions │ │ └── flexenv │ │ ├── FirebaseLogger.java │ │ ├── PlayActivity.java │ │ └── common │ │ ├── Base64EncodingHelper.java │ │ ├── BaseMessage.java │ │ ├── GcsDownloadHelper.java │ │ ├── LogEntry.java │ │ ├── RecordingHelper.java │ │ ├── SpeechMessage.java │ │ ├── SpeechTranslationException.java │ │ ├── SpeechTranslationHelper.java │ │ ├── TextMessage.java │ │ └── Translation.java │ └── res │ ├── drawable │ ├── ic_baseline_mic_24px.xml │ ├── ic_baseline_mic_none_24px.xml │ ├── mic_button_selector.xml │ └── side_nav_bar.xml │ ├── layout │ ├── activity_play.xml │ ├── app_bar_play.xml │ ├── content_play.xml │ └── nav_header_play.xml │ ├── menu │ └── activity_play_drawer.xml │ ├── mipmap-hdpi │ └── ic_launcher.png │ ├── mipmap-mdpi │ └── ic_launcher.png │ ├── mipmap-xhdpi │ └── ic_launcher.png │ ├── mipmap-xxhdpi │ └── ic_launcher.png │ ├── mipmap-xxxhdpi │ └── ic_launcher.png │ ├── values-w820dp │ └── dimens.xml │ └── values │ ├── colors.xml │ ├── dimens.xml │ ├── speech_translation.xml │ ├── strings.xml │ └── styles.xml ├── build.gradle ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── nexus5.png ├── settings.gradle └── speech-translation-device.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Built application files 2 | *.apk 3 | *.ap_ 4 | 5 | # Files for the Dalvik VM 6 | *.dex 7 | 8 | # Java class files 9 | *.class 10 | 11 | # Generated files 12 | bin/ 13 | gen/ 14 | out/ 15 | build/ 16 | 17 | # Gradle cache 18 | .gradle/ 19 | 20 | # Local configuration file (sdk path, etc) 21 | local.properties 22 | 23 | # Proguard folder generated by Eclipse 24 | proguard/ 25 | 26 | # Log Files 27 | *.log 28 | 29 | # Android Studio Navigation editor temp files 30 | .navigation/ 31 | 32 | # Mobile Tools for Java (J2ME) 33 | .mtj.tmp/ 34 | 35 | # Package Files 36 | *.jar 37 | !gradle/wrapper/gradle-wrapper.jar 38 | *.war 39 | *.ear 40 | 41 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 42 | hs_err_pid* 43 | 44 | # IDEA/Android Studio project files, because 45 | # the project can be imported from settings.gradle 46 | *.iml 47 | .idea/* 48 | !.idea/copyright 49 | !.idea/codeStyles/codeStyleConfig.xml 50 | 51 | # Keystore files 52 | *.jks 53 | 54 | # External native build folder 55 | .externalNativeBuild 56 | 57 | # Google services configuration file 58 | google-services.json 59 | 60 | # Attributes of the parent folder on macOS 61 | .DS_Store 62 | 63 | # Eclipse project files 64 | .classpath 65 | .project 66 | 67 | # Sandbox stuff 68 | _sandbox 69 | # Android Studio captures folder 70 | captures/ 71 | -------------------------------------------------------------------------------- /.kokoro/common.cfg: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Format: //devtools/kokoro/config/proto/build.proto 16 | 17 | # Download trampoline resources. 18 | gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" 19 | 20 | # Download project resources from Cloud Storage. 21 | gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/firebase-android-client" 22 | 23 | # Download project resources for Cloud Function 24 | gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/nodejs-docs-samples" 25 | 26 | # Configure the docker image for kokoro-trampoline. 27 | env_vars: { 28 | key: "TRAMPOLINE_IMAGE" 29 | value: "gcr.io/cloud-devrel-kokoro-resources/java" 30 | } 31 | 32 | 33 | # Use the trampoline script to run in docker. 34 | build_file: "firebase-android-client/.kokoro/trampoline.sh" 35 | 36 | action { 37 | define_artifacts { 38 | regex: "**/*sponge_log*" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /.kokoro/continuous.cfg: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Format: //devtools/kokoro/config/proto/build.proto 16 | 17 | # Tell trampoline which tests to run. 18 | env_vars: { 19 | key: "TRAMPOLINE_BUILD_FILE" 20 | value: "github/firebase-android-client/.kokoro/deploy-build-test.sh" 21 | } 22 | 23 | -------------------------------------------------------------------------------- /.kokoro/deploy-build-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Fail on any error. 17 | set -e 18 | 19 | # Save starting directory. The script needs it later. 20 | cwd=$(pwd) 21 | 22 | echo "Setting up Android environment…" 23 | cd ${cwd} 24 | if [ ! -d ${HOME}/android-sdk ]; then 25 | mkdir -p ${HOME}/android-sdk 26 | pushd "${HOME}/android-sdk" 27 | wget https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip 28 | unzip sdk-tools-linux-4333796.zip 29 | popd 30 | fi 31 | 32 | export ANDROID_HOME="${HOME}/android-sdk" 33 | export adb_command="$ANDROID_HOME"'/platform-tools/adb' 34 | # Install Android SDK, tools, and build tools API 29, system image, and emulator 35 | echo "y" | ${ANDROID_HOME}/tools/bin/sdkmanager \ 36 | "platforms;android-29" "tools" "platform-tools" "build-tools;29.0.3" \ 37 | "system-images;android-29;google_apis;x86_64" "emulator" 38 | 39 | export PATH=${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:${PATH} 40 | 41 | echo "Use the headless emulator build..." 42 | cp "$ANDROID_HOME"'/emulator/qemu/linux-x86_64/qemu-system-x86_64-headless' \ 43 | "$ANDROID_HOME"'/emulator/qemu/linux-x86_64/qemu-system-x86_64' 44 | 45 | echo "Move to the tools/bin directory…" 46 | cd ${ANDROID_HOME}/tools/bin 47 | echo "no" | ./avdmanager create avd -n test -k "system-images;android-29;google_apis;x86_64" 48 | echo "" 49 | echo "Start the emulator…" 50 | cd ${ANDROID_HOME}/emulator 51 | emulator -avd test -no-audio -no-window -screen no-touch & 52 | $adb_command wait-for-device 53 | 54 | echo "Setting up speech translation microservice…" 55 | 56 | git clone https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git github/nodejs-docs-samples 57 | export QT_DEBUG_PLUGINS=1 58 | export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/secrets-key.json 59 | export GCLOUD_PROJECT=nodejs-docs-samples-tests 60 | export GCF_REGION=us-central1 61 | export NODE_ENV=development 62 | export FUNCTIONS_TOPIC=integration-tests-instance 63 | export FUNCTIONS_BUCKET=$GCLOUD_PROJECT 64 | gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS" 65 | gcloud config set project $GCLOUD_PROJECT 66 | cd ./github/nodejs-docs-samples/functions/speech-to-speech/functions 67 | env 68 | gcloud components update 69 | gcloud --version 70 | gcloud functions deploy speechTranslate --runtime nodejs10 --trigger-http \ 71 | --update-env-vars ^:^OUTPUT_BUCKET=playchat-c5cc70f6-61ed-4640-91be-996721838560:SUPPORTED_LANGUAGE_CODES=en,es,fr 72 | 73 | echo "Move to the root directory of the repo…" 74 | cd ${cwd}/github/firebase-android-client 75 | 76 | # Copy the Google services configuration file and test values 77 | cp ${KOKORO_GFILE_DIR}/google-services.json app/google-services.json 78 | cp ${KOKORO_GFILE_DIR}/speech_translation_test.xml app/src/main/res/values/speech_translation.xml 79 | 80 | echo "Run tests and build APK file…" 81 | $adb_command logcat --clear 82 | $adb_command logcat v long > logcat_sponge_log & 83 | ./gradlew clean check build connectedAndroidTest 84 | $adb_command logcat --clear 85 | 86 | echo "Delete the Cloud Function…" 87 | gcloud functions delete speechTranslate 88 | -------------------------------------------------------------------------------- /.kokoro/periodic.cfg: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Format: //devtools/kokoro/config/proto/build.proto 16 | 17 | # Tell trampoline which tests to run. 18 | env_vars: { 19 | key: "TRAMPOLINE_BUILD_FILE" 20 | value: "github/firebase-android-client/.kokoro/deploy-build-test.sh" 21 | } 22 | 23 | -------------------------------------------------------------------------------- /.kokoro/presubmit.cfg: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Format: //devtools/kokoro/config/proto/build.proto 16 | 17 | # Tell trampoline which tests to run. 18 | env_vars: { 19 | key: "TRAMPOLINE_BUILD_FILE" 20 | value: "github/firebase-android-client/.kokoro/deploy-build-test.sh" 21 | } 22 | 23 | -------------------------------------------------------------------------------- /.kokoro/trampoline.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2018 Google LLC. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" 17 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Want to contribute? Great! First, read this page (including the small print at the end). 2 | 3 | ### Before you contribute 4 | Before we can use your code, you must sign the 5 | [Google Individual Contributor License Agreement] 6 | (https://cla.developers.google.com/about/google-individual) 7 | (CLA), which you can do online. The CLA is necessary mainly because you own the 8 | copyright to your changes, even after your contribution becomes part of our 9 | codebase, so we need your permission to use and distribute your code. We also 10 | need to be sure of various other things—for instance that you'll tell us if you 11 | know that your code infringes on other people's patents. You don't have to sign 12 | the CLA until after you've submitted your code for review and a member has 13 | approved it, but you must do it before we can put your code into our codebase. 14 | Before you start working on a larger contribution, you should get in touch with 15 | us first through the issue tracker with your idea so that we can help out and 16 | possibly guide you. Coordinating up front makes it much easier to avoid 17 | frustration later on. 18 | 19 | ### Code reviews 20 | All submissions, including submissions by project members, require review. We 21 | use Github pull requests for this purpose. 22 | 23 | ### The small print 24 | Contributions made by corporations are covered by a different agreement than 25 | the one above, the 26 | [Software Grant and Corporate Contributor License Agreement] 27 | (https://cla.developers.google.com/about/google-corporate). 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 5 | 1. Definitions. 6 | "License" shall mean the terms and conditions for use, reproduction, 7 | and distribution as defined by Sections 1 through 9 of this document. 8 | "Licensor" shall mean the copyright owner or entity authorized by 9 | the copyright owner that is granting the License. 10 | "Legal Entity" shall mean the union of the acting entity and all 11 | other entities that control, are controlled by, or are under common 12 | control with that entity. For the purposes of this definition, 13 | "control" means (i) the power, direct or indirect, to cause the 14 | direction or management of such entity, whether by contract or 15 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 16 | outstanding shares, or (iii) beneficial ownership of such entity. 17 | "You" (or "Your") shall mean an individual or Legal Entity 18 | exercising permissions granted by this License. 19 | "Source" form shall mean the preferred form for making modifications, 20 | including but not limited to software source code, documentation 21 | source, and configuration files. 22 | "Object" form shall mean any form resulting from mechanical 23 | transformation or translation of a Source form, including but 24 | not limited to compiled object code, generated documentation, 25 | and conversions to other media types. 26 | "Work" shall mean the work of authorship, whether in Source or 27 | Object form, made available under the License, as indicated by a 28 | copyright notice that is included in or attached to the work 29 | (an example is provided in the Appendix below). 30 | "Derivative Works" shall mean any work, whether in Source or Object 31 | form, that is based on (or derived from) the Work and for which the 32 | editorial revisions, annotations, elaborations, or other modifications 33 | represent, as a whole, an original work of authorship. For the purposes 34 | of this License, Derivative Works shall not include works that remain 35 | separable from, or merely link (or bind by name) to the interfaces of, 36 | the Work and Derivative Works thereof. 37 | "Contribution" shall mean any work of authorship, including 38 | the original version of the Work and any modifications or additions 39 | to that Work or Derivative Works thereof, that is intentionally 40 | submitted to Licensor for inclusion in the Work by the copyright owner 41 | or by an individual or Legal Entity authorized to submit on behalf of 42 | the copyright owner. For the purposes of this definition, "submitted" 43 | means any form of electronic, verbal, or written communication sent 44 | to the Licensor or its representatives, including but not limited to 45 | communication on electronic mailing lists, source code control systems, 46 | and issue tracking systems that are managed by, or on behalf of, the 47 | Licensor for the purpose of discussing and improving the Work, but 48 | excluding communication that is conspicuously marked or otherwise 49 | designated in writing by the copyright owner as "Not a Contribution." 50 | "Contributor" shall mean Licensor and any individual or Legal Entity 51 | on behalf of whom a Contribution has been received by Licensor and 52 | subsequently incorporated within the Work. 53 | 2. Grant of Copyright License. Subject to the terms and conditions of 54 | this License, each Contributor hereby grants to You a perpetual, 55 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 56 | copyright license to reproduce, prepare Derivative Works of, 57 | publicly display, publicly perform, sublicense, and distribute the 58 | Work and such Derivative Works in Source or Object form. 59 | 3. Grant of Patent License. Subject to the terms and conditions of 60 | this License, each Contributor hereby grants to You a perpetual, 61 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 62 | (except as stated in this section) patent license to make, have made, 63 | use, offer to sell, sell, import, and otherwise transfer the Work, 64 | where such license applies only to those patent claims licensable 65 | by such Contributor that are necessarily infringed by their 66 | Contribution(s) alone or by combination of their Contribution(s) 67 | with the Work to which such Contribution(s) was submitted. If You 68 | institute patent litigation against any entity (including a 69 | cross-claim or counterclaim in a lawsuit) alleging that the Work 70 | or a Contribution incorporated within the Work constitutes direct 71 | or contributory patent infringement, then any patent licenses 72 | granted to You under this License for that Work shall terminate 73 | as of the date such litigation is filed. 74 | 4. Redistribution. You may reproduce and distribute copies of the 75 | Work or Derivative Works thereof in any medium, with or without 76 | modifications, and in Source or Object form, provided that You 77 | meet the following conditions: 78 | (a) You must give any other recipients of the Work or 79 | Derivative Works a copy of this License; and 80 | (b) You must cause any modified files to carry prominent notices 81 | stating that You changed the files; and 82 | (c) You must retain, in the Source form of any Derivative Works 83 | that You distribute, all copyright, patent, trademark, and 84 | attribution notices from the Source form of the Work, 85 | excluding those notices that do not pertain to any part of 86 | the Derivative Works; and 87 | (d) If the Work includes a "NOTICE" text file as part of its 88 | distribution, then any Derivative Works that You distribute must 89 | include a readable copy of the attribution notices contained 90 | within such NOTICE file, excluding those notices that do not 91 | pertain to any part of the Derivative Works, in at least one 92 | of the following places: within a NOTICE text file distributed 93 | as part of the Derivative Works; within the Source form or 94 | documentation, if provided along with the Derivative Works; or, 95 | within a display generated by the Derivative Works, if and 96 | wherever such third-party notices normally appear. The contents 97 | of the NOTICE file are for informational purposes only and 98 | do not modify the License. You may add Your own attribution 99 | notices within Derivative Works that You distribute, alongside 100 | or as an addendum to the NOTICE text from the Work, provided 101 | that such additional attribution notices cannot be construed 102 | as modifying the License. 103 | You may add Your own copyright statement to Your modifications and 104 | may provide additional or different license terms and conditions 105 | for use, reproduction, or distribution of Your modifications, or 106 | for any such Derivative Works as a whole, provided Your use, 107 | reproduction, and distribution of the Work otherwise complies with 108 | the conditions stated in this License. 109 | 5. Submission of Contributions. Unless You explicitly state otherwise, 110 | any Contribution intentionally submitted for inclusion in the Work 111 | by You to the Licensor shall be under the terms and conditions of 112 | this License, without any additional terms or conditions. 113 | Notwithstanding the above, nothing herein shall supersede or modify 114 | the terms of any separate license agreement you may have executed 115 | with Licensor regarding such Contributions. 116 | 6. Trademarks. This License does not grant permission to use the trade 117 | names, trademarks, service marks, or product names of the Licensor, 118 | except as required for reasonable and customary use in describing the 119 | origin of the Work and reproducing the content of the NOTICE file. 120 | 7. Disclaimer of Warranty. Unless required by applicable law or 121 | agreed to in writing, Licensor provides the Work (and each 122 | Contributor provides its Contributions) on an "AS IS" BASIS, 123 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 124 | implied, including, without limitation, any warranties or conditions 125 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 126 | PARTICULAR PURPOSE. You are solely responsible for determining the 127 | appropriateness of using or redistributing the Work and assume any 128 | risks associated with Your exercise of permissions under this License. 129 | 8. Limitation of Liability. In no event and under no legal theory, 130 | whether in tort (including negligence), contract, or otherwise, 131 | unless required by applicable law (such as deliberate and grossly 132 | negligent acts) or agreed to in writing, shall any Contributor be 133 | liable to You for damages, including any direct, indirect, special, 134 | incidental, or consequential damages of any character arising as a 135 | result of this License or out of the use or inability to use the 136 | Work (including but not limited to damages for loss of goodwill, 137 | work stoppage, computer failure or malfunction, or any and all 138 | other commercial damages or losses), even if such Contributor 139 | has been advised of the possibility of such damages. 140 | 9. Accepting Warranty or Additional Liability. While redistributing 141 | the Work or Derivative Works thereof, You may choose to offer, 142 | and charge a fee for, acceptance of support, warranty, indemnity, 143 | or other liability obligations and/or rights consistent with this 144 | License. However, in accepting such obligations, You may act only 145 | on Your own behalf and on Your sole responsibility, not on behalf 146 | of any other Contributor, and only if You agree to indemnify, 147 | defend, and hold each Contributor harmless for any liability 148 | incurred by, or claims asserted against, such Contributor by reason 149 | of your accepting any such warranty or additional liability. 150 | END OF TERMS AND CONDITIONS 151 | APPENDIX: How to apply the Apache License to your work. 152 | To apply the Apache License to your work, attach the following 153 | boilerplate notice, with the fields enclosed by brackets "{}" 154 | replaced with your own identifying information. (Don't include 155 | the brackets!) The text should be enclosed in the appropriate 156 | comment syntax for the file format. We also recommend that a 157 | file or class name and description of purpose be included on the 158 | same "printed page" as the copyright notice for easier 159 | identification within third-party archives. 160 | Copyright {yyyy} {name of copyright owner} 161 | Licensed under the Apache License, Version 2.0 (the "License"); 162 | you may not use this file except in compliance with the License. 163 | You may obtain a copy of the License at 164 | http://www.apache.org/licenses/LICENSE-2.0 165 | Unless required by applicable law or agreed to in writing, software 166 | distributed under the License is distributed on an "AS IS" BASIS, 167 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 168 | See the License for the specific language governing permissions and 169 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![status: inactive](https://img.shields.io/badge/status-inactive-red.svg) 2 | 3 | This project is no longer actively developed or maintained. 4 | 5 | For new work on this check out [this sample](https://github.com/firebase/quickstart-android). 6 | 7 | # Build a mobile app using Firebase and App Engine flexible environment 8 | 9 | ![Kokoro Build Status](https://storage.googleapis.com/cloud-devrel-kokoro-resources/java/badges/firebase-android-client.svg) 10 | 11 | This repository contains the Android client sample code for the [Build a Mobile 12 | App Using Firebase and App Engine Flexible 13 | Environment](https://cloud.google.com/solutions/mobile/mobile-firebase-app-engine-flexible) 14 | solution. You can find the sample code for the backend in the 15 | [firebase-appengine-backend](../../../firebase-appengine-backend) 16 | repository. 17 | 18 | ## Build requirements 19 | 20 | - Sign up for [Firebase](https://firebase.google.com/) and create a new project 21 | in the [Firebase console](https://console.firebase.google.com/). 22 | - Deploy your backend according to the instructions in the 23 | [firebase-appengine-backend](../../../firebase-appengine-backend) 24 | repository. 25 | 26 | > **Note**: Firebase is a Google product, independent from Google Cloud 27 | > Platform. 28 | 29 | Build and test environment: 30 | 31 | - Android Studio 4.0.1 from the stable channel. 32 | - Android 8.1 (API Level 27) emulator. 33 | 34 | ## Configuration 35 | 36 | 1. Sign in to the [Firebase console](https://console.firebase.google.com) and 37 | select your backend project. 38 | 1. In the **Develop** section, select **Authentication**. 39 | 1. In the **Authentication** page, select **Sign-in Method**. 40 | 1. Select and enable the **Google** sign-in provider. 41 | 1. [Get your development environment SHA-1 42 | fingerprints](https://developers.google.com/android/guides/client-auth). 43 | 1. Provide the following information in your Firebase project settings: 44 | - **App nickname**: PlayChat 45 | - **Package name**: com.google.cloud.solutions.flexenv 46 | - **SHA certificate fingerprints**: Your development environment SHA-1 47 | fingerprints from the previous step. 48 | 1. Download the `google-services.json` file to the root directory of the `app` 49 | module in your Android project. 50 | 51 | ## Launch and test 52 | 53 | 1. Start your emulator and run the app. 54 | 1. Sign in with a Google account. 55 | 1. Select a channel from top-left menu and enter messages. 56 | 57 | The following screenshot shows messages displayed in the app: 58 | 59 | ![Nexus 5](nexus5.png) 60 | 61 | This sample supports speech messages that are automatically translated to the 62 | device language. For more information, see [Enable speech translation with 63 | Machine Learning technologies on Google Cloud Platform](SPEECH-TRANSLATION.md). 64 | 65 | ## License 66 | 67 | Copyright 2018 Google LLC. All Rights Reserved. 68 | 69 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 70 | this file except in compliance with the License. You may obtain a copy of the 71 | License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by 72 | applicable law or agreed to in writing, software distributed under the License 73 | is distributed on an "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 74 | KIND, either express or implied. See the License for the specific language 75 | governing permissions and limitations under the License. 76 | 77 | This is not an official Google product. 78 | -------------------------------------------------------------------------------- /SPEECH-TRANSLATION.md: -------------------------------------------------------------------------------- 1 | # Enable speech translation with Machine Learning technologies on Google Cloud Platform 2 | 3 | The PlayChat sample supports speech messages that are automatically translated 4 | to different languages by a microservice on Google Cloud Platform (GCP). 5 | 6 | The microservice is implemented as a [Google Cloud Function][1] that uses the 7 | [Speech-to-Text][2], [Translation][3], and [Text-to-Speech][4] APIs to process 8 | the audio messages. The translated messages are stored in a bucket on [Google 9 | Cloud Storage][5] where the PlayChat app can retrieve them. The microservice is 10 | available in the [Speech-to-Speech Translation Sample][6]. 11 | 12 | The following screenshot shows the PlayChat sample with a message translated to 13 | French. 14 | 15 | ![PlayChat sample app with a translated message](speech-translation-device.png) 16 | 17 | ## Configuring speech translation 18 | 19 | 1. Configure the PlayChat sample following the instructions on the 20 | [README](README.md) file. 21 | 1. Deploy the [Speech-to-Speech Translation Sample][6] to your GCP project. 22 | 1. Update the value of the `speechToSpeechEndpoint` field in the 23 | [speech_translation.xml][7] file with the URL of the function deployed in the 24 | previous step. 25 | 1. Make sure that your user account has read permissions to the bucket where the 26 | microservice stores the translated audio messages. For more information, see 27 | [Setting bucket permissions][8] in the Google Cloud Storage documentation. 28 | 29 | ## Launch and test 30 | 31 | 1. Start your emulator and run the app. 32 | 1. Sign in with a Google account. 33 | 1. Tap the microphone icon and record a short voice message in the language of 34 | your device. The microservice supports English, Spanish, and French by 35 | default. However, you can configure other languages. 36 | 1. Change the default language on your device settings. The PlayChat app 37 | displays the message on the new language. 38 | 1. Tap the translated message to listen to it in the language of the device. 39 | 40 | ## License 41 | 42 | Copyright 2018 Google LLC. All Rights Reserved. 43 | 44 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 45 | this file except in compliance with the License. You may obtain a copy of the 46 | License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by 47 | applicable law or agreed to in writing, software distributed under the License 48 | is distributed on an "AS-IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 49 | KIND, either express or implied. See the License for the specific language 50 | governing permissions and limitations under the License. 51 | 52 | This is not an official Google product. 53 | 54 | [1]: https://cloud.google.com/functions/ 55 | [2]: https://cloud.google.com/speech-to-text/ 56 | [3]: https://cloud.google.com/translate/ 57 | [4]: https://cloud.google.com/text-to-speech/ 58 | [5]: https://cloud.google.com/storage/ 59 | [6]: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/tree/master/functions/speech-to-speech 60 | [7]: https://github.com/GoogleCloudPlatform/firebase-android-client/blob/master/app/src/main/res/values/speech_translation.xml 61 | [8]: https://cloud.google.com/storage/docs/cloud-console#_bucketpermission 62 | -------------------------------------------------------------------------------- /app/.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'com.android.application' 2 | apply plugin: 'com.google.gms.google-services' 3 | 4 | android { 5 | compileSdkVersion 29 6 | buildToolsVersion "29.0.3" 7 | 8 | defaultConfig { 9 | applicationId "com.google.cloud.solutions.flexenv" 10 | minSdkVersion 24 11 | targetSdkVersion 29 12 | versionCode 1 13 | versionName "1.0" 14 | 15 | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" 16 | } 17 | 18 | buildTypes { 19 | release { 20 | minifyEnabled false 21 | proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' 22 | } 23 | compileOptions { 24 | sourceCompatibility JavaVersion.VERSION_1_8 25 | targetCompatibility JavaVersion.VERSION_1_8 26 | } 27 | } 28 | } 29 | 30 | dependencies { 31 | implementation fileTree(include: ['*.jar'], dir: 'libs') 32 | implementation 'androidx.appcompat:appcompat:1.2.0' 33 | implementation 'com.google.android.material:material:1.1.0' 34 | implementation 'androidx.coordinatorlayout:coordinatorlayout:1.1.0' 35 | testImplementation 'junit:junit:4.12' 36 | androidTestImplementation 'androidx.test.ext:junit:1.1.2' 37 | androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' 38 | 39 | implementation 'com.google.android.gms:play-services-base:17.4.0' 40 | implementation 'com.google.android.gms:play-services-auth:18.1.0' 41 | implementation 'com.google.firebase:firebase-auth:19.4.0' 42 | implementation 'com.google.firebase:firebase-database:19.4.0' 43 | implementation 'com.google.firebase:firebase-storage:19.2.0' 44 | implementation 'com.fasterxml.jackson.core:jackson-core:2.11.2' 45 | implementation 'com.fasterxml.jackson.core:jackson-annotations:2.11.2' 46 | implementation 'com.fasterxml.jackson.core:jackson-databind:2.11.2' 47 | implementation "androidx.constraintlayout:constraintlayout:2.0.0" 48 | implementation 'org.chromium.net:cronet-embedded:76.3809.111' 49 | androidTestImplementation 'androidx.test:rules:1.3.0' 50 | 51 | androidTestImplementation 'com.android.support:support-annotations:28.0.0' 52 | androidTestImplementation 'com.android.support.test:runner:1.0.2' 53 | androidTestImplementation 'androidx.test:runner:1.1.1' 54 | androidTestImplementation 'androidx.test:rules:1.1.1' 55 | } 56 | 57 | apply plugin: 'com.google.gms.google-services' -------------------------------------------------------------------------------- /app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # You can control the set of applied configuration files using the 3 | # proguardFiles setting in build.gradle. 4 | # 5 | # For more details, see 6 | # http://developer.android.com/guide/developing/tools/proguard.html 7 | 8 | # If your project uses WebView with JS, uncomment the following 9 | # and specify the fully qualified class name to the JavaScript interface 10 | # class: 11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 12 | # public *; 13 | #} 14 | 15 | # Uncomment this to preserve the line number information for 16 | # debugging stack traces. 17 | #-keepattributes SourceFile,LineNumberTable 18 | 19 | # If you keep the line number information, uncomment this to 20 | # hide the original source file name. 21 | #-renamesourcefileattribute SourceFile -------------------------------------------------------------------------------- /app/src/androidTest/assets/speech-recording-16khz.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/firebase-android-client/9e801a6314640a8375df56fee956a296bf5f20b7/app/src/androidTest/assets/speech-recording-16khz.wav -------------------------------------------------------------------------------- /app/src/androidTest/assets/speech-recording-24khz.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/firebase-android-client/9e801a6314640a8375df56fee956a296bf5f20b7/app/src/androidTest/assets/speech-recording-24khz.wav -------------------------------------------------------------------------------- /app/src/androidTest/java/com/google/cloud/solutions/flexenv/common/Base64EncodingHelperAndroidTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | import android.content.Context; 19 | 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | import java.io.BufferedReader; 24 | import java.io.File; 25 | import java.io.FileOutputStream; 26 | import java.io.IOException; 27 | import java.io.InputStream; 28 | import java.io.InputStreamReader; 29 | import java.io.OutputStream; 30 | 31 | import androidx.test.filters.SmallTest; 32 | import androidx.test.platform.app.InstrumentationRegistry; 33 | 34 | @SmallTest 35 | public class Base64EncodingHelperAndroidTest { 36 | 37 | @Test 38 | public void base64Encode_16Khz_Success() throws IOException { 39 | String expectedFilePath = "assets/speech-recording-16khz.b64"; 40 | InputStream expectedInputStream = this.getClass().getClassLoader().getResourceAsStream(expectedFilePath); 41 | InputStreamReader expectedInputStreamReader = new InputStreamReader(expectedInputStream); 42 | BufferedReader expectedBufferedReader = new BufferedReader(expectedInputStreamReader); 43 | StringBuilder expected = new StringBuilder(); 44 | 45 | String line; 46 | while((line = expectedBufferedReader.readLine()) != null) { 47 | expected.append(line); 48 | } 49 | 50 | String actualFilePath = "assets/speech-recording-16khz.wav"; 51 | InputStream actualInputStream = this.getClass().getClassLoader().getResourceAsStream(actualFilePath); 52 | 53 | Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); 54 | File actualFile = new File(context.getCacheDir(), "speech-recording-16khz.wav"); 55 | OutputStream output = new FileOutputStream(actualFile); 56 | byte[] buffer = new byte[1024]; 57 | int read; 58 | while ((read = actualInputStream.read(buffer)) != -1) { 59 | output.write(buffer, 0, read); 60 | } 61 | output.flush(); 62 | output.close(); 63 | actualInputStream.close(); 64 | 65 | String actual = Base64EncodingHelper.encode(actualFile); 66 | 67 | Assert.assertEquals(expected.toString(), actual); 68 | } 69 | 70 | @Test 71 | public void base64Encode_24Khz_Success() throws IOException { 72 | String expectedFilePath = "assets/speech-recording-24khz.b64"; 73 | InputStream expectedInputStream = this.getClass().getClassLoader().getResourceAsStream(expectedFilePath); 74 | InputStreamReader expectedInputStreamReader = new InputStreamReader(expectedInputStream); 75 | BufferedReader expectedBufferedReader = new BufferedReader(expectedInputStreamReader); 76 | StringBuilder expected = new StringBuilder(); 77 | 78 | String line; 79 | while((line = expectedBufferedReader.readLine()) != null) { 80 | expected.append(line); 81 | } 82 | 83 | String actualFilePath = "assets/speech-recording-24khz.wav"; 84 | InputStream actualInputStream = this.getClass().getClassLoader().getResourceAsStream(actualFilePath); 85 | 86 | Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); 87 | File actualFile = new File(context.getCacheDir(), "speech-recording-24khz.wav"); 88 | OutputStream output = new FileOutputStream(actualFile); 89 | byte[] buffer = new byte[1024]; 90 | int read; 91 | while ((read = actualInputStream.read(buffer)) != -1) { 92 | output.write(buffer, 0, read); 93 | } 94 | output.flush(); 95 | output.close(); 96 | actualInputStream.close(); 97 | 98 | String actual = Base64EncodingHelper.encode(actualFile); 99 | 100 | Assert.assertEquals(expected.toString(), actual); 101 | } 102 | 103 | @Test 104 | public void base64Encode_Fail() throws IOException { 105 | String expectedFilePath = "assets/speech-recording-24khz.b64"; 106 | InputStream expectedInputStream = this.getClass().getClassLoader().getResourceAsStream(expectedFilePath); 107 | InputStreamReader expectedInputStreamReader = new InputStreamReader(expectedInputStream); 108 | BufferedReader expectedBufferedReader = new BufferedReader(expectedInputStreamReader); 109 | StringBuilder expected = new StringBuilder(); 110 | 111 | String line; 112 | while((line = expectedBufferedReader.readLine()) != null) { 113 | expected.append(line); 114 | } 115 | 116 | String actualFilePath = "assets/speech-recording-16khz.wav"; 117 | InputStream actualInputStream = this.getClass().getClassLoader().getResourceAsStream(actualFilePath); 118 | 119 | Context context = InstrumentationRegistry.getInstrumentation().getTargetContext(); 120 | File actualFile = new File(context.getCacheDir(), "speech-recording-16khz.wav"); 121 | OutputStream output = new FileOutputStream(actualFile); 122 | byte[] buffer = new byte[1024]; 123 | int read; 124 | while ((read = actualInputStream.read(buffer)) != -1) { 125 | output.write(buffer, 0, read); 126 | } 127 | output.flush(); 128 | output.close(); 129 | actualInputStream.close(); 130 | 131 | String actual = Base64EncodingHelper.encode(actualFile); 132 | 133 | Assert.assertNotEquals(expected.toString(), actual); 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /app/src/androidTest/java/com/google/cloud/solutions/flexenv/common/SpeechTranslationHelperAndroidTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | import android.content.Context; 19 | import android.util.Log; 20 | 21 | import org.chromium.net.CronetEngine; 22 | import org.json.JSONException; 23 | import org.json.JSONObject; 24 | import org.junit.Assert; 25 | import org.junit.Before; 26 | import org.junit.Test; 27 | 28 | import java.io.BufferedReader; 29 | import java.io.IOException; 30 | import java.io.InputStream; 31 | import java.io.InputStreamReader; 32 | 33 | import androidx.test.filters.LargeTest; 34 | import androidx.test.platform.app.InstrumentationRegistry; 35 | 36 | import static org.junit.Assert.assertTrue; 37 | 38 | @LargeTest 39 | public class SpeechTranslationHelperAndroidTest { 40 | private static final String TAG = "SpeechTranslationHelperAndroidTest"; 41 | private static String base64EncodedAudioMessage; 42 | private static Context context; 43 | private static CronetEngine cronetEngine; 44 | 45 | @Before 46 | public void readSpeechRecording16khzb64File() throws IOException { 47 | String file = "assets/speech-recording-16khz.b64"; 48 | InputStream in = this.getClass().getClassLoader().getResourceAsStream(file); 49 | InputStreamReader inputStreamReader = new InputStreamReader(in); 50 | BufferedReader bufferedReader = new BufferedReader(inputStreamReader); 51 | 52 | StringBuilder stringBuilder = new StringBuilder(); 53 | 54 | String line; 55 | while((line = bufferedReader.readLine()) != null) { 56 | stringBuilder.append(line); 57 | } 58 | base64EncodedAudioMessage = stringBuilder.toString(); 59 | 60 | context = InstrumentationRegistry.getInstrumentation().getTargetContext(); 61 | 62 | CronetEngine.Builder myBuilder = new CronetEngine.Builder(context); 63 | cronetEngine = myBuilder.build(); 64 | } 65 | 66 | @Test 67 | public void translateAudioMessage_Success() throws InterruptedException { 68 | final Object waiter = new Object(); 69 | 70 | synchronized (waiter) { 71 | SpeechTranslationHelper.getInstance() 72 | .translateAudioMessage(context, cronetEngine, base64EncodedAudioMessage, 73 | 16000, new SpeechTranslationHelper.SpeechTranslationListener() { 74 | @Override 75 | public void onTranslationSucceeded(String responseBody) { 76 | try { 77 | Log.i(TAG, responseBody); 78 | JSONObject response = new JSONObject(responseBody); 79 | assertTrue(response.has("transcription")); 80 | } catch (JSONException e) { 81 | Log.i(TAG, e.getMessage()); 82 | Assert.fail(); 83 | } finally { 84 | synchronized (waiter) { 85 | waiter.notify(); 86 | } 87 | } 88 | } 89 | 90 | @Override 91 | public void onTranslationFailed(Exception e) { 92 | Log.i(TAG, e.getMessage()); 93 | Assert.fail(); 94 | synchronized (waiter) { 95 | waiter.notify(); 96 | } 97 | } 98 | }); 99 | 100 | synchronized (waiter) { 101 | waiter.wait(); 102 | } 103 | } 104 | } 105 | 106 | @Test 107 | public void translateAudioMessage_Wrong_SampleRate() throws InterruptedException { 108 | final Object waiter = new Object(); 109 | 110 | synchronized (waiter) { 111 | SpeechTranslationHelper.getInstance() 112 | .translateAudioMessage(context, cronetEngine, base64EncodedAudioMessage, 113 | 24000, new SpeechTranslationHelper.SpeechTranslationListener() { 114 | @Override 115 | public void onTranslationSucceeded(String responseBody) { 116 | Log.i(TAG, responseBody); 117 | Assert.fail(); 118 | } 119 | 120 | @Override 121 | public void onTranslationFailed(Exception e) { 122 | Log.i(TAG, e.getMessage()); 123 | Assert.assertTrue(e.getMessage().contains("INVALID_ARGUMENT: sample_rate_hertz")); 124 | synchronized (waiter) { 125 | waiter.notify(); 126 | } 127 | } 128 | }); 129 | 130 | synchronized (waiter) { 131 | waiter.wait(); 132 | } 133 | } 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 17 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/FirebaseLogger.java: -------------------------------------------------------------------------------- 1 | /* 2 | # Copyright 2016 Google LLC. 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv; 17 | 18 | // [START firebase_logger] 19 | 20 | import com.google.cloud.solutions.flexenv.common.LogEntry; 21 | import com.google.firebase.database.DatabaseReference; 22 | import com.google.firebase.database.FirebaseDatabase; 23 | 24 | /* 25 | * FirebaseLogger pushes user event logs to a specified path. 26 | * A backend servlet instance listens to 27 | * the same key and keeps track of event logs. 28 | */ 29 | class FirebaseLogger { 30 | private final DatabaseReference logRef; 31 | 32 | FirebaseLogger(String path) { 33 | logRef = FirebaseDatabase.getInstance().getReference().child(path); 34 | } 35 | 36 | public void log(String tag, String message) { 37 | LogEntry entry = new LogEntry(tag, message); 38 | logRef.push().setValue(entry); 39 | } 40 | 41 | } 42 | // [END firebase_logger] 43 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/PlayActivity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv; 17 | 18 | import android.content.Intent; 19 | import android.media.MediaPlayer; 20 | import android.net.Uri; 21 | import android.os.Bundle; 22 | import android.util.Log; 23 | import android.view.KeyEvent; 24 | import android.view.Menu; 25 | import android.view.MenuItem; 26 | import android.view.View; 27 | import android.widget.AdapterView; 28 | import android.widget.Button; 29 | import android.widget.EditText; 30 | import android.widget.ImageButton; 31 | import android.widget.ListView; 32 | import android.widget.SimpleAdapter; 33 | import android.widget.TextView; 34 | import android.widget.Toast; 35 | 36 | import androidx.annotation.NonNull; 37 | import androidx.appcompat.app.ActionBarDrawerToggle; 38 | import androidx.appcompat.app.AppCompatActivity; 39 | import androidx.appcompat.widget.Toolbar; 40 | import androidx.core.view.GravityCompat; 41 | import androidx.drawerlayout.widget.DrawerLayout; 42 | 43 | import com.google.android.gms.auth.api.signin.GoogleSignIn; 44 | import com.google.android.gms.auth.api.signin.GoogleSignInAccount; 45 | import com.google.android.gms.auth.api.signin.GoogleSignInClient; 46 | import com.google.android.gms.auth.api.signin.GoogleSignInOptions; 47 | import com.google.android.gms.common.SignInButton; 48 | import com.google.android.gms.tasks.OnCompleteListener; 49 | import com.google.android.gms.tasks.Task; 50 | import com.google.android.material.navigation.NavigationView; 51 | import com.google.cloud.solutions.flexenv.common.Base64EncodingHelper; 52 | import com.google.cloud.solutions.flexenv.common.BaseMessage; 53 | import com.google.cloud.solutions.flexenv.common.GcsDownloadHelper; 54 | import com.google.cloud.solutions.flexenv.common.RecordingHelper; 55 | import com.google.cloud.solutions.flexenv.common.SpeechMessage; 56 | import com.google.cloud.solutions.flexenv.common.SpeechTranslationHelper; 57 | import com.google.cloud.solutions.flexenv.common.TextMessage; 58 | import com.google.cloud.solutions.flexenv.common.Translation; 59 | import com.google.firebase.auth.AuthCredential; 60 | import com.google.firebase.auth.FirebaseAuth; 61 | import com.google.firebase.auth.FirebaseUser; 62 | import com.google.firebase.auth.GoogleAuthProvider; 63 | import com.google.firebase.database.ChildEventListener; 64 | import com.google.firebase.database.DataSnapshot; 65 | import com.google.firebase.database.DatabaseError; 66 | import com.google.firebase.database.DatabaseReference; 67 | import com.google.firebase.database.FirebaseDatabase; 68 | import com.google.firebase.database.ValueEventListener; 69 | 70 | import org.chromium.net.CronetEngine; 71 | import org.json.JSONException; 72 | import org.json.JSONObject; 73 | 74 | import java.io.File; 75 | import java.io.IOException; 76 | import java.text.SimpleDateFormat; 77 | import java.util.ArrayList; 78 | import java.util.Arrays; 79 | import java.util.Date; 80 | import java.util.HashMap; 81 | import java.util.List; 82 | import java.util.Locale; 83 | import java.util.Map; 84 | 85 | /* 86 | * Main activity to select a channel and exchange messages with other users 87 | * The app expects users to authenticate with Google ID. It also sends user 88 | * activity logs to a servlet instance through Firebase. 89 | */ 90 | public class PlayActivity 91 | extends AppCompatActivity 92 | implements NavigationView.OnNavigationItemSelectedListener, 93 | AdapterView.OnItemClickListener, 94 | View.OnKeyListener, 95 | View.OnClickListener { 96 | 97 | // Firebase keys commonly used with backend servlet instances 98 | private static final String IBX = "inbox"; 99 | private static final String CHS = "channels"; 100 | private static final String REQLOG = "requestLogger"; 101 | 102 | private static final int RC_SIGN_IN = 9001; 103 | 104 | private static final String TAG = "PlayActivity"; 105 | private static final String CURRENT_CHANNEL_KEY = "CURRENT_CHANNEL_KEY"; 106 | private static final String INBOX_KEY = "INBOX_KEY"; 107 | private static final String FIREBASE_LOGGER_PATH_KEY = "FIREBASE_LOGGER_PATH_KEY"; 108 | private static FirebaseLogger fbLog; 109 | 110 | private GoogleSignInClient mGoogleSignInClient; 111 | private String firebaseLoggerPath; 112 | private String inbox; 113 | private String currentChannel; 114 | private ChildEventListener channelListener; 115 | private SimpleDateFormat fmt; 116 | private CronetEngine cronetEngine; 117 | 118 | private Menu channelMenu; 119 | private TextView channelLabel; 120 | private List> messages; 121 | private SimpleAdapter messageAdapter; 122 | private EditText messageText; 123 | private TextView status; 124 | 125 | @Override 126 | protected void onCreate(Bundle savedInstanceState) { 127 | ListView messageHistory; 128 | super.onCreate(savedInstanceState); 129 | 130 | setContentView(R.layout.activity_play); 131 | Toolbar toolbar = findViewById(R.id.toolbar); 132 | setSupportActionBar(toolbar); 133 | 134 | DrawerLayout drawer = findViewById(R.id.drawer_layout); 135 | ActionBarDrawerToggle toggle = new ActionBarDrawerToggle( 136 | this, drawer, toolbar, R.string.navigation_drawer_open, 137 | R.string.navigation_drawer_close); 138 | drawer.addDrawerListener(toggle); 139 | toggle.syncState(); 140 | 141 | NavigationView navigationView = findViewById(R.id.nav_view); 142 | channelMenu = navigationView.getMenu(); 143 | navigationView.setNavigationItemSelectedListener(this); 144 | initChannels(); 145 | 146 | GoogleSignInOptions.Builder gsoBuilder = 147 | new GoogleSignInOptions.Builder(GoogleSignInOptions.DEFAULT_SIGN_IN) 148 | .requestIdToken(getString(R.string.default_web_client_id)) 149 | .requestEmail(); 150 | 151 | GoogleSignInOptions gso = gsoBuilder.build(); 152 | mGoogleSignInClient = GoogleSignIn.getClient(this, gso); 153 | 154 | SignInButton signInButton = findViewById(R.id.sign_in_button); 155 | signInButton.setSize(SignInButton.SIZE_STANDARD); 156 | signInButton.setOnClickListener(this); 157 | channelLabel = findViewById(R.id.channelLabel); 158 | Button signOutButton = findViewById(R.id.sign_out_button); 159 | signOutButton.setOnClickListener(this); 160 | 161 | ImageButton microphoneButton = findViewById(R.id.microphone_button); 162 | microphoneButton.setOnClickListener(this); 163 | 164 | messages = new ArrayList<>(); 165 | messageAdapter = new SimpleAdapter(this, messages, android.R.layout.simple_list_item_2, 166 | new String[]{"message", "meta"}, 167 | new int[]{android.R.id.text1, android.R.id.text2}); 168 | 169 | messageHistory = findViewById(R.id.messageHistory); 170 | messageHistory.setOnItemClickListener(this); 171 | messageHistory.setAdapter(messageAdapter); 172 | messageText = findViewById(R.id.messageText); 173 | messageText.setOnKeyListener(this); 174 | fmt = new SimpleDateFormat("yy.MM.dd HH:mm z", Locale.US); 175 | 176 | status = findViewById(R.id.status); 177 | } 178 | 179 | @Override 180 | public void onActivityResult(int requestCode, int resultCode, Intent data) { 181 | super.onActivityResult(requestCode, resultCode, data); 182 | if (requestCode == RC_SIGN_IN) { 183 | Task signInTask = 184 | GoogleSignIn.getSignedInAccountFromIntent(data); 185 | // If Google ID authentication is successful, obtain a token for Firebase authentication. 186 | if (signInTask.isSuccessful()) { 187 | // Sign in succeeded, proceed with account 188 | GoogleSignInAccount acct = signInTask.getResult(); 189 | status.setText(getResources().getString(R.string.authenticating_label)); 190 | AuthCredential credential = GoogleAuthProvider.getCredential( 191 | acct.getIdToken(), null); 192 | FirebaseAuth.getInstance().signInWithCredential(credential) 193 | .addOnCompleteListener(this, task -> { 194 | Log.d(TAG, "signInWithCredential:onComplete Successful: " + task.isSuccessful()); 195 | if (task.isSuccessful()) { 196 | final FirebaseUser currentUser = FirebaseAuth.getInstance().getCurrentUser(); 197 | if (currentUser != null) { 198 | inbox = "client-" + Integer.toString(Math.abs(currentUser.getUid().hashCode())); 199 | requestLogger(() -> { 200 | Log.d(TAG, "onLoggerAssigned logger id: " + inbox); 201 | fbLog.log(inbox, "Signed in"); 202 | updateUI(); 203 | }); 204 | } else { 205 | updateUI(); 206 | } 207 | } else { 208 | Log.w(TAG, "signInWithCredential:onComplete", task.getException()); 209 | status.setText(String.format( 210 | getResources().getString(R.string.authentication_failed), 211 | task.getException()) 212 | ); 213 | } 214 | }); 215 | } else if (signInTask.isCanceled()) { 216 | String message = "Google authentication was canceled. " 217 | + "Verify the SHA certificate fingerprint in the Firebase console."; 218 | Log.d(TAG, message); 219 | showErrorToast(new Exception(message)); 220 | } else { 221 | String message = "Google authentication status: " + signInTask.getResult(); 222 | Log.d(TAG, message); 223 | showErrorToast(new Exception(message)); 224 | } 225 | } 226 | } 227 | 228 | @Override 229 | public void onClick(View v) { 230 | switch (v.getId()) { 231 | case R.id.sign_out_button: 232 | signOut(); 233 | break; 234 | case R.id.sign_in_button: 235 | Intent signInIntent = mGoogleSignInClient.getSignInIntent(); 236 | // Start authenticating with Google ID first. 237 | startActivityForResult(signInIntent, RC_SIGN_IN); 238 | break; 239 | case R.id.microphone_button: 240 | translateAudioMessage(v); 241 | break; 242 | } 243 | } 244 | 245 | @Override 246 | public void onItemClick(AdapterView parent, View view, int position, long id) { 247 | if (parent.getAdapter().getItem(position) instanceof Map) { 248 | Map map = (Map) parent.getAdapter().getItem(position); 249 | if (map.containsKey("gcsBucket") && map.containsKey("gcsPath")) { 250 | String gcsBucket = map.get("gcsBucket").toString(); 251 | String gcsPath = map.get("gcsPath").toString(); 252 | playMessage(gcsBucket, gcsPath); 253 | } 254 | } 255 | } 256 | 257 | private void playMessage(String gcsBucket, String gcsPath) { 258 | String filePath = gcsBucket + "/" + gcsPath; 259 | File file = new File(getFilesDir(), filePath); 260 | 261 | if (file.exists()) { 262 | MediaPlayer mediaPlayer = MediaPlayer.create( 263 | getApplicationContext(), Uri.fromFile(file)); 264 | mediaPlayer.start(); 265 | } else { 266 | GcsDownloadHelper.getInstance().downloadGcsFile( 267 | getApplicationContext(), getCronetEngine(), gcsBucket, gcsPath, 268 | new GcsDownloadHelper.GcsDownloadListener() { 269 | @Override 270 | public void onDownloadSucceeded(File file) { 271 | MediaPlayer mediaPlayer = MediaPlayer.create( 272 | getApplicationContext(), Uri.fromFile(file)); 273 | mediaPlayer.start(); 274 | } 275 | 276 | @Override 277 | public void onDownloadFailed(Exception e) { 278 | showErrorToast(e); 279 | Log.e(TAG, e.getLocalizedMessage()); 280 | } 281 | } 282 | ); 283 | } 284 | } 285 | 286 | private void signOut() { 287 | mGoogleSignInClient.signOut() 288 | .addOnCompleteListener(this, new OnCompleteListener() { 289 | @Override 290 | public void onComplete(@NonNull Task task) { 291 | FirebaseAuth.getInstance().signOut(); 292 | DatabaseReference databaseReference = FirebaseDatabase.getInstance().getReference(); 293 | databaseReference.removeEventListener(channelListener); 294 | databaseReference.onDisconnect(); 295 | inbox = null; 296 | runOnUiThread(PlayActivity.this::updateUI); 297 | fbLog.log(inbox, "Signed out"); 298 | } 299 | }); 300 | } 301 | 302 | @Override 303 | public boolean onKey(View v, int keyCode, KeyEvent event) { 304 | if ((event.getAction() == KeyEvent.ACTION_DOWN) && (keyCode == KeyEvent.KEYCODE_ENTER)) { 305 | FirebaseUser currentUser = FirebaseAuth.getInstance().getCurrentUser(); 306 | if (currentUser != null) { 307 | FirebaseDatabase.getInstance().getReference().child(CHS + "/" + currentChannel) 308 | .push() 309 | .setValue(new TextMessage(messageText.getText().toString(), currentUser.getDisplayName(), BaseMessage.MESSAGE_TYPE_TEXT)); 310 | return true; 311 | } else { 312 | return false; 313 | } 314 | } 315 | return false; 316 | } 317 | 318 | private void addMessage(String msgString, String meta) { 319 | Map message = new HashMap<>(); 320 | message.put("message", msgString); 321 | message.put("meta", meta); 322 | messages.add(message); 323 | 324 | messageAdapter.notifyDataSetChanged(); 325 | messageText.setText(""); 326 | } 327 | 328 | private void addMessage(String msgString, String meta, String gcsBucket, String gcsPath) { 329 | Map message = new HashMap<>(); 330 | // 🔈 Prepend a speaker emoji to text. 331 | message.put("message", "\uD83D\uDD08" + msgString); 332 | message.put("meta", meta); 333 | message.put("gcsBucket", gcsBucket); 334 | message.put("gcsPath", gcsPath); 335 | messages.add(message); 336 | 337 | messageAdapter.notifyDataSetChanged(); 338 | } 339 | 340 | private void translateAudioMessage(View v) { 341 | ImageButton microphoneButton = (ImageButton) v; 342 | if (RecordingHelper.getInstance().hasRequiredPermissions(getApplicationContext())) { 343 | if (!RecordingHelper.getInstance().isRecording()) { 344 | RecordingHelper.getInstance().startRecording(new RecordingHelper.RecordingListener() { 345 | @Override 346 | public void onRecordingSucceeded(File output) { 347 | String base64EncodedAudioMessage; 348 | try { 349 | base64EncodedAudioMessage = Base64EncodingHelper.encode(output); 350 | SpeechTranslationHelper.getInstance().translateAudioMessage( 351 | getApplicationContext(), 352 | getCronetEngine(), 353 | base64EncodedAudioMessage, 354 | 16000, 355 | new SpeechTranslationHelper.SpeechTranslationListener() { 356 | @Override 357 | public void onTranslationSucceeded(String responseBody) { 358 | Log.i(TAG, responseBody); 359 | try { 360 | final FirebaseUser currentUser = FirebaseAuth.getInstance().getCurrentUser(); 361 | if (currentUser != null) { 362 | SpeechMessage speechMessage = new SpeechMessage( 363 | new JSONObject(responseBody), 364 | currentUser.getDisplayName(), 365 | BaseMessage.MESSAGE_TYPE_SPEECH 366 | ); 367 | FirebaseDatabase.getInstance().getReference().child(CHS + "/" + currentChannel) 368 | .push() 369 | .setValue(speechMessage); 370 | } 371 | } catch (JSONException e) { 372 | showErrorToast(e); 373 | Log.e(TAG, e.getLocalizedMessage()); 374 | } 375 | microphoneButton.setImageDrawable(getDrawable(R.drawable.ic_baseline_mic_none_24px)); 376 | } 377 | 378 | @Override 379 | public void onTranslationFailed(Exception e) { 380 | showErrorToast(e); 381 | Log.e(TAG, e.getLocalizedMessage()); 382 | } 383 | }); 384 | } catch (IOException e) { 385 | showErrorToast(e); 386 | Log.e(TAG, e.getLocalizedMessage()); 387 | } 388 | } 389 | 390 | @Override 391 | public void onRecordingFailed(Exception e) { 392 | showErrorToast(e); 393 | Log.e(TAG, e.getLocalizedMessage()); 394 | } 395 | }); 396 | microphoneButton.setImageDrawable(getDrawable(R.drawable.ic_baseline_mic_24px)); 397 | } else { 398 | RecordingHelper.getInstance().stopRecording(); 399 | } 400 | } else { 401 | RecordingHelper.getInstance().requestRequiredPermissions(this); 402 | } 403 | } 404 | 405 | /** 406 | * Creates an instance of the CronetEngine class. 407 | * Instances of CronetEngine require a lot of resources. Additionally, their creation is slow 408 | * and expensive. It's recommended to delay the creation of CronetEngine instances until they 409 | * are required and reuse them as much as possible. 410 | * 411 | * @return An instance of CronetEngine. 412 | */ 413 | private synchronized CronetEngine getCronetEngine() { 414 | if (cronetEngine == null) { 415 | CronetEngine.Builder myBuilder = new CronetEngine.Builder(this); 416 | cronetEngine = myBuilder.build(); 417 | } 418 | return cronetEngine; 419 | } 420 | 421 | private void updateUI() { 422 | FirebaseUser currentUser = FirebaseAuth.getInstance().getCurrentUser(); 423 | if (currentUser != null) { 424 | findViewById(R.id.sign_in_button).setVisibility(View.GONE); 425 | findViewById(R.id.sign_out_button).setVisibility(View.VISIBLE); 426 | findViewById(R.id.channelLabel).setVisibility(View.VISIBLE); 427 | findViewById(R.id.messageText).setVisibility(View.VISIBLE); 428 | findViewById(R.id.messageHistory).setVisibility(View.VISIBLE); 429 | 430 | if (speechTranslationEnabled()) { 431 | findViewById(R.id.microphone_button).setVisibility(View.VISIBLE); 432 | } 433 | 434 | status.setText( 435 | String.format(getResources().getString(R.string.signed_in_label), 436 | currentUser.getDisplayName()) 437 | ); 438 | findViewById(R.id.status).setVisibility(View.VISIBLE); 439 | 440 | // Select the first channel in the array if there's no channel selected 441 | switchChannel(currentChannel != null ? currentChannel : 442 | getResources().getStringArray(R.array.channels)[0]); 443 | } else { 444 | findViewById(R.id.sign_in_button).setVisibility(View.VISIBLE); 445 | findViewById(R.id.sign_out_button).setVisibility(View.GONE); 446 | findViewById(R.id.channelLabel).setVisibility(View.GONE); 447 | findViewById(R.id.messageText).setVisibility(View.GONE); 448 | findViewById(R.id.microphone_button).setVisibility(View.GONE); 449 | findViewById(R.id.messageHistory).setVisibility(View.GONE); 450 | findViewById(R.id.status).setVisibility(View.GONE); 451 | ((TextView) findViewById(R.id.status)).setText(""); 452 | } 453 | } 454 | 455 | @Override 456 | public void onBackPressed() { 457 | DrawerLayout drawer = findViewById(R.id.drawer_layout); 458 | if (drawer.isDrawerOpen(GravityCompat.START)) { 459 | drawer.closeDrawer(GravityCompat.START); 460 | } else { 461 | super.onBackPressed(); 462 | } 463 | } 464 | 465 | @Override 466 | public boolean onNavigationItemSelected(@NonNull MenuItem item) { 467 | DrawerLayout drawer = findViewById(R.id.drawer_layout); 468 | drawer.closeDrawer(GravityCompat.START); 469 | 470 | switchChannel(item.toString()); 471 | 472 | return true; 473 | } 474 | 475 | private void switchChannel(String channel) { 476 | messages.clear(); 477 | 478 | String msg = "Switching channel to '" + channel + "'"; 479 | if (fbLog != null) 480 | fbLog.log(inbox, msg); 481 | 482 | // Switching a listener to the selected channel. 483 | DatabaseReference databaseReference = FirebaseDatabase.getInstance().getReference(); 484 | databaseReference.child(CHS + "/" + currentChannel).removeEventListener(channelListener); 485 | currentChannel = channel; 486 | databaseReference.child(CHS + "/" + currentChannel).addChildEventListener(channelListener); 487 | 488 | channelLabel.setText(currentChannel); 489 | } 490 | 491 | @Override 492 | public void onSaveInstanceState(Bundle outState) { 493 | outState.putString(CURRENT_CHANNEL_KEY, currentChannel); 494 | outState.putString(INBOX_KEY, inbox); 495 | outState.putString(FIREBASE_LOGGER_PATH_KEY, firebaseLoggerPath); 496 | super.onSaveInstanceState(outState); 497 | } 498 | 499 | @Override 500 | protected void onRestoreInstanceState(Bundle savedInstanceState) { 501 | currentChannel = savedInstanceState.getString(CURRENT_CHANNEL_KEY); 502 | inbox = savedInstanceState.getString(INBOX_KEY); 503 | firebaseLoggerPath = savedInstanceState.getString(FIREBASE_LOGGER_PATH_KEY); 504 | FirebaseUser currentUser = FirebaseAuth.getInstance().getCurrentUser(); 505 | if (currentUser != null) { 506 | fbLog = new FirebaseLogger(firebaseLoggerPath); 507 | updateUI(); 508 | } 509 | super.onRestoreInstanceState(savedInstanceState); 510 | } 511 | 512 | // [START request_logger] 513 | /* 514 | * Request that a servlet instance be assigned. 515 | */ 516 | private void requestLogger(final LoggerListener loggerListener) { 517 | final DatabaseReference databaseReference = FirebaseDatabase.getInstance().getReference(); 518 | databaseReference.child(IBX + "/" + inbox).addListenerForSingleValueEvent(new ValueEventListener() { 519 | public void onDataChange(@NonNull DataSnapshot snapshot) { 520 | if (snapshot.exists() && snapshot.getValue(String.class) != null) { 521 | firebaseLoggerPath = IBX + "/" + snapshot.getValue(String.class) + "/logs"; 522 | fbLog = new FirebaseLogger(firebaseLoggerPath); 523 | databaseReference.child(IBX + "/" + inbox).removeEventListener(this); 524 | loggerListener.onLoggerAssigned(); 525 | } 526 | } 527 | 528 | public void onCancelled(@NonNull DatabaseError error) { 529 | Log.e(TAG, error.getDetails()); 530 | } 531 | }); 532 | 533 | databaseReference.child(REQLOG).push().setValue(inbox); 534 | } 535 | // [END request_logger] 536 | 537 | /* 538 | * Initialize predefined channels as activity menu. 539 | * Once a channel is selected, ChildEventListener is attached and 540 | * waits for messages. 541 | */ 542 | private void initChannels() { 543 | String[] channelArray = getResources().getStringArray(R.array.channels); 544 | Log.d(TAG, "Channels : " + Arrays.toString(channelArray)); 545 | for (String topic : channelArray) { 546 | channelMenu.add(topic); 547 | } 548 | 549 | channelListener = new ChildEventListener() { 550 | @Override 551 | public void onChildAdded(@NonNull DataSnapshot snapshot, String prevKey) { 552 | if (snapshot.hasChild("/messageType")) { 553 | String messageType = snapshot.child("/messageType").getValue(String.class); 554 | if (messageType != null) { 555 | // Extract attributes from appropriate message object to display on the screen. 556 | if (messageType.equals(BaseMessage.MESSAGE_TYPE_TEXT)) { 557 | TextMessage message = snapshot.getValue(TextMessage.class); 558 | if (message != null) { 559 | addMessage(message.getText(), fmt.format(new Date(message.getTimeLong())) + " " 560 | + message.getDisplayName()); 561 | } 562 | } else if (messageType.equals(BaseMessage.MESSAGE_TYPE_SPEECH)) { 563 | SpeechMessage message = snapshot.getValue(SpeechMessage.class); 564 | String language = getApplicationContext() 565 | .getResources() 566 | .getConfiguration() 567 | .getLocales() 568 | .get(0).getLanguage(); 569 | if (message != null) { 570 | Translation translation = message.getTranslation(language); 571 | addMessage(translation.getText(), fmt.format(new Date(message.getTimeLong())) + " " 572 | + message.getDisplayName(), message.getGcsBucket(), translation.getGcsPath()); 573 | } 574 | } 575 | } 576 | } 577 | } 578 | 579 | @Override 580 | public void onCancelled(@NonNull DatabaseError error) { 581 | Log.e(TAG, error.getDetails()); 582 | } 583 | 584 | @Override 585 | public void onChildChanged(@NonNull DataSnapshot snapshot, String prevKey) { 586 | } 587 | 588 | @Override 589 | public void onChildRemoved(@NonNull DataSnapshot snapshot) { 590 | } 591 | 592 | @Override 593 | public void onChildMoved(@NonNull DataSnapshot snapshot, String prevKey) { 594 | } 595 | }; 596 | } 597 | 598 | private boolean speechTranslationEnabled() { 599 | String speechEndpoint = getString(R.string.speechToSpeechEndpoint); 600 | return !speechEndpoint.contains("YOUR-PROJECT-ID"); 601 | } 602 | 603 | private void showErrorToast(Exception e) { 604 | runOnUiThread( 605 | () -> Toast.makeText( 606 | getApplicationContext(), 607 | e.getLocalizedMessage(), 608 | Toast.LENGTH_LONG).show() 609 | ); 610 | } 611 | 612 | /** 613 | * A listener to get notifications about server-side loggers. 614 | */ 615 | private interface LoggerListener { 616 | /** 617 | * Called when a logger has been assigned to this client. 618 | */ 619 | void onLoggerAssigned(); 620 | } 621 | } 622 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/common/Base64EncodingHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | import android.util.Base64; 19 | import android.util.Log; 20 | 21 | import java.io.DataInputStream; 22 | import java.io.File; 23 | import java.io.FileInputStream; 24 | import java.io.IOException; 25 | 26 | /** 27 | * Abstract class that provides a method to base64 encode a file. The app encodes speech data to 28 | * send it to a Google Cloud Function that translates the speech to other languages. 29 | */ 30 | public abstract class Base64EncodingHelper { 31 | private static final String TAG = "Base64EncodingHelper"; 32 | 33 | /** 34 | * Encodes the contents to a file in base64 format and returns them as a string. 35 | * @param inputFile The file that contains the speech message to encode. 36 | * @return The base64 encoded data in a string. 37 | * @throws IOException An exception thrown when the input file is not found or can't be closed. 38 | */ 39 | // [START encode] 40 | public static String encode(File inputFile) throws IOException { 41 | byte[] data = new byte[(int) inputFile.length()]; 42 | DataInputStream input = new DataInputStream(new FileInputStream(inputFile)); 43 | int readBytes = input.read(data); 44 | Log.i(TAG, readBytes + " read from input file."); 45 | input.close(); 46 | return Base64.encodeToString(data, Base64.NO_WRAP); 47 | } 48 | // [END encode] 49 | } 50 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/common/BaseMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | import com.fasterxml.jackson.annotation.JsonIgnore; 19 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 20 | import com.google.firebase.database.ServerValue; 21 | 22 | import java.util.Map; 23 | 24 | /** 25 | * BaseMessage declares properties common to all types of messages. 26 | */ 27 | @SuppressWarnings({"WeakerAccess", "unused", "SameReturnValue"}) 28 | @JsonIgnoreProperties(ignoreUnknown = true) 29 | public abstract class BaseMessage { 30 | public static final String MESSAGE_TYPE_TEXT = "text"; 31 | public static final String MESSAGE_TYPE_SPEECH = "speech"; 32 | 33 | private static final String TAG = "BaseMessage"; 34 | 35 | private String displayName; 36 | private String messageType; 37 | private Long time; 38 | 39 | public String getDisplayName() { return displayName; } 40 | public void setDisplayName(String displayName) { this.displayName = displayName; } 41 | 42 | public String getMessageType() { return this.messageType; } 43 | public void setMessageType(String messageType) { this.messageType = messageType; } 44 | 45 | public Map getTime() { return ServerValue.TIMESTAMP; } 46 | public void setTime(Long time) { this.time = time; } 47 | 48 | @JsonIgnore 49 | public Long getTimeLong() { return time;} 50 | } 51 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/common/GcsDownloadHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | import android.content.Context; 19 | import android.net.Uri; 20 | import android.util.Log; 21 | 22 | import com.google.android.gms.tasks.OnCompleteListener; 23 | import com.google.firebase.storage.FirebaseStorage; 24 | import com.google.firebase.storage.StorageReference; 25 | 26 | import org.chromium.net.CronetEngine; 27 | import org.chromium.net.CronetException; 28 | import org.chromium.net.UrlRequest; 29 | import org.chromium.net.UrlResponseInfo; 30 | 31 | import java.io.File; 32 | import java.io.FileOutputStream; 33 | import java.io.IOException; 34 | import java.nio.ByteBuffer; 35 | import java.nio.channels.FileChannel; 36 | import java.util.concurrent.Executor; 37 | import java.util.concurrent.Executors; 38 | 39 | /** 40 | * Singleton class that makes authenticated requests to a Google Cloud Storage bucket to get the 41 | * files that contain translated speech messages. 42 | */ 43 | public class GcsDownloadHelper { 44 | private static final String GCS_DOWNLOAD_HTTP_METHOD = "GET"; 45 | private static final String TAG = "GcsDownloadHelper"; 46 | 47 | private static GcsDownloadHelper _instance; 48 | 49 | private GcsDownloadHelper() { } 50 | 51 | public static GcsDownloadHelper getInstance() { 52 | if(_instance == null) { 53 | _instance = new GcsDownloadHelper(); 54 | } 55 | return _instance; 56 | } 57 | 58 | /** 59 | * Performs an authenticated request to a Google Cloud Storage bucket to download the object in 60 | * the specified path. Returns a pointer to the downloaded file in the local storage. 61 | * @param context The application context. 62 | * @param gcsBucket The Google Cloud Storage bucket that contains the object to download. 63 | * @param gcsPath The path of the object in the bucket. 64 | * @param downloadListener The callback to deliver the results to. 65 | */ 66 | public void downloadGcsFile(Context context, CronetEngine cronetEngine, String gcsBucket, 67 | String gcsPath, GcsDownloadListener downloadListener) { 68 | String gcsUrl = "gs://" + gcsBucket + "/" + gcsPath; 69 | 70 | try { 71 | getGcsDownloadUri(gcsUrl, task -> { 72 | if (task.isSuccessful()) { 73 | Uri downloadUri = task.getResult(); 74 | if (downloadUri != null) { 75 | UrlRequest request = buildGcsRequest(context, cronetEngine, gcsBucket, gcsPath, 76 | task.getResult(), downloadListener); 77 | request.start(); 78 | } 79 | } else { 80 | Exception e = task.getException(); 81 | if (e != null) { 82 | Log.e(TAG, e.getLocalizedMessage()); 83 | downloadListener.onDownloadFailed(e); 84 | } 85 | } 86 | }); 87 | } catch(IllegalArgumentException e) { 88 | // FirebaseStorage rejected the gcsBucket. 89 | // Verify that the gcsBucket is associated with the Firebase app. 90 | Log.e(TAG, e.getLocalizedMessage()); 91 | downloadListener.onDownloadFailed(e); 92 | } 93 | } 94 | 95 | private void getGcsDownloadUri(String gcsUrl, OnCompleteListener getDownloadUriListener) 96 | throws IllegalArgumentException { 97 | // [START gcs_download_uri] 98 | FirebaseStorage storage = FirebaseStorage.getInstance(); 99 | StorageReference gsReference = storage.getReferenceFromUrl(gcsUrl); 100 | gsReference.getDownloadUrl().addOnCompleteListener(getDownloadUriListener); 101 | // [END gcs_download_uri] 102 | } 103 | 104 | private UrlRequest buildGcsRequest(Context context, CronetEngine cronetEngine, 105 | String gcsBucket, String gcsPath, Uri gcsDownloadUri, 106 | GcsDownloadListener downloadListener) { 107 | Executor executor = Executors.newSingleThreadExecutor(); 108 | 109 | UrlRequest.Builder requestBuilder = cronetEngine.newUrlRequestBuilder( 110 | gcsDownloadUri.toString(), new UrlRequest.Callback(){ 111 | FileChannel outputChannel; 112 | String localPath; 113 | File downloadedFile; 114 | @Override 115 | public void onRedirectReceived(UrlRequest request, UrlResponseInfo info, String newLocationUrl) { 116 | Log.i(TAG, "onRedirectReceived method called."); 117 | request.followRedirect(); 118 | } 119 | @Override 120 | public void onResponseStarted(UrlRequest request, UrlResponseInfo info) { 121 | Log.i(TAG, "onResponseStarted method called."); 122 | int httpStatusCode = info.getHttpStatusCode(); 123 | switch (httpStatusCode) { 124 | case 200: 125 | localPath = context.getFilesDir() + "/" + gcsBucket + "/" + gcsPath; 126 | downloadedFile = new File(localPath); 127 | try { 128 | // Check if parent folders exists 129 | File languageFolder = downloadedFile.getParentFile(); 130 | if(!languageFolder.exists()) { 131 | if(!languageFolder.mkdirs()) { 132 | String message = "Failed to create directory " 133 | + languageFolder.getAbsolutePath(); 134 | downloadListener.onDownloadFailed(new IOException(message)); 135 | } 136 | } 137 | outputChannel = new FileOutputStream(localPath).getChannel(); 138 | request.read(ByteBuffer.allocateDirect((int) info.getReceivedByteCount())); 139 | } catch (IOException e) { 140 | downloadListener.onDownloadFailed(e); 141 | } 142 | break; 143 | case 403: 144 | String errorMessage = "HTTP status code: " + httpStatusCode + 145 | ". Does the user have read access to the GCS bucket?"; 146 | downloadListener.onDownloadFailed(new SpeechTranslationException(errorMessage)); 147 | break; 148 | default: 149 | errorMessage = "Unexpected HTTP status code: " + httpStatusCode; 150 | downloadListener.onDownloadFailed(new SpeechTranslationException(errorMessage)); 151 | break; 152 | } 153 | } 154 | @Override 155 | public void onReadCompleted(UrlRequest request, UrlResponseInfo info, ByteBuffer byteBuffer) { 156 | Log.i(TAG, "onReadCompleted method called."); 157 | request.read(ByteBuffer.allocateDirect((int)info.getReceivedByteCount())); 158 | byteBuffer.flip(); 159 | try { 160 | outputChannel.write(byteBuffer); 161 | } catch (IOException e) { 162 | downloadListener.onDownloadFailed(e); 163 | } 164 | } 165 | @Override 166 | public void onSucceeded(UrlRequest request, UrlResponseInfo info) { 167 | Log.i(TAG, "onSucceeded method called."); 168 | int httpStatusCode = info.getHttpStatusCode(); 169 | if(httpStatusCode == 200) { 170 | try { 171 | outputChannel.close(); 172 | File downloadedFile = new File(localPath); 173 | downloadListener.onDownloadSucceeded(downloadedFile); 174 | } catch (IOException e) { 175 | downloadListener.onDownloadFailed(e); 176 | } 177 | } 178 | } 179 | @Override 180 | public void onFailed(UrlRequest request, UrlResponseInfo responseInfo, CronetException error) { 181 | Log.e(TAG, "The request failed.", error); 182 | downloadListener.onDownloadFailed(error); 183 | } 184 | } , executor) 185 | .setHttpMethod(GCS_DOWNLOAD_HTTP_METHOD); 186 | 187 | return requestBuilder.build(); 188 | } 189 | 190 | public interface GcsDownloadListener { 191 | void onDownloadSucceeded(File file); 192 | void onDownloadFailed(Exception e); 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/common/LogEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | import com.fasterxml.jackson.annotation.JsonIgnore; 19 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 20 | import com.google.firebase.database.ServerValue; 21 | 22 | import java.util.Map; 23 | 24 | /* 25 | * An instance of LogEntry represents a user event log, such as sign in/out and switching a channel. 26 | */ 27 | @SuppressWarnings({"WeakerAccess", "unused", "SameReturnValue"}) 28 | @JsonIgnoreProperties(ignoreUnknown = true) 29 | public class LogEntry { 30 | private String tag; 31 | private String log; 32 | private Long time; 33 | 34 | public LogEntry(String tag, String log) { 35 | this.tag = tag; 36 | this.log = log; 37 | } 38 | 39 | public String getTag() { return tag; } 40 | public void setTag(String tag) { this.tag = tag; } 41 | public String getLog() { return log; } 42 | public void setLog(String log) { this.log = log; } 43 | 44 | public Map getTime() { return ServerValue.TIMESTAMP; } 45 | public void setTime(Long time) { this.time = time; } 46 | 47 | @JsonIgnore 48 | public Long getTimeLong() { return time; } 49 | } 50 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/common/RecordingHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | import android.Manifest; 19 | import android.app.Activity; 20 | import android.content.Context; 21 | import android.content.pm.PackageManager; 22 | import android.media.AudioFormat; 23 | import android.media.AudioRecord; 24 | import android.media.MediaRecorder; 25 | import android.os.Environment; 26 | import androidx.core.content.ContextCompat; 27 | import android.util.Log; 28 | 29 | import java.io.BufferedOutputStream; 30 | import java.io.DataInputStream; 31 | import java.io.DataOutputStream; 32 | import java.io.File; 33 | import java.io.FileInputStream; 34 | import java.io.FileNotFoundException; 35 | import java.io.FileOutputStream; 36 | import java.io.IOException; 37 | import java.nio.ByteBuffer; 38 | import java.nio.ByteOrder; 39 | import java.nio.charset.Charset; 40 | 41 | /** 42 | * Singleton class that records audio from the microphone on the device and writes it to a file in 43 | * PCM-16 (wave) format. 44 | */ 45 | public class RecordingHelper { 46 | // [START recording_parameters] 47 | private static final int AUDIO_SOURCE = MediaRecorder.AudioSource.UNPROCESSED; 48 | private static final int SAMPLE_RATE_IN_HZ = 16000; 49 | private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_MONO; 50 | private static final int AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT; 51 | // [END recording_parameters] 52 | private static final int RECORD_PERMISSIONS_REQUEST_CODE = 15623; 53 | private static final String TAG = "RecordingHelper"; 54 | 55 | private static final int BUFFER_SIZE = AudioRecord.getMinBufferSize(SAMPLE_RATE_IN_HZ, CHANNEL_CONFIG, AUDIO_FORMAT); 56 | private static final String RAW_FILE_PATH = Environment.getExternalStorageDirectory().getPath() + "/speech-recording.raw"; 57 | private static final String WAV_FILE_PATH = Environment.getExternalStorageDirectory().getPath() + "/speech-recording.wav"; 58 | 59 | private static RecordingHelper _instance; 60 | 61 | private AudioRecord audioRecord; 62 | private boolean isRecording = false; 63 | private BufferedOutputStream outputStream; 64 | 65 | private RecordingHelper() { } 66 | 67 | public static RecordingHelper getInstance() { 68 | if(_instance == null) { 69 | _instance = new RecordingHelper(); 70 | } 71 | return _instance; 72 | } 73 | 74 | public boolean isRecording() { 75 | return isRecording; 76 | } 77 | 78 | /** 79 | * Starts recording audio from the device microphone. The client must call stopRecording() 80 | * before this method can process the recorded audio and write the audio file to disk. 81 | * @param recordingListener The callback to deliver the results to. 82 | */ 83 | public void startRecording(final RecordingListener recordingListener) { 84 | isRecording = true; 85 | 86 | new Thread(() -> { 87 | android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_URGENT_AUDIO); 88 | 89 | byte data[] = new byte[BUFFER_SIZE]; 90 | audioRecord = new AudioRecord( 91 | AUDIO_SOURCE, 92 | SAMPLE_RATE_IN_HZ, 93 | CHANNEL_CONFIG, 94 | AUDIO_FORMAT, 95 | BUFFER_SIZE 96 | ); 97 | 98 | audioRecord.startRecording(); 99 | 100 | try { 101 | outputStream = new BufferedOutputStream(new FileOutputStream(RAW_FILE_PATH)); 102 | } catch (FileNotFoundException e) { 103 | Log.e(TAG, "Couldn't find file: " + RAW_FILE_PATH, e); 104 | recordingListener.onRecordingFailed(e); 105 | } 106 | 107 | // This loop runs until the client calls stopRecording(). 108 | while (isRecording) { 109 | int status = audioRecord.read(data, 0, data.length); 110 | 111 | if (status == AudioRecord.ERROR_INVALID_OPERATION || status == AudioRecord.ERROR_BAD_VALUE) { 112 | Log.e(TAG, "Couldn't read data"); 113 | recordingListener.onRecordingFailed(new IOException()); 114 | } 115 | 116 | try { 117 | outputStream.write(data, 0, data.length); 118 | } catch (IOException e) { 119 | Log.e(TAG, "Couldn't save data", e); 120 | recordingListener.onRecordingFailed(e); 121 | } 122 | } 123 | 124 | // After the client calls stopRecording(), this method processes the recorded audio. 125 | try { 126 | outputStream.close(); 127 | audioRecord.stop(); 128 | audioRecord.release(); 129 | 130 | Log.v(TAG, "Recording stopped"); 131 | 132 | File rawFile = new File(RAW_FILE_PATH); 133 | File wavFile = new File(WAV_FILE_PATH); 134 | saveAsWave(rawFile, wavFile); 135 | recordingListener.onRecordingSucceeded(wavFile); 136 | } catch (IOException e) { 137 | Log.e(TAG, "File error", e); 138 | recordingListener.onRecordingFailed(e); 139 | } 140 | }).start(); 141 | } 142 | 143 | public void stopRecording() { 144 | isRecording = false; 145 | } 146 | 147 | public boolean hasRequiredPermissions(Context context) { 148 | int recordAudioPermissionCheck = ContextCompat.checkSelfPermission( 149 | context, Manifest.permission.RECORD_AUDIO); 150 | int writeExternalStoragePermissionCheck = ContextCompat.checkSelfPermission( 151 | context, Manifest.permission.WRITE_EXTERNAL_STORAGE); 152 | return recordAudioPermissionCheck == PackageManager.PERMISSION_GRANTED && 153 | writeExternalStoragePermissionCheck == PackageManager.PERMISSION_GRANTED; 154 | } 155 | 156 | public void requestRequiredPermissions(Activity activity) { 157 | activity.requestPermissions( 158 | new String[]{ 159 | Manifest.permission.RECORD_AUDIO, 160 | Manifest.permission.WRITE_EXTERNAL_STORAGE 161 | }, 162 | RECORD_PERMISSIONS_REQUEST_CODE 163 | ); 164 | } 165 | 166 | private void saveAsWave(final File rawFile, final File waveFile) throws IOException { 167 | byte[] rawData = new byte[(int) rawFile.length()]; 168 | try (DataInputStream input = new DataInputStream(new FileInputStream(rawFile))) { 169 | int readBytes; 170 | do { 171 | readBytes = input.read(rawData); 172 | } 173 | while(readBytes != -1); 174 | } 175 | 176 | try (DataOutputStream output = new DataOutputStream(new FileOutputStream(waveFile))) { 177 | // WAVE specification 178 | Charset asciiCharset = Charset.forName("US-ASCII"); 179 | // Chunk ID: "RIFF" string in US-ASCII charset—4 bytes Big Endian 180 | output.write("RIFF".getBytes(asciiCharset)); 181 | // Chunk size: The size of the actual sound data plus the rest 182 | // of this header (36 bytes)—4 bytes Little Endian 183 | output.write(convertToLittleEndian(36 + rawData.length)); 184 | // Format: "WAVE" string in US-ASCII charset—4 bytes Big Endian 185 | output.write("WAVE".getBytes(asciiCharset)); 186 | // Subchunk 1 ID: "fmt " string in US-ASCII charset—4 bytes Big Endian 187 | output.write("fmt ".getBytes(asciiCharset)); 188 | // Subchunk 1 size: The size of the subchunk. 189 | // It must be 16 for PCM—4 bytes Little Endian 190 | output.write(convertToLittleEndian(16)); 191 | // Audio format: Use 1 for PCM—2 bytes Little Endian 192 | output.write(convertToLittleEndian((short)1)); 193 | // Number of channels: This sample only supports one channel—2 bytes Little Endian 194 | output.write(convertToLittleEndian((short)1)); 195 | // Sample rate: The sample rate in hertz—4 bytes Little Endian 196 | output.write(convertToLittleEndian(SAMPLE_RATE_IN_HZ)); 197 | // Bit rate: SampleRate * NumChannels * BitsPerSample/8—4 bytes Little Endian 198 | output.write(convertToLittleEndian(SAMPLE_RATE_IN_HZ * 2)); 199 | // Block align: NumChannels * BitsPerSample/8—2 bytes Little Endian 200 | output.write(convertToLittleEndian((short)2)); 201 | // Bits per sample: 16 bits—2 bytes Little Endian 202 | output.write(convertToLittleEndian((short)16)); 203 | // Subchunk 2 ID: "fmt " string in US-ASCII charset—4 bytes Big Endian 204 | output.write("data".getBytes(asciiCharset)); 205 | // Subchunk 2 size: The size of the actual audio data—4 bytes Little Endian 206 | output.write(convertToLittleEndian(rawData.length)); 207 | 208 | // Audio data: Sound data bytes—Little Endian 209 | short[] rawShorts = new short[rawData.length / 2]; 210 | ByteBuffer.wrap(rawData).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(rawShorts); 211 | ByteBuffer bytes = ByteBuffer.allocate(rawData.length); 212 | for (short s : rawShorts) { 213 | bytes.putShort(s); 214 | } 215 | 216 | output.write(readFile(rawFile)); 217 | } 218 | } 219 | 220 | private byte[] readFile(File f) throws IOException { 221 | int size = (int) f.length(); 222 | byte bytes[] = new byte[size]; 223 | byte tmpBuff[] = new byte[size]; 224 | try (FileInputStream fis = new FileInputStream(f)) { 225 | int read = fis.read(bytes, 0, size); 226 | if (read < size) { 227 | int remain = size - read; 228 | while (remain > 0) { 229 | read = fis.read(tmpBuff, 0, remain); 230 | System.arraycopy(tmpBuff, 0, bytes, size - remain, read); 231 | remain -= read; 232 | } 233 | } 234 | } 235 | return bytes; 236 | } 237 | 238 | private byte[] convertToLittleEndian(Object value) { 239 | int size; 240 | if(value.getClass().equals(Integer.class)) { 241 | size = 4; 242 | } else if (value.getClass().equals(Short.class)) { 243 | size = 2; 244 | } else { 245 | throw new IllegalArgumentException("Only int and short types are supported"); 246 | } 247 | 248 | byte[] littleEndianBytes = new byte[size]; 249 | ByteBuffer byteBuffer = ByteBuffer.allocate(size); 250 | byteBuffer.order(ByteOrder.LITTLE_ENDIAN); 251 | 252 | if(value.getClass().equals(Integer.class)) { 253 | byteBuffer.putInt((int)value); 254 | } else if (value.getClass().equals(Short.class)) { 255 | byteBuffer.putShort((short)value); 256 | } 257 | 258 | byteBuffer.flip(); 259 | byteBuffer.get(littleEndianBytes); 260 | 261 | return littleEndianBytes; 262 | } 263 | 264 | public interface RecordingListener { 265 | void onRecordingSucceeded(File output); 266 | void onRecordingFailed(Exception e); 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/common/SpeechMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 19 | 20 | import org.json.JSONArray; 21 | import org.json.JSONException; 22 | import org.json.JSONObject; 23 | 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | 27 | /** 28 | * SpeechMessage represents a speech-based message that includes translations provided by a 29 | * Google Cloud Function. 30 | * For more information about the function implementation, see 31 | * https://github.com/GoogleCloudPlatform/nodejs-docs-samples/tree/master/functions/speech-to-speech 32 | */ 33 | @SuppressWarnings({"WeakerAccess", "unused", "SameReturnValue"}) 34 | @JsonIgnoreProperties(ignoreUnknown = true) 35 | public class SpeechMessage extends BaseMessage { 36 | private static final String TAG = "SpeechMessage"; 37 | 38 | private String gcsBucket; 39 | private String transcription; 40 | private List translations; 41 | 42 | public SpeechMessage() { } 43 | 44 | public SpeechMessage(JSONObject jsonAudioMessage, String displayName, String messageType) throws JSONException { 45 | setDisplayName(displayName); 46 | setMessageType(messageType); 47 | setTranscription(jsonAudioMessage.getString("transcription")); 48 | setGcsBucket(jsonAudioMessage.getString("gcsBucket")); 49 | 50 | JSONArray translationArray = jsonAudioMessage.getJSONArray("translations"); 51 | List translations = new ArrayList<>(); 52 | for (int i = 0; i < translationArray.length(); i++) { 53 | Translation translation = new Translation(); 54 | translation.setLanguageCode(translationArray.getJSONObject(i).getString("languageCode")); 55 | translation.setText(translationArray.getJSONObject(i).getString("text")); 56 | translation.setGcsPath(translationArray.getJSONObject(i).getString("gcsPath")); 57 | translations.add(translation); 58 | } 59 | this.translations = translations; 60 | } 61 | 62 | public String getGcsBucket() { return gcsBucket; } 63 | public void setGcsBucket(String gcsBucket) { this.gcsBucket = gcsBucket; } 64 | 65 | public String getTranscription() { 66 | return transcription; 67 | } 68 | public void setTranscription(String transcription) { 69 | this.transcription = transcription; 70 | } 71 | 72 | public Translation getTranslation(String languageCode) { 73 | Translation translation = null; 74 | for (Translation item : translations) { 75 | if (item.getLanguageCode().equals(languageCode)) { 76 | translation = item; 77 | break; 78 | } 79 | } 80 | return translation; 81 | } 82 | public List getTranslations() { 83 | return translations; 84 | } 85 | public void setTranslations(List translations) { this.translations = translations; } 86 | } 87 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/common/SpeechTranslationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | /** 19 | * Exception for speech translation related errors. 20 | */ 21 | class SpeechTranslationException extends Exception { 22 | SpeechTranslationException(String errorMessage) { 23 | super(errorMessage); 24 | } 25 | } -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/common/SpeechTranslationHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | import android.content.Context; 19 | import android.util.Log; 20 | 21 | import com.google.cloud.solutions.flexenv.R; 22 | 23 | import org.chromium.net.CronetEngine; 24 | import org.chromium.net.CronetException; 25 | import org.chromium.net.UploadDataProviders; 26 | import org.chromium.net.UrlRequest; 27 | import org.chromium.net.UrlResponseInfo; 28 | import org.json.JSONException; 29 | import org.json.JSONObject; 30 | 31 | import java.nio.ByteBuffer; 32 | import java.nio.charset.Charset; 33 | import java.util.concurrent.Executor; 34 | import java.util.concurrent.Executors; 35 | 36 | /** 37 | * Singleton class that makes requests to a Google Cloud Function that translates speech messages. 38 | * For more information about the function implementation, see 39 | * https://github.com/GoogleCloudPlatform/nodejs-docs-samples/tree/master/functions/speech-to-speech 40 | */ 41 | public class SpeechTranslationHelper { 42 | private static final String TAG = "SpeechTranslationHelper"; 43 | private static final String SPEECH_TRANSLATE_HTTP_METHOD = "POST"; 44 | private static final String SPEECH_TRANSLATE_CONTENT_TYPE = "application/json"; 45 | private static final String SPEECH_TRANSLATE_ENCODING = "LINEAR16"; 46 | 47 | private static SpeechTranslationHelper _instance; 48 | 49 | private SpeechTranslationHelper() { } 50 | 51 | public static SpeechTranslationHelper getInstance() { 52 | if(_instance == null) { 53 | _instance = new SpeechTranslationHelper(); 54 | } 55 | return _instance; 56 | } 57 | 58 | /** 59 | * Performs a request to a Google Cloud Function that translates speech messages. Returns a JSON 60 | * string with information about the response. The response includes information about the audio 61 | * files that the client can download at a different time. 62 | * @param context The application context 63 | * @param base64EncodedAudioMessage The base64-encoded audio message 64 | * @param sampleRateInHertz The sample rate in hertz 65 | * @param translationListener The callback to deliver the results to. 66 | */ 67 | public void translateAudioMessage(Context context, CronetEngine cronetEngine, 68 | String base64EncodedAudioMessage, int sampleRateInHertz, 69 | SpeechTranslationListener translationListener) { 70 | // [START json_request_body] 71 | JSONObject requestBody = new JSONObject(); 72 | try { 73 | requestBody.put("encoding", SPEECH_TRANSLATE_ENCODING); 74 | requestBody.put("sampleRateHertz", sampleRateInHertz); 75 | requestBody.put("languageCode", context.getResources().getConfiguration().getLocales().get(0)); 76 | requestBody.put("audioContent", base64EncodedAudioMessage); 77 | } catch(JSONException e) { 78 | Log.e(TAG, e.getLocalizedMessage()); 79 | translationListener.onTranslationFailed(e); 80 | } 81 | // [END json_request_body] 82 | 83 | byte[] requestBodyBytes = requestBody.toString().getBytes(); 84 | UrlRequest request = buildSpeechTranslationRequest(context, cronetEngine, requestBodyBytes, translationListener); 85 | request.start(); 86 | } 87 | 88 | private UrlRequest buildSpeechTranslationRequest(Context context, CronetEngine cronetEngine, 89 | byte[] requestBody, 90 | SpeechTranslationListener translationListener) { 91 | Executor executor = Executors.newSingleThreadExecutor(); 92 | String speechTranslateEndpoint = context.getString(R.string.speechToSpeechEndpoint); 93 | UrlRequest.Builder requestBuilder = cronetEngine.newUrlRequestBuilder( 94 | speechTranslateEndpoint, new UrlRequest.Callback(){ 95 | StringBuilder responseBodyBuilder; 96 | @Override 97 | public void onRedirectReceived(UrlRequest request, UrlResponseInfo info, String newLocationUrl) { 98 | Log.i(TAG, "onRedirectReceived method called."); 99 | request.followRedirect(); 100 | } 101 | @Override 102 | public void onResponseStarted(UrlRequest request, UrlResponseInfo info) { 103 | Log.i(TAG, "onResponseStarted method called."); 104 | int httpStatusCode = info.getHttpStatusCode(); 105 | if (httpStatusCode == 200 || httpStatusCode == 400) { 106 | responseBodyBuilder = new StringBuilder(); 107 | request.read(ByteBuffer.allocateDirect((int)info.getReceivedByteCount())); 108 | } else { 109 | request.cancel(); 110 | String errorMessage = "Unexpected HTTP status code: " + httpStatusCode; 111 | translationListener.onTranslationFailed(new SpeechTranslationException(errorMessage)); 112 | } 113 | } 114 | @Override 115 | public void onReadCompleted(UrlRequest request, UrlResponseInfo info, ByteBuffer byteBuffer) { 116 | Log.i(TAG, "onReadCompleted method called."); 117 | byteBuffer.flip(); 118 | responseBodyBuilder.append(Charset.forName("UTF-8").decode(byteBuffer).toString()); 119 | byteBuffer.clear(); 120 | request.read(byteBuffer); 121 | } 122 | @Override 123 | public void onSucceeded(UrlRequest request, UrlResponseInfo info) { 124 | Log.i(TAG, "onSucceeded method called."); 125 | int httpStatusCode = info.getHttpStatusCode(); 126 | if(httpStatusCode == 200) { 127 | translationListener.onTranslationSucceeded(responseBodyBuilder.toString()); 128 | } else if(httpStatusCode == 400) { 129 | String errorMessage = responseBodyBuilder.toString(); 130 | translationListener.onTranslationFailed(new SpeechTranslationException(errorMessage)); 131 | } 132 | } 133 | @Override 134 | public void onFailed(UrlRequest request, UrlResponseInfo responseInfo, CronetException error) { 135 | Log.e(TAG, "The request failed.", error); 136 | translationListener.onTranslationFailed(error); 137 | } 138 | } , executor) 139 | .setHttpMethod(SPEECH_TRANSLATE_HTTP_METHOD) 140 | .addHeader("Content-Type", SPEECH_TRANSLATE_CONTENT_TYPE) 141 | .setUploadDataProvider(UploadDataProviders.create(requestBody), executor); 142 | 143 | return requestBuilder.build(); 144 | } 145 | 146 | public interface SpeechTranslationListener { 147 | void onTranslationSucceeded(String responseBody); 148 | void onTranslationFailed(Exception e); 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/common/TextMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 19 | 20 | /** 21 | * TextMessage represents a text-based message. 22 | */ 23 | @SuppressWarnings({"WeakerAccess", "unused", "SameReturnValue"}) 24 | @JsonIgnoreProperties(ignoreUnknown = true) 25 | public class TextMessage extends BaseMessage { 26 | private static final String TAG = "TextMessage"; 27 | 28 | private String text; 29 | 30 | public TextMessage() { } 31 | 32 | public TextMessage(String text, String displayName, String messageType) { 33 | setText(text); 34 | setDisplayName(displayName); 35 | setMessageType(messageType); 36 | } 37 | 38 | public String getText() { return text; } 39 | public void setText(String text) { this.text = text; } 40 | } 41 | -------------------------------------------------------------------------------- /app/src/main/java/com/google/cloud/solutions/flexenv/common/Translation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google LLC. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package com.google.cloud.solutions.flexenv.common; 17 | 18 | /** 19 | * Represents a translation included in a speech-based message. The translation is provided by a 20 | * Google Cloud Function. For more information about the function implementation, see 21 | * https://github.com/GoogleCloudPlatform/nodejs-docs-samples/tree/master/functions/speech-to-speech 22 | */ 23 | @SuppressWarnings({"WeakerAccess", "unused"}) 24 | public class Translation { 25 | private String gcsPath; 26 | private String languageCode; 27 | private String text; 28 | 29 | public Translation() { } 30 | 31 | public String getGcsPath() { return gcsPath; } 32 | public void setGcsPath(String gcsPath) { this.gcsPath = gcsPath; } 33 | 34 | public String getLanguageCode() { return languageCode; } 35 | public void setLanguageCode(String languageCode) { this.languageCode = languageCode; } 36 | 37 | public String getText() { return text; } 38 | public void setText(String text) { this.text = text; } 39 | } -------------------------------------------------------------------------------- /app/src/main/res/drawable/ic_baseline_mic_24px.xml: -------------------------------------------------------------------------------- 1 | 6 | 9 | 10 | -------------------------------------------------------------------------------- /app/src/main/res/drawable/ic_baseline_mic_none_24px.xml: -------------------------------------------------------------------------------- 1 | 6 | 9 | 10 | -------------------------------------------------------------------------------- /app/src/main/res/drawable/mic_button_selector.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /app/src/main/res/drawable/side_nav_bar.xml: -------------------------------------------------------------------------------- 1 | 3 | 9 | -------------------------------------------------------------------------------- /app/src/main/res/layout/activity_play.xml: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | 15 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /app/src/main/res/layout/app_bar_play.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 14 | 15 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /app/src/main/res/layout/content_play.xml: -------------------------------------------------------------------------------- 1 | 2 | 20 | 21 | 22 | 29 | 30 | 39 | 40 |