├── .github └── ISSUE_TEMPLATE │ └── bug-report--custom-deployment-.md ├── .gitignore ├── LICENSE ├── ThirdPartyContributors.md ├── docs ├── README.md ├── README_dfki_mcqdev.md ├── assets │ ├── always_location_permission_android.mp4 │ ├── always_location_permission_ios.mp4 │ ├── android_app_data_restriction.mp4 │ ├── apply_new_ui_update.mp4 │ ├── confirmation_example.mp4 │ ├── dont_force_kill_video.mp4 │ ├── dont_force_kill_video_with_text.mp4 │ ├── e-mission-both │ │ ├── firefox_webdev_menu.png │ │ ├── gforms_survey_response.png │ │ ├── google_auth_client_id_creation_screen.png │ │ ├── google_auth_client_id_selection.png │ │ ├── google_auth_client_id_success.png │ │ ├── google_auth_screenshot.png │ │ ├── launch_survey_sample_code.png │ │ ├── launch_survey_sample_ui.png │ │ ├── sel_xpath_copy.png │ │ └── uuid_input_field_selection.png │ ├── editedAndroid_10.mp4 │ ├── editedAndroid_11.mp4 │ ├── editedAndroid_6_9.mp4 │ ├── editediOS_11_12.mp4 │ ├── editediOS_14.mp4 │ ├── future_work │ │ ├── benchmarking_strategy_static.png │ │ ├── graphene-arch.png │ │ ├── graphene_arch_diagram.png │ │ ├── intel-SDK-arch.png │ │ ├── privacy_examples_bad.png │ │ ├── privacy_final_design.png │ │ └── scone-arch.png │ ├── ios_turn_off_background_data.MP4 │ ├── ios_turn_off_cellular.mp4 │ ├── overview │ │ ├── easiest_way_to_change_text │ │ │ ├── branch_selection_github_ui.png │ │ │ └── navigate_to_file_and_edit.png │ │ ├── meco2_demo_short.mp4 │ │ ├── mode_purpose_short.mp4 │ │ ├── quickstart │ │ │ ├── diary_view_success.png │ │ │ ├── expected_ios_error.png │ │ │ └── login_screen_test_user.png │ │ ├── survey_demo_shorter.mp4 │ │ └── tripaware_emotion_demo_short.mp4 │ ├── test_changes_to_a_plugin │ │ ├── copy-project-url.png │ │ └── fork-project.png │ └── tutorials │ │ ├── change_trip_end_prompt │ │ ├── default_prompt_android.png │ │ ├── default_prompt_expand_ios.png │ │ ├── default_prompt_expand_view_ios.png │ │ ├── default_prompt_ios.png │ │ ├── new_prompt_android.png │ │ ├── new_prompt_expand_view_ios.png │ │ └── new_prompt_ios.png │ │ └── simple_client_changes │ │ ├── consent_for_trantor_study.png │ │ ├── how_to_logout.png │ │ └── new_title_my_cool_app.png ├── contribute_to_the_doc │ ├── CONTRIBUTING.md │ └── migration_reason.md ├── dev │ ├── archi │ │ ├── data_model.md │ │ ├── mode_inference_pipeline_design.md │ │ ├── module_structure.md │ │ └── pipeline_details.md │ ├── back │ │ ├── adding_a_new_data_type.md │ │ ├── algo_changes_outcome.md │ │ ├── config_environment_variables.md │ │ └── supporting_user_inputs.md │ ├── front │ │ ├── accessing_public_heatmap_and_incident_information.md │ │ ├── create_a_new_custom_client.md │ │ ├── guidelines_to_support_i18n.md │ │ ├── high_level_faq.md │ │ ├── how_to_add_a_language.md │ │ ├── how_to_embed_an_external_survey_in_the_app.md │ │ ├── how_to_test_changes _to_a_plugin.md │ │ ├── loading_data_for_emulator.md │ │ ├── phone_module_structure.md │ │ ├── samples │ │ │ ├── test_external_surveys.patch │ │ │ └── test_survey.html │ │ ├── test_translations_in_devapp.md │ │ ├── trip_labelling_options.md │ │ └── you_have_a_branch_for_your_custom_client_now_what.md │ ├── future │ │ ├── analysis.md │ │ ├── collabathon_ideas.md │ │ ├── differential_privacy_deep_dive_lit_review.md │ │ ├── foundations.md │ │ ├── more_custom_auto_config.md │ │ ├── overview.md │ │ ├── privacy.md │ │ ├── scalability.md │ │ ├── secure_execution_options.md │ │ └── unanswered_questions.md │ ├── publicdashboard │ │ └── Working_with_public_dashboard.md │ └── tutorial │ │ ├── 2018-08-20.2018-08-20.timeline │ │ ├── 2018-08-21.2018-08-21.timeline │ │ ├── android-debugging.md │ │ ├── hands-on-overview.md │ │ ├── logistics.md │ │ └── small_ui_changes │ │ ├── change_trip_end_prompt.md │ │ ├── changes_needed_when_the_set_of_tabs_changes.md │ │ ├── easiest_way_to_change_text.md │ │ ├── quickstart.md │ │ ├── remove_tab.md │ │ ├── scheduling-trip-end-prompt.md │ │ ├── simple_client_changes.md │ │ └── summary_and_consent.md ├── install │ ├── configuring_authentication.md │ ├── deploying_production_docker.md │ ├── deploying_your_own_server_to_production_manual_install.md │ ├── manual_install.md │ └── multi_layer_stack.md ├── manage │ ├── adding_limesurvey_support.md │ ├── pushing_surveys_from_the_server_to_the_phone.md │ ├── requesting_data_as_a_collaborator.md │ ├── troubleshooting_app_faq.md │ └── troubleshooting_server_faq.md ├── project_committee.md └── use │ ├── data_monitoring.md │ ├── deployments.md │ ├── end_user.md │ └── start_a_project.md ├── mkdocs.yml ├── readthedocs.yml └── requirements.txt /.github/ISSUE_TEMPLATE/bug-report--custom-deployment-.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report (custom deployment) 3 | about: Create a report to help us improve 4 | title: "\U0001F41B " 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | Please add screenshots (redacted appropriately) to help explain your problem. Videos would be even better, if they do not contain privacy-sensitive information. 25 | 26 | **If you have a custom deployment, please indicate which commit is at HEAD in your deployment:** 27 | - Server: [e.g. 7b031eed8399e9b4dbb1ffd9581f5d703d0e5078] 28 | - Phone [e.g. 420cd2f84b1841be427a533e48527c85ff1b60dd] 29 | - Admin dashboard [if relevant]: 30 | - Public dashboard [if relevant]: 31 | 32 | **If you are using an NREL deployment (e.g. at XXX-openpath.nrel.gov):** 33 | - Phone version from the app or play store 34 | 35 | **Smartphone (please complete the following information):** 36 | - Device: [e.g. iPhone6] 37 | - OS: [e.g. iOS8.1] 38 | - Version [e.g. 22] 39 | 40 | **Please ** 41 | Add any other context about the problem here. 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2020-2021, Alliance for Sustainable Energy 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /ThirdPartyContributors.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2020, UC Regents 2 | 3 | From its inception to mid-2020, this project was supported by Prof. David 4 | Culler and Prof. Randy Katz at UC Berkeley through various grants. 5 | 6 | In addition to NSF CISE Expeditions Award CCF-1730628, this research was 7 | supported in part by gifts from Alibaba, Amazon Web Services, Ant Financial, 8 | Arm, CapitalOne, Ericsson, Facebook, Google, Huawei, Intel, Microsoft, 9 | Scotiabank, Splunk and VMware. This work was also supported in part by the 10 | CONIX Research Center, one of six centers in JUMP, a Semiconductor Research 11 | Corporation (SRC) program sponsored by DARPA. 12 | 13 | Contributions owned by their respective contributors. 14 | 15 | This project would not be possible without contributors (code, documentation, 16 | issues, ideas) from across the world. Thank you! 17 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Welcome to the NREL OpenPATH Documentation! 2 | 3 | [![Documentation Status](https://readthedocs.org/projects/e-mission/badge/?version=latest)](https://e-mission.readthedocs.io/en/latest/?badge=latest) 4 | 5 | This repo contains all the documentation for the e-mission project, and [almost ALL THE ISSUES](https://github.com/e-mission/e-mission-docs/issues/). More information on the rationale behind the change can be found in the [migration reasons](contribute_to_the_doc/migration_reason.md). 6 | 7 | There are some specialized READMEs [in the individual repositories](https://github.com/e-mission), but they are gradually being moved in here. This means that if you have any questions, you should first search here and if you don't find any [existing issues](https://github.com/e-mission/e-mission-docs/issues/), you should [file an issue here](https://github.com/e-mission/e-missiond-docs/issue). 8 | 9 | ## Getting started 10 | - [NREL OpenPATH web site](https://nrel.gov/openpath) 11 | - [CODE on GitHub](https://github.com/e-mission) 12 | - [LICENCE BSD-3](https://github.com/e-mission/e-mission-docs/blob/master/docs/LICENSE.md) 13 | - Thanks to [@PatGendre](https://github.com/PatGendre) from [FabMob](https://github.com/fabmob/), you can now [Read The Docs](https://e-mission.readthedocs.io/en/latest/) 14 | 15 | ## What is NREL OpenPATH (née e-mission) 16 | NREL OpenPATH (née e-mission) is an open source mobility platform developed at the [RISE](http://rise.cs.berkeley.edu/) and [BETS](https://bets.cs.berkeley.edu/) labs in the UC Berkeley EECS Department. It is currently maintained by the National Renewable Energy Laboratory (NREL) with support from the US Department of Energy (DOE) Vehicle Technologies Office (VTO). 17 | 18 | E-mission includes a mobile application for Android and iOS, with user consent, automatically collects the user's travel patterns and sends them to the server so as to derive personal mobility information and analyses; depending on user consent for sharing his/her data, the data can also be used for aggregate mobility data studies. The application is also a tool for collecting information filled in by the user (such as incidents, ground truth information about his/her trip purpose and transportation mode, or answers to questions asked in external surveys). 19 | The server is a python web application, the data is stored in a mongoDB database; 20 | the client is a Cordova application for both Android and iOS. 21 | The application has been initially designed to be reused in research and academic projects either for conducting and as a good learning project for CS students. It is also freely reusable for [any other user cases](use/deployments.md). If you are using e-mission in your work, please submit a PR, or send shankari@eecs.berkeley.edu an email so that you can be added to this list. 22 | 23 | ### End to end apps 24 | @PatGendre generated this list of closed source apps that are similar to e-mission. Since Patrick is from France, they have a strong European focus. If you would like to add other companies to this list, please [send a pull request](contribute_to_the_doc/CONTRIBUTING.md). 25 | 26 | | Company | Region | Company | Region | Company | Region | Company | Region | 27 | |---------------|--------------|-------------|--------------|---------------|------------|---------------|------------| 28 | | [MOBIDOT](http://www.mobidot.nl/en/about_mobidot.php) | NL | [MotionTag](https://motion-tag.com/en/mobility/) | DE | [TravelVu](https://en.trivector.se/it-systems/travelvu/) | SE | [RMove](https://rmove.rsginc.com/) | US | 29 | | [radar.io](https://radar.io/documentation) | US | [BetterPoint](https://www.betterpoints.ltd/about-us/) | UK | [TrackAndKnow](https://trackandknowproject.eu/) | EU-wide | | | 30 | 31 | ### SDKs 32 | Although e-mission is an end-to-end system, it is also modular. The sensing components are cordova plugins and can be incorporated into any app. The server modules are open source and should be able to process any data in the correct format. All the deployments so far, [with the exception of UW](https://github.com/e-mission/e-mission-docs/issues/501), have focused on customizing the custom app UI and have not really modified the plugin list. If you are interested in experimenting with plugin subsets, I am happy to work more closely with you. Please also [let the community know how it goes](https://github.com/e-mission/e-mission-docs/issues/505). @PatGendre also generated this comparative list of background location tracking SDKs. If you would like to add other SDKs to this list, please [send a pull request](contribute_to_the_doc/CONTRIBUTING.md). 33 | 34 | | Project | Open/Closed source | Notes | 35 | |---------------|--------------------|-------| 36 | | [Itinerum](https://www.itinerum.ca/) | Open source |app, not SDK. will merge in 2020, pending funding | 37 | | [TransistorSoft Background Geolocation](https://github.com/transistorsoft/react-native-background-geolocation)| Closed source | Much richer set of interfaces, including react native and flutter | 38 | | [Sentiance](https://docs.sentiance.com/sdk/) | Closed source | Includes trip and mode detection | 39 | | [Radar.io](https://radar.io/documentation/sdk) | Closed source | the API seems to be limited to geofences, focused on retail? | 40 | | [LocoKit](https://www.bigpaua.com/locokit/docs/) | Open source | iOS only, "ML based" | 41 | | [moprim](https://www.moprim.com/products/) | Closed source | no public documentation | 42 | 43 | ## Gallery (a first glimpse at the app UI) 44 | E-mission has been reused in several projects, here are [short videos showing different versions of the UI](https://nextcloud.damajash.org/s/MTE4y3tJeX4g6sG), which have been presented at the last TRB conference in January 2020 : 45 | - E-mission provides [personal mobility metrics including CO2 emissions](https://nextcloud.damajash.org/s/aFPRpfLYy9GPoe2) (MeCO2 a CO2 coach developed in Berlin by DFKI's OSLab) 46 | - E-mission can [let the end-user complete the mode and purpose](https://nextcloud.damajash.org/s/f2z28gK7e84WPWS) after trip completion 47 | - E-mission can [embark third-pary online surveys](https://nextcloud.damajash.org/s/CRZg4zQY2gKWMnF) (a feature developed by the University of New South Wales) 48 | - E-mission can be adapted to [mobility nudge experimentations like the Tripaware project](https://nextcloud.damajash.org/s/n2ESGrRYdCmSaX6) conducted at UC Berkley in 2018 49 | - E-mission can be integrated with external survey tools by [automatically filling in survey fields with sensed information](https://nextcloud.damajash.org/s/4CxeNpWDAiqSkpa). The survey prompt can [even be context sensitive](https://nextcloud.damajash.org/s/3qp8Mcwcy36HjSi). 50 | 51 | ## Overview 52 | Read these papers for understand context. 53 | 54 | - [TRB paper describing how to use e-mission functionality]([https://people.eecs.berkeley.edu/~shankari/emission_trb_2017_paper.pdf](https://www.researchgate.net/publication/327115169_e-mission_An_Open-Source_Smartphone_Platform_for_Collecting_Human_Travel_Data)) 55 | - [The in-review paper on the e-mission architecture, please do not distribute](https://www2.eecs.berkeley.edu/Pubs/TechRpts/2019/EECS-2019-88.html) 56 | - e-mission started as [Shankari's PhD thesis](https://www2.eecs.berkeley.edu/Pubs/TechRpts/2019/EECS-2019-180.html) 57 | - A proof of concept of [user private clouds in @njriasan's thesis](https://www2.eecs.berkeley.edu/Pubs/TechRpts/2020/EECS-2020-130.html) 58 | - A proof of concept for differentially private queries aggregating information over UPC [in @jackcsullivan's thesis](https://www2.eecs.berkeley.edu/Pubs/TechRpts/2019/EECS-2019-69.html) 59 | 60 | ## roadmap 61 | - The next features and enhancement can be guessed from the [ISSUES](https://github.com/e-mission/e-mission-docs/issues) 62 | - We have formed a [project commmittee](project_committee.md) to help guide the direction and roadmap 63 | - The current large scale research features are described in [Develop/Future](dev/future/overview.md) 64 | -------------------------------------------------------------------------------- /docs/README_dfki_mcqdev.md: -------------------------------------------------------------------------------- 1 | We were contacted by DFKI to create an emission tracking app for Android and iOS that While first also considering development of a new code base, we finally stayed on the path of contributing to the existing repository because it wouldn't be a big difference in time needed for development but we could contribute to a greater project and maybe also profit from future community updates. 2 | 3 | The project is based on Cordova with Ionic, though it uses very old version and also lots of deprecated npm packages which makes a lot of newer Ionic features unavailable. 4 | 5 | Documentation is located in e-mission-docs repository. 6 | 4 are the relevant repositories for the DFKI project: 7 | 1. e-mission-phone: Most of our development happens here. This is the code for all the frontend phone stuff. Since most of the logic happens on the server side or in dedicated Cordova plugins,that have their own respective repository, this really is mostly frontend. 8 | 2. E-mission-server: this is the server that processes all trips in the pipeline. We have our own instance of this server running on digital ocean but there isn't a DFKI specific version for this. The code for the server to be use is from the gis-mode-detection branch. 9 | 3. e-mission-docs: a central documentation repository for all e-mission projects and parts. 10 | 4. E-mission-devapp: the modification of phonegap devapp that will track location in background even when there is no app connected. 11 | 12 | -------------------------------------------------------------------------------- /docs/assets/always_location_permission_android.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/always_location_permission_android.mp4 -------------------------------------------------------------------------------- /docs/assets/always_location_permission_ios.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/always_location_permission_ios.mp4 -------------------------------------------------------------------------------- /docs/assets/android_app_data_restriction.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/android_app_data_restriction.mp4 -------------------------------------------------------------------------------- /docs/assets/apply_new_ui_update.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/apply_new_ui_update.mp4 -------------------------------------------------------------------------------- /docs/assets/confirmation_example.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/confirmation_example.mp4 -------------------------------------------------------------------------------- /docs/assets/dont_force_kill_video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/dont_force_kill_video.mp4 -------------------------------------------------------------------------------- /docs/assets/dont_force_kill_video_with_text.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/dont_force_kill_video_with_text.mp4 -------------------------------------------------------------------------------- /docs/assets/e-mission-both/firefox_webdev_menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/e-mission-both/firefox_webdev_menu.png -------------------------------------------------------------------------------- /docs/assets/e-mission-both/gforms_survey_response.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/e-mission-both/gforms_survey_response.png -------------------------------------------------------------------------------- /docs/assets/e-mission-both/google_auth_client_id_creation_screen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/e-mission-both/google_auth_client_id_creation_screen.png -------------------------------------------------------------------------------- /docs/assets/e-mission-both/google_auth_client_id_selection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/e-mission-both/google_auth_client_id_selection.png -------------------------------------------------------------------------------- /docs/assets/e-mission-both/google_auth_client_id_success.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/e-mission-both/google_auth_client_id_success.png -------------------------------------------------------------------------------- /docs/assets/e-mission-both/google_auth_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/e-mission-both/google_auth_screenshot.png -------------------------------------------------------------------------------- /docs/assets/e-mission-both/launch_survey_sample_code.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/e-mission-both/launch_survey_sample_code.png -------------------------------------------------------------------------------- /docs/assets/e-mission-both/launch_survey_sample_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/e-mission-both/launch_survey_sample_ui.png -------------------------------------------------------------------------------- /docs/assets/e-mission-both/sel_xpath_copy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/e-mission-both/sel_xpath_copy.png -------------------------------------------------------------------------------- /docs/assets/e-mission-both/uuid_input_field_selection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/e-mission-both/uuid_input_field_selection.png -------------------------------------------------------------------------------- /docs/assets/editedAndroid_10.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/editedAndroid_10.mp4 -------------------------------------------------------------------------------- /docs/assets/editedAndroid_11.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/editedAndroid_11.mp4 -------------------------------------------------------------------------------- /docs/assets/editedAndroid_6_9.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/editedAndroid_6_9.mp4 -------------------------------------------------------------------------------- /docs/assets/editediOS_11_12.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/editediOS_11_12.mp4 -------------------------------------------------------------------------------- /docs/assets/editediOS_14.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/editediOS_14.mp4 -------------------------------------------------------------------------------- /docs/assets/future_work/benchmarking_strategy_static.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/future_work/benchmarking_strategy_static.png -------------------------------------------------------------------------------- /docs/assets/future_work/graphene-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/future_work/graphene-arch.png -------------------------------------------------------------------------------- /docs/assets/future_work/graphene_arch_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/future_work/graphene_arch_diagram.png -------------------------------------------------------------------------------- /docs/assets/future_work/intel-SDK-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/future_work/intel-SDK-arch.png -------------------------------------------------------------------------------- /docs/assets/future_work/privacy_examples_bad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/future_work/privacy_examples_bad.png -------------------------------------------------------------------------------- /docs/assets/future_work/privacy_final_design.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/future_work/privacy_final_design.png -------------------------------------------------------------------------------- /docs/assets/future_work/scone-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/future_work/scone-arch.png -------------------------------------------------------------------------------- /docs/assets/ios_turn_off_background_data.MP4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/ios_turn_off_background_data.MP4 -------------------------------------------------------------------------------- /docs/assets/ios_turn_off_cellular.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/ios_turn_off_cellular.mp4 -------------------------------------------------------------------------------- /docs/assets/overview/easiest_way_to_change_text/branch_selection_github_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/overview/easiest_way_to_change_text/branch_selection_github_ui.png -------------------------------------------------------------------------------- /docs/assets/overview/easiest_way_to_change_text/navigate_to_file_and_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/overview/easiest_way_to_change_text/navigate_to_file_and_edit.png -------------------------------------------------------------------------------- /docs/assets/overview/meco2_demo_short.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/overview/meco2_demo_short.mp4 -------------------------------------------------------------------------------- /docs/assets/overview/mode_purpose_short.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/overview/mode_purpose_short.mp4 -------------------------------------------------------------------------------- /docs/assets/overview/quickstart/diary_view_success.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/overview/quickstart/diary_view_success.png -------------------------------------------------------------------------------- /docs/assets/overview/quickstart/expected_ios_error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/overview/quickstart/expected_ios_error.png -------------------------------------------------------------------------------- /docs/assets/overview/quickstart/login_screen_test_user.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/overview/quickstart/login_screen_test_user.png -------------------------------------------------------------------------------- /docs/assets/overview/survey_demo_shorter.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/overview/survey_demo_shorter.mp4 -------------------------------------------------------------------------------- /docs/assets/overview/tripaware_emotion_demo_short.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/overview/tripaware_emotion_demo_short.mp4 -------------------------------------------------------------------------------- /docs/assets/test_changes_to_a_plugin/copy-project-url.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/test_changes_to_a_plugin/copy-project-url.png -------------------------------------------------------------------------------- /docs/assets/test_changes_to_a_plugin/fork-project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/test_changes_to_a_plugin/fork-project.png -------------------------------------------------------------------------------- /docs/assets/tutorials/change_trip_end_prompt/default_prompt_android.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/tutorials/change_trip_end_prompt/default_prompt_android.png -------------------------------------------------------------------------------- /docs/assets/tutorials/change_trip_end_prompt/default_prompt_expand_ios.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/tutorials/change_trip_end_prompt/default_prompt_expand_ios.png -------------------------------------------------------------------------------- /docs/assets/tutorials/change_trip_end_prompt/default_prompt_expand_view_ios.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/tutorials/change_trip_end_prompt/default_prompt_expand_view_ios.png -------------------------------------------------------------------------------- /docs/assets/tutorials/change_trip_end_prompt/default_prompt_ios.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/tutorials/change_trip_end_prompt/default_prompt_ios.png -------------------------------------------------------------------------------- /docs/assets/tutorials/change_trip_end_prompt/new_prompt_android.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/tutorials/change_trip_end_prompt/new_prompt_android.png -------------------------------------------------------------------------------- /docs/assets/tutorials/change_trip_end_prompt/new_prompt_expand_view_ios.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/tutorials/change_trip_end_prompt/new_prompt_expand_view_ios.png -------------------------------------------------------------------------------- /docs/assets/tutorials/change_trip_end_prompt/new_prompt_ios.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/tutorials/change_trip_end_prompt/new_prompt_ios.png -------------------------------------------------------------------------------- /docs/assets/tutorials/simple_client_changes/consent_for_trantor_study.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/tutorials/simple_client_changes/consent_for_trantor_study.png -------------------------------------------------------------------------------- /docs/assets/tutorials/simple_client_changes/how_to_logout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/tutorials/simple_client_changes/how_to_logout.png -------------------------------------------------------------------------------- /docs/assets/tutorials/simple_client_changes/new_title_my_cool_app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/e-mission/e-mission-docs/70e233766d31de3ad393e20910db40aa5fb00265/docs/assets/tutorials/simple_client_changes/new_title_my_cool_app.png -------------------------------------------------------------------------------- /docs/contribute_to_the_doc/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to the documentation 2 | 3 | 1. [Create a fork to make your changes](https://guides.github.com/activities/forking/) 4 | 1. Create a branch for your change 5 | 1. Add documentation pages to `docs`, and supporting images to `assets`. You can do this directly through the github UI by creating or uploading files. Note the structure for `assets`. 6 | 1. [Link images to text](https://guides.github.com/features/mastering-markdown/) 7 | 1. Submit a pull request 8 | 1. Please use relative paths (e.g. `../../assets/...`) instead of absolute paths (e.g. https://github.com/e-mission/e-mission-docs/assets/...) while including images. This will ensure that they work even after merging and cloning. 9 | 10 | 11 | ## Overall structure ## 12 | 13 | This ASCII diagram shows the parallel tree structures for the docs and assets. 14 | It is simplified and adapted from https://github.com/phonegap/phonegap-docs 15 | 16 | ``` 17 | e-mission-docs/ 18 | | 19 | |__ assets/ # Images, etc 20 | | |__ overview/ # overview, end to end documentation that stiches together various components 21 | | | |__ page1/ # documentation page image directory 22 | | | |__ image1.png # image in page1 23 | | | |__ image2.png # image in page1 24 | | | |__ ... # image in page1 25 | | | 26 | | |__ tutorials/ # step by step instructions to get users more familiar with making changes 27 | | 28 | |__ docs/ # The documentation 29 | | | 30 | | |__ overview/ # overview, end to end documentation that stiches together various components 31 | | | |__ page1.md # documentation pages 32 | | | .... 33 | | | 34 | | |__ tutorials/ # step by step instructions to get users more familiar with making changes 35 | ``` 36 | 37 | -------------------------------------------------------------------------------- /docs/contribute_to_the_doc/migration_reason.md: -------------------------------------------------------------------------------- 1 | ## Reasons for migrating all the docs to one repository ## 2 | 3 | Before this migration, all docs were in wikis. But that structure had several problems: 4 | - since we had separate repositories for the components that should work together to form an end-to-end system, there was no single unifying location to describe the high level overview 5 | - we had to sometimes duplicate documentation across repositories, which makes it challenging to keep them in sync 6 | - the community cannot submit documentation since only users with write access can update the wiki 7 | 8 | In order to address this, we create a separate repository for documentation. 9 | - All the end-to-end stuff can go here. 10 | - The community can contribute by submitting pull requests that I can merge after review. 11 | - The documentation can still be in markdown so that it can be directly edited in github without worrying about the git workflow. 12 | - Later, we can use a static website generator such as DocPad (https://docpad.org), Hugo (https://gohugo.io/) or readthedocs (https://readthedocs.org/) to make a prettier looking site. 13 | -------------------------------------------------------------------------------- /docs/dev/archi/data_model.md: -------------------------------------------------------------------------------- 1 | # Data model # 2 | 3 | One of the frequently asked questions by projects that use e-mission is "where 4 | is the data model"? The answer is that it is stored in code. 5 | 6 | The main reason is that there is no dedicated documentation team to keep the 7 | code and documentation up to date, and I didn't want to cause confusion by 8 | having obsolete docs. 9 | 10 | So the data model is represented by wrapper classes, stored in `emission/core/wrapper`. 11 | - Each python wrapper class contains the list of fields, along with a brief description. 12 | - Classes can inherit from other classes (e.g. `cleaned_trip` inherits from `trip`) 13 | - `Entry` is a special class that represents an entry in the timeseries, 14 | including both `data` and `metadata`. 15 | - A full list of classes, including a brief description of each type, is in 16 | `emission/core/wrapper/entry.py` 17 | - The class represents the `data` part of an `entry` 18 | 19 | The whole wrapper mechanism is based on the `attrdict` module, with some 20 | extensions for validating not just which fields exist in the object, but 21 | which are valid for a particular class. 22 | 23 | ``` 24 | In [1]: import emission.core.get_database as edb 25 | Connecting to database URL localhost 26 | 27 | In [2]: import emission.core.wrapper.entry as ecwe 28 | 29 | In [3]: entry_dict = edb.get_timeseries_db().find_one() 30 | 31 | In [4]: entry_dict["metadata"]["key"] 32 | Out[4]: 'stats/server_api_time' 33 | 34 | In [5]: entry = ecwe.Entry(entry_dict) 35 | 36 | In [6]: entry.metadata.key 37 | Out[6]: 'stats/server_api_time' 38 | 39 | In [7]: type(entry) 40 | Out[7]: emission.core.wrapper.entry.Entry 41 | 42 | In [8]: type(entry.data) 43 | Out[8]: emission.core.wrapper.statsevent.Statsevent 44 | 45 | In [9]: entry.data.name 46 | Out[9]: 'POST_/usercache/get' 47 | 48 | In [10]: entry.data.reading 49 | Out[10]: 0.41276121139526367 50 | ``` 51 | 52 | The wrappers also so some basic validation of attributes so that errors can be 53 | caught at compile time. Even if that is a bit un-pythonic :) 54 | 55 | ``` 56 | In [11]: entry.beta 57 | --------------------------------------------------------------------------- 58 | AttributeError Traceback (most recent call last) 59 | in () 60 | ----> 1 entry.beta 61 | 62 | /Users/shankari/e-mission/e-mission-server/emission/core/wrapper/wrapperbase.py in __getattr__(self, key) 63 | 73 return self._build(key, self[key]) 64 | 74 else: 65 | ---> 75 raise AttributeError("property %s is not defined for %s" % (key, self.__class__.__name__)) 66 | 76 67 | 77 def _writable(self, key): 68 | 69 | AttributeError: property beta is not defined for Entry 70 | 71 | In [12]: entry.data.beading 72 | --------------------------------------------------------------------------- 73 | AttributeError Traceback (most recent call last) 74 | in () 75 | ----> 1 entry.data.beading 76 | 77 | /Users/shankari/e-mission/e-mission-server/emission/core/wrapper/wrapperbase.py in __getattr__(self, key) 78 | 73 return self._build(key, self[key]) 79 | 74 else: 80 | ---> 75 raise AttributeError("property %s is not defined for %s" % (key, self.__class__.__name__)) 81 | 76 82 | 77 def _writable(self, key): 83 | 84 | AttributeError: property beading is not defined for Statsevent 85 | ``` 86 | 87 | Note that, in order to be reproducible, our data model is designed to be 88 | **read-only**. This means that if you work on some data and generate some 89 | output, that output is a separate object which you should store separately. 90 | This ensures that we can blow away all analysis results at any time and 91 | recreate them. Because of this, the wrappers are designed to be **read-only** 92 | as well. 93 | 94 | ``` 95 | In [13]: entry.data.reading = 50000 96 | --------------------------------------------------------------------------- 97 | AttributeError Traceback (most recent call last) 98 | in () 99 | ----> 1 entry.data.reading = 50000 100 | 101 | /Users/shankari/e-mission/e-mission-server/emission/core/wrapper/wrapperbase.py in __setattr__(self, key, value) 102 | 95 return super(WrapperBase, self).__setattr__(key, value) 103 | 96 else: 104 | ---> 97 raise AttributeError("property %s is read-only" % key) 105 | 98 else: 106 | 99 raise AttributeError("property %s is not defined for %s" % (key, self.__class__.__name__)) 107 | 108 | AttributeError: property reading is read-only 109 | ``` 110 | 111 | In order to create a new entry, you can use createEntry with a data object. 112 | There are examples of creating entries all over the analysis pipeline, but 113 | here's an example similar to the one above. 114 | 115 | We first create the data object 116 | 117 | ``` 118 | In [14]: import emission.core.wrapper.statsevent 119 | 120 | In [15]: new_data = emission.core.wrapper.statsevent.Statsevent() 121 | 122 | In [16]: new_data.name = "modified" 123 | 124 | In [17]: new_data.reading = 5000 125 | 126 | In [19]: new_data.ts = 12345678 127 | 128 | In [20]: new_data.fmt_time = "this is the formatted_time" 129 | 130 | In [21]: new_data 131 | Out[21]: Statsevent({'name': 'modified', 'reading': 5000, 'ts': 12345678, 'fmt_time': 'this is the formatted_time'}) 132 | ``` 133 | 134 | We can't set the old data!! 135 | 136 | ``` 137 | In [22]: entry.data 138 | Out[22]: Statsevent({'reading': 0.41276121139526367, 'name': 'POST_/usercache/get', 'ts': 1481533136.761428}) 139 | 140 | In [23]: entry.data = new_data 141 | --------------------------------------------------------------------------- 142 | AttributeError Traceback (most recent call last) 143 | in () 144 | ----> 1 entry.data = new_data 145 | 146 | /Users/shankari/e-mission/e-mission-server/emission/core/wrapper/wrapperbase.py in __setattr__(self, key, value) 147 | 95 return super(WrapperBase, self).__setattr__(key, value) 148 | 96 else: 149 | ---> 97 raise AttributeError("property %s is read-only" % key) 150 | 98 else: 151 | 99 raise AttributeError("property %s is not defined for %s" % (key, self.__class__.__name__)) 152 | 153 | AttributeError: property data is read-only 154 | ``` 155 | 156 | We create a new entry using `create_entry` 157 | 158 | ``` 159 | In [24]: new_entry = ecwe.Entry.create_entry(entry.user_id, entry.metadata.key, new_data 160 | ...: ) 161 | 162 | In [25]: new_entry.data 163 | Out[25]: Statsevent({'name': 'modified', 'reading': 5000, 'ts': 12345678, 'fmt_time': 'this is the formatted_time'}) 164 | ``` 165 | 166 | -------------------------------------------------------------------------------- /docs/dev/archi/mode_inference_pipeline_design.md: -------------------------------------------------------------------------------- 1 | # Mode Inference Pipeline Design 2 | --- 3 | 4 | ## Design goals ## 5 | 1. Separate model building and model application steps 6 | 1. Seed model building from old-style (Moves) data 7 | 1. Support multiple models in parallel 8 | 9 | ## Serializing the model ## 10 | Both 1. and 2. imply that we should save the model and re-load it. 11 | We can use either standard `pickle` or `jsonpickle` to convert the model to json (with the caveat that it is custom to a particular version of scikit-learn) 12 | 13 | But how should we store these models? 14 | 1. Store to disk 15 | 2. Store to database 16 | 17 | Once we move to user-specific models, we can store them to the database. In fact, if we generate models periodically, we might want to store the history of them in the database. But right now, since we don't allow users to edit/confirm modes, we will start with the seed model built from old-style (Moves) data in the Backup and Section databases. Since we don't generate any more Moves-style data, this model will never change. 18 | 19 | So should it be stored on disk? 20 | 21 | Another thing to note is that we may want to continue to use the old-style (Moves) data until we build up a critical mass of new data. I just checked, and it is `14104 + 7439 = 21543` entries, which is pretty good. It looks like we can combine random forest models 22 | https://stackoverflow.com/questions/28489667/combining-random-forest-models-in-scikit-learn 23 | Not sure about other models from scikit-learn. 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /docs/dev/archi/module_structure.md: -------------------------------------------------------------------------------- 1 | # Module Structure 2 | --- 3 | 4 | The e-mission server project largely follows the standard python project structure. 5 | However, it is a complex project with a lot of modules, so we present a high level overview of the modules, their structure and their purpose below. This representation might be more useful to visual thinkers. 6 | 7 | ![A visual representation of the structure below](https://github.com/shankari/e-mission-server/blob/mega_restructure/figs/e-mission-server-module-structure.png) 8 | 9 | - `runAllTests`: UNIX and windows scripts to run all the tests 10 | - `e-mission-*py`: UNIX and windows wrappers that invoke python and ipython with the appropriate python paths 11 | - `conf`: Contains sample files that should be renamed 12 | - `config.json` 13 | - `net` 14 | - `auth`: Keys for authentication services. 15 | - `google.json`: Google is the only one that we have integrated with so far 16 | - `push_notify`: Keys for sending push notifications 17 | - `parse.json`: We have currently integrated with parse for iOS push notifications. We can also try other providers, or roll our own builtin solution. 18 | - `usercache`: Need to create if/when we integrate with other service providers 19 | - `ext_service`: API keys for external services 20 | - `moves` 21 | - `facebook`: Once Sid integrates with it 22 | - `analysis`: Various constants for the analysis 23 | - `classification`: Constants for the trip and section segmentation 24 | - `bin`: utility python scripts that call functions defined in the main module. These can be simple bash-like scripts that just contain a sequence of calls to the real functions. These are likely to be somewhat fragile, because they are only tested when run. But since the underlying libraries are tested regularly, fixing them should be as easy as matching the current method signature of the library. 25 | - `emission`: main python library 26 | - `client`: Ability to provide client or study specific defaults in addition to user-specific ones. *This currently in draft form and needs extensive thought and restructuring.* 27 | - `net`: communication with the outside world 28 | - `api`: the webserver and our API interface to the outside world 29 | - `bottle.py`: lightweight webserver 30 | - `webapp.py`: file which defines the API interface 31 | - `auth`: provider of authentication services. We will use only JWT based authentication, but there are many providers. Or we can integrate with something like https://oauth.io/home 32 | - `abstract.py`: abstract interface to the auth provider 33 | - `none.py`: simple implementation that can be used for testing without registering for any keys 34 | - `google.py`: google's provider 35 | - potentially others if people want to add them 36 | - `usercache`: bidirectional sync mechanism 37 | - `abstract.py`: abstract interface to the user cache 38 | - `builtin.py`: builtin implementation using mongodb collections 39 | - potentially others if people want to add them 40 | - `ext_service`: calls to read data from external services 41 | - `moves`: integration with the moves lifelogger 42 | - `sdk.py`: SDK for integrating with the moves API 43 | - `collect.py`: our script that uses the SDK to pull data from moves 44 | - `register.py`: our script that handles the OAuth workflow with moves 45 | - `facebook`: similar integration, once Sid gets to it 46 | - `facebook_sdk.py` 47 | - `collect.py` 48 | - `register.py` 49 | - `push_notify`: push notifications 50 | - `ios_parse.py`: currently using parse 51 | - `android.py`: if we want to use it for something specific... 52 | - `storage`: Handling long-term storage 53 | - `timeseries`: Storage of timeseries events 54 | - `abstract.py`: abstract interface to the timeseries 55 | - `builtin.py`: builtin implementation using MongoDB collections 56 | - potentially others if we need performance improvements 57 | - `decorations`: Meaningful decorations on the timeseries (see https://github.com/e-mission/e-mission-data-collection/wiki/Long-term-data-format-design-considerations). Do we need these in addition to wrapper classes? Think! What about different types of trips? 58 | - `locations.py` 59 | - `trips.py`: utility queries to traverse the trip hierarchy? 60 | - `sections.py` 61 | - `core` 62 | - `wrapper`: Simple wrappers that can load and store the documents in the database, with appropriate conversions 63 | - `simulation`: generation of fake data based on various models 64 | - `trip_generation`: generate trips based on tour models. 65 | - other fake data streams??? 66 | - `analysis`: The intelligence on top of the plumbing 67 | - `plotting`: Libraries for plotting various things (locations, trips, sections, etc) using the framework du jour 68 | - `pygmaps`: Plotting using google maps 69 | - `leaflet`: Plotting using OSM + Leaflet 70 | - `classification`: Various techniques for taking the streams of sensor data and "classifying" them. For our main use case, this involves segmenting them into trips and sections. For other streams, this could involve segmenting into areas of high and low fuel consumption, or good and bad pavement, or ??? 71 | - `cleaning`: 72 | - `segmentation`: 73 | - `inference`: infer the mode of the segments based on various data 74 | - `modelling`: analysis of classified data to determine models 75 | - `tour_model`: model a user's tours 76 | - `user_model`: model a user's preferences 77 | - `anomaly_detection`: find anomalies that don't match the model? 78 | - `results`: Condense the classifications and models into user facing summaries. 79 | - `precompute`: script to precompute all results ahead of time 80 | - `carbon.py`: carbon footprint for the user 81 | - `participation_score.py`: score focused mainly around engagement with the app, and confirming trips, with some components related to behavior change 82 | - `recommendation`: generate user recommendations based on a user's tour models and user models. This is a little more complex than the others, because we can use the feedback from the recommendations to modify the user model, so it is bidirectional 83 | - `ui`: the user interface 84 | - `server`: the server side webapp 85 | - `phone`: the screens for the phone. It is not clear how these should be sent to the phone (whether as part of the sync, or just by loading from github). We will figure that out when we finish that part of the sync code. For now, this is a place to put the existing screens that we have. 86 | - `carbon` 87 | - `result_template.html` 88 | - `js` 89 | - `jquery` and so on and so forth 90 | - `game` 91 | - `index.html` 92 | - `js`: ionic/angular javascript framework 93 | - app.js 94 | - controllers.js 95 | - `leaderboard` 96 | - `index.html` 97 | - `js` 98 | - `app.js` 99 | - `controllers.js` 100 | - `tour_model` 101 | - `index.html` 102 | - `js` 103 | - `leaflet.js` 104 | - `display.js` 105 | - `recommendations` 106 | - `index.html` 107 | - `js` 108 | - `leaflet.js` 109 | - `app.js` 110 | - `controllers.js` 111 | -------------------------------------------------------------------------------- /docs/dev/archi/pipeline_details.md: -------------------------------------------------------------------------------- 1 | ## Pipeline design and running ## 2 | 3 | As we have seen multiple times, the data flow involves non-immutable, read-only 4 | data directly from the user, either through automatic sensing on the phone, or 5 | through user inputs. This immutable data is processed by the pipeline to 6 | generate inferred results. The pipeline state controls the running of the pipeline. 7 | 8 | ### Execution model ### 9 | The pipeline for a single user is intended to be single-threaded, with the 10 | stages running sequentially. Pipelines for multiple users can run in parallel. 11 | 12 | In addition, every pipeline stage should: 13 | - only work on fresh data, for improved performance 14 | - work on all unprocessed data, for correctness 15 | 16 | We ensure this by maintaining state for each pipeline stage. The state includes 17 | - *the time at which the stage started running*: This is set when the stage 18 | starts running and unset when the stage completes. It is used to enforce 19 | single threading; if this value is set for a particular, then other instances 20 | of the stage should not run. 21 | - *the timestamp of the last data point processed* Note that this is the 22 | *not* the timestamp at which the stage was last run. Since the data can be 23 | buffered at various stages along the way, data that was collected before the 24 | current time might not have made it to the server yet. On every pass, the stage 25 | processes data in the range (last ts processed, now) 26 | 27 | ### Resetting the pipeline ### 28 | If your pipeline state is messed up in any way, you can just reset the pipeline. The raw data is still present, so you can re-run the pipeline again. Re-running the pipeline after a reset should generate the same results, although it may take a LOOOONG time depending on the data that you have. 29 | - if you are running the pipeline on a small amount of local data (e.g. to 30 | reproduce on your desktop), you can reset the entire pipeline (`$ ./e-mission-py.bash bin/reset_pipeline.py -all`) and re-run 31 | - if you ran the pipeline for a single user, you can reset the pipeline for that user (`$ ./e-mission-py.bash bin/reset_pipeline.py [-u|-e]`) 32 | - if you are running the pipeline on an ongoing production server, reset 33 | the pipeline to a specific date and re-run (`$ ./e-mission-py.sh bin/reset_pipeline.py -h` for the format). Note that this is a trickier 34 | operation, so you may run into corner cases. In that case, file an issue, or 35 | fix it youself and send me a pull request! 36 | 37 | Note that deleting the pipeline state entries without fully resetting the pipeline *will not work*. The processed results will not be deleted correctly, so you will end up with two sets of trips and sections, which will then mess up the downstream stages. 38 | 39 | ### Troubleshooting ### 40 | #### `curr_run_ts` = ...., while processing pipeline #### 41 | If you see the error `curr_state.curr_run_ts = [0-9]*, while processing pipeline`, it means that the stage is marked as still running. 42 | Steps to resolve: 43 | 1. Make sure that there isn't any other instance of the pipeline running (e.g. something like `$ ps -aef | grep intake`) 44 | 1. Assuming that is true, [reset the pipeline](#resetting_the_pipeline) 45 | You can reset the pipeline only back to the day before the error occurred (`-d` option). Also, before the pipeline to run the pipeline, you have to check that for the user(s) for which you had the error, the curr_ is not null in mongodb (Example: `db.getCollection('Stage_pipeline_state').find({"user_id": LUUID("3a2299d0-605a-4d92-bef8-1058d145d301")})`) or you will have to reset the `curr_run_ts` to `null` for that user and for the pipeline stage where it is not `null`. 46 | 47 | #### No errors, but data not processed #### 48 | Make sure that the pipeline state is such that your new data is actually considered fresh. Note that on every run, we only read data after the last timestamp processed. Each pipeline stage will print the time range that is considering, and how much fresh data matched it. 49 | 50 | For example, the following logs indicate that when this stage was run, it starts from the beginning `None` and finds 617 locations to process. At the end, we have finished processing until 2018-10-08T02:58:11.356361. 51 | 52 | ``` 53 | 2018-10-17 20:15:32,290:INFO:140735562040192:**********UUID 2b267782-a822-4b4b-954b-7f5b216c7eef: segmenting into trips********** 54 | 2018-10-17 20:15:32,291:INFO:140735562040192:For stage PipelineStages.TRIP_SEGMENTATION, start_ts is None 55 | 2018-10-17 20:15:32,296:DEBUG:140735562040192:curr_query = {'user_id': UUID('2b267782-a822-4b4b-954b-7f5b216c7eef'), '$or': [{'metadata.key': 'background/filtered_location'}], 'metadata.write_ts': {'$lte': 1539832527.2915819}}, sort_key = metadata.write_ts 56 | 2018-10-17 20:15:32,296:DEBUG:140735562040192:orig_ts_db_keys = ['background/filtered_location'], analysis_ts_db_keys = [] 57 | 2018-10-17 20:15:32,583:DEBUG:140735562040192:finished querying values for ['background/filtered_location'], count = 173 58 | 2018-10-17 20:15:32,671:DEBUG:140735562040192:finished querying values for [], count = 0 59 | 2018-10-17 20:15:32,677:DEBUG:140735562040192:orig_ts_db_matches = 173, analysis_ts_db_matches = 0 60 | 2018-10-17 20:15:32,718:DEBUG:140735562040192:Found 173 results 61 | 2018-10-17 20:15:32,726:DEBUG:140735562040192:After de-duping, converted 173 points to 173 62 | 2018-10-17 20:15:33,519:INFO:140735562040192:For stage PipelineStages.TRIP_SEGMENTATION, last_ts_processed = 2018-10-08T02:58:11.356361 63 | ``` 64 | 65 | If we re-run the pipeline for the same user and have not received any new data in the interim, we will see that we query from the last processed ts (`2018-10-08T02:58:11.356361`), which returns only 3 points. So there really isn't any new data to process. 66 | 67 | ``` 68 | 2018-10-26 15:04:41,426:INFO:140736223740800:**********UUID 2b267782-a822-4b4b-954b-7f5b216c7eef: segmenting into trips********** 69 | 2018-10-26 15:04:41,427:INFO:140736223740800:For stage PipelineStages.TRIP_SEGMENTATION, start_ts = 2018-10-08T02:58:11.356361 70 | 2018-10-26 15:04:41,432:DEBUG:140736223740800:curr_query = {'invalid': {'$exists': False}, 'user_id': UUID('2b267782-a822-4b4b-954b-7f5b216c7eef'), '$or': [{'metadata.key': 'background/filtered_location'}], 'metadata.write_ts': {'$lte': 1540591476.4273138, '$gte': 1538967491.356361}}, sort_key = metadata.write_ts 71 | 2018-10-26 15:04:41,432:DEBUG:140736223740800:orig_ts_db_keys = ['background/filtered_location'], analysis_ts_db_keys = [] 72 | 2018-10-26 15:04:41,432:DEBUG:140736223740800:finished querying values for ['background/filtered_location'] 73 | 2018-10-26 15:04:41,432:DEBUG:140736223740800:finished querying values for [] 74 | 2018-10-26 15:04:41,462:DEBUG:140736223740800:Found 3 results 75 | 2018-10-26 15:04:41,470:DEBUG:140736223740800:After de-duping, converted 3 points to 3 76 | ``` 77 | If this happens, you want to force the pipeline to treat the old data as fresh. Which means that you basically need to [reset the pipeline](#resetting-the-pipeline). 78 | 79 | -------------------------------------------------------------------------------- /docs/dev/back/adding_a_new_data_type.md: -------------------------------------------------------------------------------- 1 | # Adding a New Data Type 2 | --- 3 | 4 | More advanced users may want to collect new types of data. The app currently stores data in a local sqlite database and pushes it to the server at the end of every trip, and potentially every hour (if there is unpushed data). 5 | 6 | This document outlines the steps to save a new datastructure to the phone buffer and have it show up in the timeseries object of the server. 7 | 8 | #### Planning #### 9 | 10 | 1. You need to decide what kind of data it is. Is it sensor data (e.g. 11 | location), or a message (e.g. state machine transitions) or as local-only 12 | storage (e.g. notification configuration)? 13 | 14 | 1. You need to decide the data format. 15 | 16 | On the phone, the data format can be: 17 | - a native class, which allows data validation and easier access of the elements. For example, SimpleLocation, with android (https://github.com/e-mission/e-mission-data-collection/blob/master/src/android/wrapper/SimpleLocation.java) and iOS (https://github.com/e-mission/e-mission-data-collection/blob/master/src/ios/Wrapper/SimpleLocation.m) versions, OR 18 | - pure JSON, which requires no new classes, but is harder to use because the fields have to be accessed as strings 19 | 20 | On the server, the data is in JSON (e.g. 21 | https://github.com/e-mission/e-mission-server/blob/master/emission/core/wrapper/location.py). In addition to creating such an object, you need to 22 | - include a reference to it in `entry.py` 23 | - in `builtin_timeseries.py` 24 | - add formatters that can expand fields if necessary (e.g. https://github.com/e-mission/e-mission-server/blob/master/emission/net/usercache/formatters/android/location.py) 25 | 26 | An example of a single PR that includes all the changes required to add an 27 | object on the server is https://github.com/e-mission/e-mission-server/pull/517 28 | 29 | #### Client changes #### 30 | 31 | 1. You need to store the object. 32 | - If you are storing the object as a native class, use `putSensorData`, `putMessage`, or `putLocalStorage` depending on your object type (android: https://github.com/e-mission/e-mission-data-collection/blob/master/src/android/location/LocationChangeIntentService.java#L89 or ios: ) 33 | 34 | android: 35 | 36 | SimpleLocation simpleLoc = new SimpleLocation(loc); 37 | uc.putSensorData(R.string.key_usercache_location, simpleLoc); 38 | 39 | iOS: 40 | 41 | Transition* transitionWrapper = [Transition new]; 42 | transitionWrapper.currState = [TripDiaryStateMachine getStateName:self.currState]; 43 | transitionWrapper.transition = transition; 44 | transitionWrapper.ts = [BuiltinUserCache getCurrentTimeSecs]; 45 | [[BuiltinUserCache database] putMessage:@"key.usercache.transition" value:transitionWrapper]; 46 | 47 | The key in both these cases is mapped to the relevant string in the code (e.g. `R.string.key_usercache_location` is mapped to `background/location`). This mapping is in a resource XML for android and a plist for iOS. You need to add your own resource files to your plugin, similar to the ones in https://github.com/e-mission/cordova-usercache/tree/master/res, and then add entries to plugin.xml that copy them to the resource locations, similar to (https://github.com/e-mission/cordova-usercache/blob/master/plugin.xml) 48 | 49 | 50 | ... 51 | 52 | 53 | - If you are storing the object as raw JSON, use similar methods, but pass in a JSONObject on android, an NSDictionary on iOS and a JSON object in Javascript 54 | 55 | ``` 56 | JSONObject configWrapper = UserCacheFactory.getUserCache(ctxt).getLocalStorage(eventName, false); 57 | ... 58 | if (modified) { 59 | UserCacheFactory.getUserCache(ctxt).putLocalStorage(eventName, configWrapper); 60 | } 61 | ``` 62 | 63 | ``` 64 | $window.cordova.plugins.BEMUserCache.putMessage(MODE_CONFIRM_KEY, $scope.draftMode).then(function () { 65 | ..... 66 | } 67 | ``` 68 | 69 | 1. You might want to read the object to perform local computation during the 70 | trip, before it is pushed to the server. If so, for sensor data and messages, 71 | you can get the first n entries, the last n entries, and all entries in a 72 | particular time range - e.g. `getFirstMessages`, `getLastMessages` and 73 | `getMessagesForInterval`. Caveats: 74 | - If you are using native classes, you need to pass in the class on iOS. 75 | - Local storage is supposed to be kv pairs, so it only supports get. It is 76 | also the only type that supports remove. 77 | 78 | Examples: 79 | android 80 | 81 | SimpleLocation[] last10Points = uc.getLastSensorData(R.string.key_usercache_filtered_location, pointsToQuery , SimpleLocation.class); 82 | 83 | iOS 84 | 85 | NSArray* last10Points = [[BuiltinUserCache database] 86 | getLastSensorData:@"key.usercache.location" nEntries:10 87 | wrapperClass:[SimpleLocation class]]; 88 | 89 | 90 | iOS, local storage 91 | 92 | NSDictionary* notifyConfigWrapper = [[BuiltinUserCache database] getLocalStorage:eventName 93 | 94 | Javascript, messages 95 | 96 | return UnifiedDataLoader.getUnifiedMessagesForInterval("statemachine/transition", tq) 97 | 98 | To check whether the data was saved correctly to the phone database, you can 99 | email the database to yourself (Profile -> Check sensed data -> Email). The database 100 | will be sent as an attachment. It is an sqlite3 database and can be opened using 101 | 102 | $ sqlite3 103 | 104 | 105 | #### Push the data to the server #### 106 | 107 | To test this, you need to be running a server (https://github.com/e-mission/e-mission-server/README.md). The easiest option is to run the server on your laptop. Ensure that your phone is connected to your server - if needed, by configuring `www/json/connectionConfig.json` accordingly (https://github.com/e-mission/e-mission-phone#end-to-end-testing) 108 | 109 | The data is normally pushed automatically to the server, but you may want to force it during testing. In order to ensure that the data for the trip can be retrieved during the trip for local processing, only data upto the last `TRIP_ENDED` transition is pushed to the server. This means that you need to start and end a trip every time you push data. 110 | 111 | You can start and end trips by directly manipulating the state machine in the developer zone. `EXIT_GEOFENCE` followed by `STOPPED_MOVING` on android or `TRIP_ENDED` will generate a trip. Force syncing after that should push the data to the server. Check out the troubleshooting section to confirm if this worked correctly (https://github.com/e-mission/e-mission-server/wiki/Troubleshooting-tips-(FAQ)#is-the-data-getting-to-the-server-correctly). 112 | 113 | #### Access the data on the server #### 114 | 115 | When the data gets to the server, it is in the usercache. When the regular pipeline runs (https://github.com/e-mission/e-mission-server/wiki/Deploying-your-own-server-to-production#the-analysis-pipeline), the formatters are run on it, and it moves from the usercache to the timeseries. 116 | 117 | A unified view of the both the usercache and the timeseries is provided by the `cache_series` datastructure and can be used to confirm that the objects were saved correctly. 118 | (https://github.com/e-mission/e-mission-server/wiki/Troubleshooting-tips-(FAQ)#confirm-that-the-entries-are-really-in-the-database) 119 | 120 | If they were originally visible, but are lost when the pipeline runs, there is an error with the formatter (https://github.com/e-mission/e-mission-server/wiki/Troubleshooting-tips-(FAQ)#my-data-disappears-when-i-run-the-intake-pipeline). If they are visible even after the formatter runs, you can use other timeseries operations to manipulate the data. 121 | -------------------------------------------------------------------------------- /docs/dev/back/algo_changes_outcome.md: -------------------------------------------------------------------------------- 1 | ## Assessing the impact of algorithm changes ## 2 | 3 | One of the big challenges with making changes to the algorithm is that the unit 4 | tests currently ensure that the algorithm generates identical results. This is 5 | useful from a code perspective (see 6 | https://github.com/e-mission/e-mission-server/pull/761), but it is challenging 7 | if we want to improve the algorithm because now all the unit tests fail. 8 | 9 | We need to manually go through the results of the unit tests and verify that 10 | they are correct before we can merge the related changes to master. 11 | 12 | This has been hard to do, which is why the `gis-based-mode-detection` branch 13 | has been languishing without being merged to master for two years. 14 | 15 | However, I finally have a mechanism to run the two different versions of the 16 | algorithms, visualize the differences, and check in modified ground truth. This 17 | is not the perfect solution for comparing the analysis results, but it is a 18 | reasonable start. 19 | 20 | ### High level design ### 21 | 22 | I've figured out how to run test cases outside the test framework and pass in additional flags. 23 | We modify the test cases to accept the branch name, and `evaluation` and `persistence` flags: 24 | - If the "evaluation" flag is set: 25 | - the test registers the newly created user at the beginning of the test with an `email` = `_` 26 | - the test does not clear the entries for the user when the test is complete 27 | - if the "persistence" flag is set: 28 | - the test saves the downloaded ground truth into `/tmp/_.json` 29 | 30 | ### Testing procedure ### 31 | 32 | After these modifications, we can use the `bin/compare_algo_changes.py` script to compare the results: 33 | - Run the test cases on the old branch by passing in the branch name, and an "evaluation" flag 34 | - Run the test cases on the new branch by passing in the branch name, and "evaluation" and "persistence" flags 35 | - For each test, connect to the server with two different phones, one to `_`, and the other to `_` 36 | - Ensure that the `_` results are equal to or better than `_` 37 | - If they are, replace the ground truth file in the repo with the one from `/tmp/_.json` 38 | - If they are not, try to tweak until they are, then replace 39 | - If it is not possible to tweak further, but the tradeoff is still towards the new branch being better, replace 40 | - Once all ground truth files have been replaced, ensure that tests pass, and then push to master 41 | 42 | Related PR: https://github.com/e-mission/e-mission-server/pull/762 43 | -------------------------------------------------------------------------------- /docs/dev/back/config_environment_variables.md: -------------------------------------------------------------------------------- 1 | # Configuring Environment Variables 2 | 3 | The search results below identify some environment variables that need to be manually configured. 4 | 5 | #### DB, Webserver, Push Config 6 | - Use this [search](https://github.com/search?q=repo%3Ae-mission%2Fe-mission-server+%2Fconfig.get%5C%28%5C%22%5BA-Z%5D%2F&type=code) to obtain usage of `DB, WEBSERVER, PUSH` environment variables in the [e-mission-server](https://github.com/e-mission/e-mission-server) repository. 7 | 8 | #### AWS Cognito Credentials 9 | 10 | - Use this [search](https://github.com/search?q=repo%3Ae-mission%2Fop-admin-dashboard+%2F.getenv%5C%28%5C%22%5BA-Z%5D%2F&type=code) to obtain usage of `COGNITO` variables in the [admin-dashboard](https://github.com/e-mission/op-admin-dashboard) repository. 11 | 12 | #### Overpass, Nominatim, Geofabrik 13 | 14 | - Use this [search](https://github.com/search?q=repo%3Ae-mission%2Fe-mission-server+%2F.environ.get%5C%28%5C%22%5BA-Z%5D%2F&type=code) to obtain usage of `Overpass, Nominatim, Geofabrik` environment variables in the [e-mission-server](https://github.com/e-mission/e-mission-server) repository. 15 | - Use this [search](https://github.com/search?q=repo%3Ae-mission%2Fe-mission-server+geofabrik&type=code) to view some variables that are set using GitHub secrets in the [e-mission-server](https://github.com/e-mission/e-mission-server) GitHub workflows. -------------------------------------------------------------------------------- /docs/dev/back/supporting_user_inputs.md: -------------------------------------------------------------------------------- 1 | # Supporting user inputs # 2 | 3 | One of the characteristics of a mobilityscope is that it should support user 4 | inputs in addition to sensed data and the associated inferences. The data that 5 | we have seen for this typically falls into three categories. 6 | 7 | 1. **Data not directly related to sensed values**: e.g. incident reports 8 | 1. **Data providing additional information about sensed values**: e.g. information on how a particular trip was planned 9 | 1. **Data providing ground truth for analysis performed from sensed data**: e.g. confirmation of mode or purpose 10 | 11 | ## High level design ## 12 | 13 | For (3) in particular, it is interesting to think through how to store the values. The naive approach would be to store the results into the analysed values - for example, a trip whose mode is _corrected_ by a user can have its mode field set. But this conflicts with the need to allow the analysis results to be re-generated for reproducibility. If the mode for a trip is corrected, and then the trip is deleted, then the mode information is also lost. More details on the debate around the decision are in [#516](https://github.com/e-mission/e-mission-server/issues/516). 14 | 15 | Therefore, we decided that all user inputs are immutable. They will be stored as separate objects and can be combined with analysis results as needed. 16 | 17 | ## External Surveys v/s Custom Objects ## 18 | 19 | Given that user inputs are to be stored separately from analysis results anyway, there are two options for collecting them. 20 | 21 | ### External surveys ### 22 | e-mission supports hooking up to surveys created using web survey platforms such as qualtrics, survey monkey or google forms. In order to link the results to the sensed data, selected fields in the survey can be populated with the user and trip information. If only the user UUID is populated, no PII is sent to the web survey platform. If both user and trip fields are populated, trip start and end times are sent. These do not appear to be highly privacy sensitive in the absence of location information, but you should probably disclose it in your consent document. 23 | 24 | For more details on how to hook up external surveys, either in response to a button press, or in response to a notification, see [the more detailed instructions](../front/how_to_embed_an_external_survey_in_the_app.md) 25 | 26 | - **Pro:** _Ease of use_. External surveys are easy to create since they have nice survey building tools 27 | - **Con:** _Less flexibility_. Unless the survey platform provides an API to query for results, the communication is one-way. The survey results typically cannot be reflected back on the phone UI. 28 | 29 | ### Custom objects ### 30 | e-mission supports storing user results directly on the server. They are stored as `manul/*` objects. Existing examples are `manual/mode_confirm` and `manual/purpose_confirm`. Since the objects are separate from any trips, you need to add information to the objects that allow them to be matched with any analysis results. You can look at the `mode_confirm` and `purpose_confirm` addition [issue](https://github.com/e-mission/e-mission-server/issues/532) and [pull request](https://github.com/e-mission/e-mission-server/pull/563) for more details. As you can see, for a given trip, the matching user input objects can be retrieved using `get_user_input_for_trip_object` in `emission/storage/decorations/trip_queries.py`. 31 | 32 | 33 | If you need to add a new object type that is more complex than mode and purpose, you need to do book-keeping to register the object for use. The changes are pretty small in isolation, but there are quite a few of them. [This guide](adding_a_new_data_type.md) covers the details of adding new object types on both the phone and the server; you probably want to just add the objects as JSON. 34 | 35 | - **Pro:** _Flexibility:_ Have the survey depend on the sensed value in complex forms. Show the completed trip, skip certain questions based on motion activity results/time of day, etc. Stored values can directly be used in analysis results and reflected back to the user. 36 | - **Con:** _Complex to use:_ You have to support the objects on both the server and the phone and change the UI to read the values 37 | -------------------------------------------------------------------------------- /docs/dev/front/accessing_public_heatmap_and_incident_information.md: -------------------------------------------------------------------------------- 1 | # Accessing Public Heatmap and Incident Information 2 | --- 3 | 4 | There are three main public API calls exposed by the e-mission API server. All 5 | of them share the following characteristics: 6 | - they are `HTTP POST` calls 7 | - they do not currently require an API key for access. If we get popular enough 8 | to be the target for DDOS attacks, we will revisit this decision. 9 | - they take a time range as input. The time range can be specified either in 10 | UTC, as a range, or in local time, as a filter. 11 | 12 | These APIs are all currently very slow, because the prototype system is running 13 | on a single AWS host. The performance will improve after Nov 1st, when I split the 14 | functionality among multiple hosts. 15 | 16 | The three public API calls are: 17 | - `/result/heatmap/pop.route/` 18 | - `/result/heatmap/incidents/` 19 | - `/result/metrics/` 20 | 21 | The `` is either `timestamp` or `local_date`. The actual query range 22 | is specified in POST fields. 23 | 24 | Without any further ado, here are some examples of how to use these 25 | calls. I've chosen python as the language for demonstrating them since most 26 | data scientists are familiar with it, but any language that can send http post 27 | requests will work as well. 28 | 29 | ### Travel point queries ### 30 | 31 | #### Retrieving data from 2nd Oct to 3rd Oct across all modes and the entire world #### 32 | 33 | ``` 34 | $ ipython 35 | Python 2.7.11 |Anaconda 2.0.1 (x86_64)| (default, Dec 6 2015, 18:57:58) 36 | Type "copyright", "credits" or "license" for more information. 37 | 38 | IPython 4.2.0 -- An enhanced Interactive Python. 39 | ? -> Introduction and overview of IPython's features. 40 | %quickref -> Quick reference. 41 | help -> Python's own help system. 42 | object? -> Details about 'object', use 'object??' for extra details. 43 | 44 | In [1]: import arrow 45 | 46 | In [2]: start_ts = arrow.get("2017-10-02").timestamp 47 | 48 | In [3]: end_ts = arrow.get("2017-10-03").timestamp 49 | 50 | In [4]: post_fields = { 51 | "modes": None, 52 | "start_time": start_ts, 53 | "end_time": end_ts, 54 | "sel_region": None 55 | } 56 | 57 | In [5]: import requests 58 | 59 | In [6]: result = requests.post("https://e-mission.eecs.berkeley.edu/result/heatmap/pop.route/timestamp", json=post_fields) 60 | 61 | In [7]: result.status_code 62 | Out[7]: 200 63 | 64 | In [8]: result.json()['lnglat'][0:5] 65 | Out[8]: 66 | [[-117.07978918462588, 32.769079182443484], 67 | [-122.26395704827273, 37.87331020231461], 68 | [-110.77774474290837, 35.15781189961038], 69 | [-110.77774474290837, 35.15781189961038], 70 | [-110.77774474290837, 35.15781189961038]] 71 | 72 | In [9]: len(result.json()['lnglat']) 73 | Out[9]: 16607 74 | ``` 75 | 76 | #### Modifying the prior query to only retrieve walk and bike data #### 77 | 78 | ``` 79 | In [10]: post_fields["modes"] = ["WALKING", "ON_FOOT", "BICYCLING"] 80 | 81 | In [11]: result = requests.post("https://e-mission.eecs.berkeley.edu/result/heatmap/pop.route/timestamp", json=post_fields) 82 | 83 | In [12]: result.status_code 84 | Out[12]: 200 85 | 86 | In [13]: len(result.json()['lnglat']) 87 | Out[13]: 3312 88 | ``` 89 | 90 | #### Modifying the prior query to only retrieve data around Hearst Street in Berkeley, CA #### 91 | 92 | ``` 93 | In [15]: result = requests.post("https://e-mission.eecs.berkeley.edu/result/heatmap/pop.route/timestamp", json=post_fields) 94 | 95 | In [16]: len(result.json()['lnglat']) 96 | Out[16]: 112 97 | ``` 98 | 99 | ### Incident queries ### 100 | 101 | Incident queries are very similar to heatmap queries, only sent to a different URL. Some other differences are: 102 | - they don't support modes 103 | - since they are user-reported and not collected in the background, there are 104 | typically few of them. 105 | 106 | 107 | #### Retrieving all incidents from June to Oct #### 108 | 109 | ``` 110 | $ ipython 111 | Python 2.7.11 |Anaconda 2.0.1 (x86_64)| (default, Dec 6 2015, 18:57:58) 112 | Type "copyright", "credits" or "license" for more information. 113 | 114 | IPython 4.2.0 -- An enhanced Interactive Python. 115 | ? -> Introduction and overview of IPython's features. 116 | %quickref -> Quick reference. 117 | help -> Python's own help system. 118 | object? -> Details about 'object', use 'object??' for extra details. 119 | 120 | In [1]: import arrow 121 | 122 | In [2]: start_ts = arrow.get("2017-06-01").timestamp 123 | 124 | In [3]: end_ts = arrow.get("2017-10-01").timestamp 125 | 126 | In [4]: post_fields = { 127 | "start_time": start_ts, 128 | "end_time": end_ts, 129 | "sel_region": None 130 | } 131 | 132 | In [5]: import requests 133 | 134 | In [7]: result = requests.post("https://e-mission.eecs.berkeley.edu/result/heatmap/incidents/timestamp", json=post_fields) 135 | 136 | In [8]: result.status_code 137 | Out[8]: 200 138 | 139 | In [9]: len(result.json()["incidents"]) 140 | Out[9]: 42 141 | 142 | In [10]: result.json()["incidents"][0:5] 143 | Out[10]: 144 | [{u'fmt_time': u'2017-06-09T10:51:41-07:00', 145 | u'loc': {u'coordinates': [-122.06568002700806, 37.36993549376299], 146 | u'type': u'Point'}, 147 | u'local_dt': {u'day': 9, 148 | u'hour': 10, 149 | u'minute': 51, 150 | u'month': 6, 151 | u'second': 41, 152 | u'timezone': u'America/Los_Angeles', 153 | u'weekday': 4, 154 | u'year': 2017}, 155 | u'stress': 0, 156 | u'ts': 1497030701}, 157 | {u'fmt_time': u'2017-06-09T10:04:41-07:00', 158 | u'loc': {u'coordinates': [-122.07375526777469, 37.38112434837758], 159 | u'type': u'Point'}, 160 | u'local_dt': {u'day': 9, 161 | u'hour': 10, 162 | u'minute': 4, 163 | u'month': 6, 164 | u'second': 41, 165 | u'timezone': u'America/Los_Angeles', 166 | u'weekday': 4, 167 | u'year': 2017}, 168 | u'stress': 0, 169 | u'ts': 1497027881}, 170 | .... 171 | ``` 172 | 173 | This query can be also modified to return incidents for a particular area. Note that incidents cannot be filtered by mode, so any mode field in the query is ignored. 174 | -------------------------------------------------------------------------------- /docs/dev/front/create_a_new_custom_client.md: -------------------------------------------------------------------------------- 1 | # Create a new custom client 2 | --- 3 | 4 | Steps to create a new custom client with a new app store entry. 5 | 6 | 1. copy over `config.xml` 7 | 1. change the package in the `id` 8 | 1. reset the `android-versionCode`, `ios-CFBundleVersion` and `version` 9 | 1. create a new appid in apple (https://developer.apple.com) with the new id specified in step 2 10 | 1. the current version of the push plugin requires firebase to be configured even if you are not using i 11 | - create a new firebase project for the new package (https://console.firebase.google.com) 12 | - set up an iOS app with the new appid from the previous step 13 | - go to Settings -> Cloud Messaging 14 | - upload your APNs Authentication Key (create one from the apple developer account under Keys -> APNs key if you don't have one) 15 | - get the `google-services.json` and `GoogleService-Info.plist` and put them into the project root 16 | - Setting up Firebase for both Android and iOS is necessary, even if you only need one platform to work. 17 | 1. *optional: iff you want to support UI channels* create a new entry for the app in the ionic console 18 | - copy the app id (next to the project name) into the `config.xml` under `cordova-plugin-ionic` 19 | 1. copy over `package.json` and make the same changes as `config.xml` 20 | 1. follow the full installation steps from the README (https://github.com/e-mission/e-mission-phone#updating-the-e-mission--plugins-or-adding-new-plugins). 21 | - Note: you should copy over your modified `config.xml` and `package.json` only after running 22 | ``` 23 | $ ./bin/configure_xml_and_json.js cordovabuild 24 | ``` 25 | - When Cordova properly processes your custom `config.xml`, it should output the correct package names when running `cordova prepare`: 26 | ``` 27 | Using cordova-fetch for cordova-ios@^4.5.4 28 | Adding ios project... 29 | Creating Cordova project for the iOS platform: 30 | Path: platforms/ios 31 | Package: 32 | Name: 33 | iOS project created with cordova-ios@4.5.5 34 | Discovered platform "android@^6.4.0" in config.xml or package.json. Adding it to the project 35 | Using cordova-fetch for cordova-android@^6.4.0 36 | Adding android project... 37 | Creating Cordova project for the Android platform: 38 | Path: platforms/android 39 | Package: 40 | Name: 41 | Activity: MainActivity 42 | Android target: android-26 43 | ``` 44 | 1. create or copy over relevant HTML + javascript into `www` 45 | 1. create an icon under `resources/icon.png` 46 | 1. copy over the hooks (`./hooks`) and any notification icons needed (`resources/android/*`) 47 | 1. If you want to use google auth, create google auth credentials correctly for the new package name and include them into `www/json/connectionConfig.json` 48 | 1. Before you push your modifications to your own fork on GitHub, make sure you add `google-services.json` and `GoogleService-Info.plist` to `.gitignore`. 49 | 1. After you have finished testing, follow the standard instructions to create a production release and upload to the app/play stores. For android, you can use the `bin/sign_and_align_keys.sh` as a template. Note that this script will not work for you without modification - it has hardcoded paths. If you don't know how to edit shell scripts to match your environment, feel free to use android studio instead. 50 | -------------------------------------------------------------------------------- /docs/dev/front/guidelines_to_support_i18n.md: -------------------------------------------------------------------------------- 1 | # Guidelines to support i18n 2 | --- 3 | 4 | The phone app now supports i18n. In order to do that, we put most of the strings seen by the user in external files translated in different languages. To ensure i18n for the application, you have to follow some guidelines. 5 | 6 | ## While working on native Android 7 | 8 | Android provides a native support to localize its applications by using `resources`. By default, Android is storing its strings in **xml** files in the `res/values/`directory. If you want to localize those strings, you can create another `values-` folder which will contains those same **xml** files but with your translations. 9 | 10 | ```xml 11 | 12 | 13 | value 14 | 15 | ``` 16 | 17 | When you are creating your plugin for **e-mission**, you will want to put your strings into an external **xml** file to ensure the i18n support and then place the file in `res/values`. You can then call those strings in your code by using: 18 | 19 | ```java 20 | context.getString(R.string.); 21 | ``` 22 | 23 | It is also important to add the resource file in the `plugin.xml`, you can do it by adding the following line into it which will copy the file in the right directory when adding the plugin. 24 | 25 | ```xml 26 | 27 | ``` 28 | You can find further information in the [Android documentation](https://developer.android.com/guide/topics/resources/localization). 29 | 30 | ## While working on native iOS 31 | 32 | iOS uses the same logic as Android but the strings are placed in `strings` files instead of `xml`. Moreover, those files are placed in an `en.lproj` directory and the translated files in a `.lproj` directory. 33 | 34 | If you want to call those strings in your code, you can use `NSLocalizedStringFromTable` in Objective-C. 35 | 36 | ```objc 37 | NSLocalizedStringFromTable(@"key", @"filename", nil) 38 | ``` 39 | 40 | Like Android, do not forget to add the following line in your `plugin.xml`. 41 | 42 | ```xml 43 | 44 | ``` 45 | 46 | You can find further information in the [iOS documentation](https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPInternational/LocalizingYourApp/LocalizingYourApp.html#//apple_ref/doc/uid/10000171i-CH5-SW1). 47 | 48 | ## While working on the Cordova phone app 49 | 50 | To support i18n in the phone app, we are using [angular-translate](https://angular-translate.github.io/). Each strings will be placed into the file `i18n/en.json`. You can define a *namespace* when you are working on a specific file to ensure a better readability. 51 | 52 | ```json 53 | "trip-confirm": { 54 | "recenttrip": "Recent trip from: {{startTime}} → to: {{endTime}}", 55 | "continue": "Continue", 56 | "done": "Done", 57 | "services-please-fill-in": "Please fill in the {{text}} not listed.", 58 | "services-cancel": "Cancel", 59 | "services-save": "Save" 60 | } 61 | ``` 62 | 63 | ### In HTML 64 | 65 | When you want to read a string in your HTML file, you can do it by adding `translate` to the HTML tag and by putting `{'key of the string'}` between the tags. 66 | 67 | ```html 68 |
{{'trip-confirm.recenttrip'}}
69 | ``` 70 | 71 | You can also define a namespace for the child of the tag by adding `translate-namespace=`. 72 | 73 | ```html 74 |
75 |
{{'.recenttrip'}}
76 |
77 | ``` 78 | 79 | It is also possible to use variables within translations by simply adding the variables into the translation and then interpolating them by adding `translate-values={variablename: variable}`. 80 | 81 | ```html 82 |
83 | ``` 84 | 85 | You can find more information about **Variable Replacement** in the [doc](https://angular-translate.github.io/docs/#/guide/06_variable-replacement). 86 | 87 | ### In JavaScript 88 | 89 | >**WARNING**: Do not forget to add `$translate` to your controller or service. 90 | 91 | If you want to retrieve the language used by the user, you can use: 92 | 93 | ```js 94 | $translate.use(); 95 | ``` 96 | 97 | To translate a string directly in the Javascript, you can use: 98 | 99 | ```js 100 | $translate.instant('key of the string'); 101 | ``` 102 | 103 | ### Working with date 104 | 105 | Because each language has different format for date and hour, we are using the JavaScript library [moment](https://momentjs.com/) to handle that. 106 | 107 | To display hours in the adequate format, you can use: 108 | 109 | ```js 110 | moment(dt).format("LT"); 111 | ``` 112 | 113 | To display date: 114 | 115 | ```js 116 | moment(day).format('LL'); 117 | ``` 118 | 119 | You can also use the following lines to retrieve weekdays and months: 120 | 121 | ```js 122 | moment.weekdaysMin() 123 | moment.monthsShort() 124 | ``` 125 | -------------------------------------------------------------------------------- /docs/dev/front/high_level_faq.md: -------------------------------------------------------------------------------- 1 | # High-level FAQ 2 | --- 3 | 4 | #### How do you deploy a custom UI on the base app? #### 5 | https://github.com/e-mission/e-mission-phone/wiki/You-have-a-branch-for-your-custom-client.-Now-what%3F 6 | 7 | #### I get a blank screen while loading the e-mission phone UI #### 8 | This is the White Screen Of Death (WSOD) and typically indicates that you have an error preventing the scripts from loading. 9 | - First make sure that you have followed all setup instructions. Some of them download Javascript dependencies into the project. If they are not run, the files will be missing, and you will get errors. If in doubt, re-run the setup instructions, they are idempotent. 10 | - Next, check the UI logs in the terminal where you run the UI development server. UI logs are redirected by default, so if the error occurred after logging was set up, you can see the error there. 11 | - Otherwise, connect to the app using a debugger. 12 | - For iOS, the debugger is built in to Safari (https://www.lifewire.com/how-to-enable-safari-develop-menu-2260894) 13 | - for android, the debugger is built in to chrome (http://geeklearning.io/apache-cordova-and-remote-debugging-on-android/). 14 | 15 | There is ample documentation on how to use these tools - e.g. search for "debug javascript error in cordova using safari" for tutorials, etc. You might also find [this video](https://people.eecs.berkeley.edu/~shankari/syntax_error_wsod.mov) useful. 16 | 17 | #### Can I connect the devapp to a real server with proper authentication for proper end-to-end testing? #### 18 | Yes. If you are using google auth, you [need to get keys from me and configure them](https://github.com/e-mission/e-mission-server/wiki/Configuring-authentication#more-detailsfaq). If you want to run the app on the phone for real end-to-end testing, [it is easier on android than on iOS](https://github.com/e-mission/e-mission-docs/blob/dev-app-auth-clarification/docs/overview/high_level_faq.md#how-to-get-the-app-onto-a-real-phone). 19 | 20 | #### How to get the app onto a real phone #### 21 | This really depends on whether you still plan to change the UI, or whether you are largely done with the UI changes and want to build a custom app for deployment, potentially with a different set of plugins. 22 | - *For changing the UI*: [Use the devapp on your phone](https://github.com/e-mission/e-mission-devapp/blob/master/README.md#installing-on-a-real-phone) 23 | - *For building a custom app*: [Build the phone app directly](https://github.com/e-mission/e-mission-phone/blob/master/README.md#updating-the-e-mission--plugins-or-adding-new-plugins) 24 | 25 | #### I get an error while adding plugins #### 26 | 27 | Sometimes, if you are on a poor internet connection, you will encounter errors related to adding plugins while building the native versions of the phone apps. Retrying will just cause a different set of plugins to fail, since the underlying issue is the internet quality. In this case, a workaround is to manually clone and add the repositories, which seems to work. 28 | 29 | ##### Plugin clone errors ##### 30 | 31 | When you are trying to build and run the native code, you sometimes get plugin clone errors 32 | 33 | ``` 34 | $ npm run phonegap -- run ios | 2>&1 | grep --context=3 "Failed to restore plugin" 35 | ... 36 | Failed to restore plugin "phonegap-plugin-contentsync" from config.xml. You might need to try adding it again. Error: Failed to fetch plugin git+https://github.com/e-mission/phonegap-plugin-contentsync.git via registry. 37 | ... 38 | Failed to restore plugin "de.appplant.cordova.plugin.local-notification-ios9-fix" from config.xml. You might need to try adding it again. Error: Failed to fetch plugin https://github.com/shankari/cordova-plugin-local-notifications.git via registry. 39 | ... 40 | Failed to restore plugin "cordova-plugin-inappbrowser" from config.xml. You might need to try adding it again. Error: Failed to fetch plugin git+https://github.com/shankari/cordova-plugin-inappbrowser.git via registry. 41 | ``` 42 | ##### Workaround ##### 43 | 44 | The workaround is to manually clone the repo(s), add the local copy(ies) and rebuild. So for the case above, you would do: 45 | 46 | ``` 47 | $ cd .. 48 | $ git clone https://github.com/e-mission/phonegap-plugin-contentsync.git 49 | $ git clone https://github.com/shankari/cordova-plugin-local-notifications.git 50 | $ git clone https://github.com/shankari/cordova-plugin-inappbrowser.git 51 | $ cd e-mission-devapp/ 52 | $ cordova plugin add ../phonegap-plugin-contentsync/ 53 | $ cordova plugin add ../cordova-plugin-local-notifications/ 54 | $ cordova plugin add ../cordova-plugin-inappbrowser/ 55 | $ cordova run ios 56 | ``` 57 | -------------------------------------------------------------------------------- /docs/dev/front/how_to_add_a_language.md: -------------------------------------------------------------------------------- 1 | # How to add a language 2 | --- 3 | 4 | For now, only English, French and Italian are available but in the future we hope to cover more languages. 5 | 6 | ## Update the phone app 7 | 8 | ### Adding a language 9 | 10 | For now, each language is declare staticly in `www/js/app.js`. 11 | 12 | ```js 13 | $translateProvider 14 | .fallbackLanguage('en') 15 | .registerAvailableLanguageKeys(['en', 'fr'], { 16 | 'en_*': 'en', 17 | 'fr_*': 'fr', 18 | '*': 'en' 19 | }) 20 | .determinePreferredLanguage() 21 | .useStaticFilesLoader({ 22 | prefix: 'i18n/', 23 | suffix: '.json' 24 | }); 25 | ``` 26 | 27 | To add a language to the app, we have to register a new language key. If I want to add deutsch, I will modify the file like below: 28 | 29 | ```js 30 | $translateProvider 31 | .fallbackLanguage('en') 32 | .registerAvailableLanguageKeys(['en', 'fr','de'], { 33 | 'en_*': 'en', 34 | 'fr_*': 'fr', 35 | 'de_*': 'de', 36 | '*': 'en' 37 | }) 38 | .determinePreferredLanguage() 39 | .useStaticFilesLoader({ 40 | prefix: 'i18n/', 41 | suffix: '.json' 42 | }); 43 | ``` 44 | 45 | ### Changing the fallback language 46 | 47 | By default, the fallback language is english but it can easily be modified by just changing `en` in the line below by the key of your language in `www/js/app.js`. 48 | 49 | ```js 50 | .fallbackLanguage('en') 51 | ``` 52 | 53 | You can find more information about the fallback language in the [documentation](https://angular-translate.github.io/docs/#/guide/08_fallback-languages). 54 | 55 | ### Translating the files 56 | 57 | All the text that needs to be translated are in https://github.com/e-mission/e-mission-translate. Please double check the key list against the corresponding `en` versions since the non-en translations are not guaranteed to be updated. The mapping from the translated files to the corresponding `en` files is in the `README`. 58 | 59 | ## Adding custom translations 60 | 61 | If you do not want to use the translation provided by `e-mission-translate` you can either fork the repo or just delete the line below from the `config.xml` to stop using the hook which is pulling the translation from the git repo. 62 | 63 | ```xml 64 | 65 | ``` 66 | 67 | If you chose to fork the repo, do not forget to add `bin/conf/translate_config.json` and add the link of the forked repo as below. 68 | 69 | ```json 70 | { 71 | "url": "https://github.com/e-mission/e-mission-translate" 72 | } 73 | ``` 74 | -------------------------------------------------------------------------------- /docs/dev/front/how_to_test_changes _to_a_plugin.md: -------------------------------------------------------------------------------- 1 | # How to test changes to a plugin locally 2 | 3 | ## Prerequisites 4 | You should have Git installed on you local machine. 5 | 6 | You will also need a local copy of the [e-mission-phone](https://github.com/e-mission/e-mission-phone) project. Follow these [instructions](https://github.com/e-mission/e-mission-phone/blob/master/README.md) to get a copy. 7 | 8 | ## 1. Fork the plugin repository 9 | Go to the GitHub URL of the plugin you would like to test for example; ` https://github.com/e-mission/cordova-usercache` and click on the **Fork** button. 10 | 11 | ![Fork a project](../../assets/test_changes_to_a_plugin/fork-project.png) 12 | 13 | **Fork** will create a copy of the repository in your Github account so you can make changes to the project. 14 | 15 | ## 2. Create a local copy on your computer 16 | Navigate to your own Github copy of the repository and copy the url. 17 | 18 | ![Fork a project](../../assets/test_changes_to_a_plugin/copy-project-url.png) 19 | 20 | Open your terminal or git bash window. 21 | 22 | Move to the location on your computer where you want to create a copy of the project. 23 | 24 | For example; 25 | ```bash 26 | $ cd ~/plugins 27 | ``` 28 | Run the following command: 29 | ```bash 30 | $ git clone https://github.com//.git 31 | ``` 32 | > Substitute your Github username for `` and the plugin repository name for ``. 33 | 34 | You now have a local copy on your computer. 35 | 36 | ## 3. Add the local copy of your plugin to the e-mission phone project 37 | First, open another terminal or git bash window and move to the location on your computer where you have your local `e-mission-phone` project. 38 | 39 | For example: 40 | ```bash 41 | $ cd ~/workshop/e-mission-phone 42 | ``` 43 | 44 | Next, remove the existing plugin 45 | ```bash 46 | $ cordova plugin remove edu.berkeley.eecs.emission. 47 | ``` 48 | 49 | Then, add your local plugin to the `e-mission-phone` 50 | 51 | ```bash 52 | $ cordova plugin add ~/plugins/ 53 | ``` 54 | > Substitute the name of the repository for ``. 55 | 56 | run `cordova prepare` to recompile the project. 57 | 58 | > Any time you make changes to the plugin, remove it and re-add it to the e-mission-phone project. 59 | 60 |

61 | 62 | # Quick and Easy development using the IDE 63 | While the above approach helps test small changes, It is very time-consuming when it comes to complex ones. A faster way is to use an IDE for the appropriate platform. 64 | 65 | ## 1. First, prepare and build your e-mission-phone project to generates all the required files. 66 | 67 | ```bash 68 | # Copy files into platforms for building 69 | $ cordova prepare 70 | 71 | # Build the platform specific code 72 | cordova build 73 | ``` 74 | > Replace the `` with either `ios` or `android`. 75 | 76 | ## 2. Then, open the project in your IDE. 77 | 78 | ### For Android development: 79 | 80 | Launch Android Studio and select the Android platform directory in your project `(/e-mission-phone/platforms/android)`. 81 | 82 | Answer yes for the Gradle Sync question 83 | 84 | Once it finishes importing, you should be able to build and run the app directly from Android Studio. See [Android Studio Overview](https://developer.android.com/studio/intro/index.html) and [Building and Running from Android Studio](https://developer.android.com/studio/run/index.html) for more details. 85 | 86 | ### For IOS development: 87 | Launch Xcode and select the IOS platform directory in your project `(/e-mission-phone/platforms/ios/emission.xcworkspace)`. 88 | 89 | > Note: The last path (`/emission.xcworkspace`) is your app name appended to `.xcworkspace`. In our case, the app name is `emission`. 90 | 91 | ## 3. Finally, test your plugin using your IDE. 92 | 93 | Make the necessary changes to the native code and test them in the IDE. 94 | 95 | Then copy the code changes from the IDE back into your local plugin. 96 | -------------------------------------------------------------------------------- /docs/dev/front/loading_data_for_emulator.md: -------------------------------------------------------------------------------- 1 | # How to Load Data into Local Server for Use with devapp 2 | 3 | In order to fully test the functionality of the devapp, you will usually need to create an opcode with data. You **can** login to the devapp with a "real" opcode (ie the one you are logged into on a physical phone), which is useful for debugging. For more general development purposes, however, it is often more useful to work with sample data, loaded into an opcode of your choice. There are a few useful routes provided by the server for accomplishing this. 4 | 5 | ## Load Test Data from Server 6 | 7 | You can find sample data in `e-mission-server/emission/tests/data/real_examples`. 8 | Then, you can load the data into an opcode with the following two scripts. Make sure you have activated the emission environment first. 9 | 10 | ```./e-mission-py.bash bin/debug/load_timeline_for_day_and_user.py emission/tests/data/real_examples/[file of your choice] nrelop_[complete valid opcode]``` 11 | 12 | ```./e-mission-py.bash bin/debug/intake_single_user.py -e nrelop_[same valid opcode]``` 13 | 14 | After these two commands run sucessfully, you should be able to run the server and login with the opcode you created in the devapp. 15 | 16 | ## Load Data from Local File 17 | 18 | Sometimes, you will have specific data that you want to load into the emulator. Once you have the two g-zipped files in a local directory, you can load the data into an opcode: 19 | 20 | `./e-mission-py bin/debug/load_multi_timeline_for_range.py [path/to/files/with/prefix]` -- this path might look something like `/Users/name/Downloads/my_test_data/test_data` where `my_test_data` is a folder containing the two g-zipped files that each start with `test_data` 21 | 22 | The script will automatically create a "user email" for this opcode, by default it will be "user-0". This can be updated with: `edb.get_uuid_db().update_one({"user_email": "user-0"}, {"$set": {"user_email": "nrelop_[valid opcode]"}})` 23 | 24 | Then, process the data by running the pipeline. 25 | 26 | `./e-mission-py.bash bin/debug/intake_single_user.py -e [user-0 or updated]` 27 | 28 | If you updated to a valid opcode, you can now start the server and log into the devapp with that opcode 29 | 30 | ## Note on Working with Deployment Configs 31 | 32 | There are development configs designed to be used with the local server. They include `dev-emulator-program`, `dev-emulator-study`, and `dev-emulator-timeuse`. 33 | 34 | Sometimes, it is useful to load data into a deployment config for testing, demo, or debugging purposes. If you want to use the local server and a deployment config, you need to take a few extra steps. 35 | 1. Fork `nrel-openpath-deploy-configs` 36 | 2. Modify the config you want to work on, making sure that there is no `server` defined (if no server is defined, the app will connect to the local server, as desired). 37 | 3. Update the downloadURL in `dynamicConfig` for the version of `e-mission-phone` you are running to `https://raw.githubusercontent.com/[MY-REPO]/nrel-openpath-deploy-configs/[MY-BRANCH]/configs/${label}.nrel-op.json` 38 | 4. Now you can use a deployment config with the local server 39 | 40 | ## Notes on making up a valid opcode 41 | - opcodes must begin with `nrelop` 42 | - opcodes must contain the name of a config (ie `dev-emulator-study` or `nrel-commute`) 43 | - some configs require a subgroup (ie `default` or `test`) 44 | - the endings to opcodes can be anything, it is most useful to choose something memorable (ie `myTestData` or `test2`) 45 | 46 | All together the structure should be `nrelop_configName_subgroup_ending` 47 | -------------------------------------------------------------------------------- /docs/dev/front/phone_module_structure.md: -------------------------------------------------------------------------------- 1 | # Phone module structure 2 | --- 3 | 4 | The phone code is based on ionic1 (https://ionicframework.com/docs/v1/), which in turn is based on angular1 (https://docs.angularjs.org/tutorial/). 5 | 6 | In order to work with this code, you should be familiar with ionic1 and angular1. There are a ton of resources for working with both ionic1 and angular1 on the internet (e.g. https://codepen.io/search/pens?q=ionic&limit=all&type=type-pens) and we will not reproduce them here. 7 | 8 | In addition, you can look at the branches, in particular, `cci-berkeley`, `opentoall` and `interscity` for code examples that you may want to incorporate. You can cherry-pick particular changes or ranges of changes, or just copy the changes manually. You should be familiar with basic git commands before you start this process. 9 | 10 | In general, the structure of the phone code is modular, with directories delineating tabs or common functionality. 11 | Both the templates and the javascript has individual directories. As of Nov 2017, these directories and rough mappings are: 12 | 13 | - `common`: common trips, tour model 14 | - `control`: all the stuff around controlling data collection = the profile screen. Note that some of these files are copied over from plugins 15 | - `diary`: the basic travel diary 16 | - `goals`: the game 17 | - `incident`: the end of trip prompt + particular location selection code 18 | - `intro`: onboarding screens 19 | - `metrics`: dashboard 20 | - `recent`: debugging code, shows up in the profile 21 | - `splash`: startup code 22 | 23 | Note that we use services quite a bit in the javascript, so not all the code will be in the controllers. 24 | 25 | Some tabs are special in the onboarding process - e.g. the metrics screen is the first one, the heatmap tab is where the user is redirected if she rejects consent. More details on how best to mess around with the tabs are in https://github.com/e-mission/e-mission-phone/wiki/Changes-needed-when-the-set-of-tabs-changes 26 | 27 | 28 | -------------------------------------------------------------------------------- /docs/dev/front/samples/test_external_surveys.patch: -------------------------------------------------------------------------------- 1 | diff --git a/www/js/intro.js b/www/js/intro.js 2 | index 5cc9f5b6..83da8e77 100644 3 | --- a/www/js/intro.js 4 | +++ b/www/js/intro.js 5 | @@ -1,6 +1,6 @@ 6 | 'use strict'; 7 | 8 | -angular.module('emission.intro', ['emission.splash.startprefs', 9 | +angular.module('emission.intro', ['emission.splash.startprefs', 'emission.survey.launch', 10 | 'ionic-toast']) 11 | 12 | .config(function($stateProvider) { 13 | @@ -19,7 +19,7 @@ angular.module('emission.intro', ['emission.splash.startprefs', 14 | }) 15 | 16 | .controller('IntroCtrl', function($scope, $state, $window, $ionicSlideBoxDelegate, 17 | - $ionicPopup, $ionicHistory, ionicToast, $timeout, CommHelper, StartPrefs, $translate) { 18 | + $ionicPopup, $ionicHistory, ionicToast, $timeout, CommHelper, StartPrefs, $translate, SurveyLaunch) { 19 | 20 | $scope.platform = $window.device.platform; 21 | $scope.osver = $window.device.version.split(".")[0]; 22 | @@ -161,7 +161,7 @@ angular.module('emission.intro', ['emission.splash.startprefs', 23 | // $scope.next(); 24 | ionicToast.show(userEmail, 'middle', false, 2500); 25 | CommHelper.registerUser(function(successResult) { 26 | - $scope.finish(); 27 | + $scope.next(); 28 | }, function(errorResult) { 29 | $scope.alertError('User registration error', errorResult); 30 | $scope.finish(); 31 | @@ -189,6 +189,16 @@ angular.module('emission.intro', ['emission.splash.startprefs', 32 | } 33 | }; 34 | 35 | + $scope.launchWithID = function () { 36 | + SurveyLaunch.startSurveyWithID("https://berkeley.qualtrics.com/SE/?SID=SV_9t6KnGlK1qxOGGN", "QR~QID3"); 37 | + } 38 | + 39 | + $scope.launchWithXPath = function () { 40 | + SurveyLaunch.startSurveyWithXPath( 41 | + "https://docs.google.com/forms/d/e/1FAIpQLSd_7VICYb8AfLQPIt8pg9AYsXW_rrHQBfsv-NSviO7Dgk7fyg/viewform", 42 | + "/html/body/div/div[2]/form/div[2]/div/div[2]/div[1]/div/div/div[2]/div/div[1]/div/div[1]/input"); 43 | + } 44 | + 45 | $scope.finish = function() { 46 | // this is not a promise, so we don't need to use .then 47 | StartPrefs.markIntroDone(); 48 | diff --git a/www/templates/intro/intro.html b/www/templates/intro/intro.html 49 | index f4fdc0e3..8295cf47 100644 50 | --- a/www/templates/intro/intro.html 51 | +++ b/www/templates/intro/intro.html 52 | @@ -12,5 +12,8 @@ 53 | 54 | 55 | 56 | + 57 | + 58 | + 59 | 60 | 61 | -------------------------------------------------------------------------------- /docs/dev/front/samples/test_survey.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |
4 |
5 | 8 | 11 |
12 |
13 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /docs/dev/front/test_translations_in_devapp.md: -------------------------------------------------------------------------------- 1 | # How to Specify Translation Source 2 | 3 | When adding a language or testing translation updates, it can be useful to see changes in the devapp. Follow the steps bellow to point your devapp to a specific branch for translations. 4 | 5 | In the GitHub CLI for your e-mission-phone repo, navigate into the locales folder: 6 | 7 | `cd locales` 8 | 9 | Clear the remote tracking, and set the appropriate upstream and origin repositories: 10 | 11 | `git remote rm origin` 12 | 13 | `git remote add upstream https://github.com/e-mission/e-mission-translate` 14 | 15 | `git remote add origin https://github.com/[your_account]/e-mission-translate` 16 | 17 | Check what has been set: 18 | 19 | `git remote -v` 20 | 21 | Checkout the branch you wish to test 22 | 23 | `git checkout --track origin/[branch_to_test]` 24 | 25 | Now, the translations you see in the emulator will be pulled from that branch, and your edits in locales can be pushed into your fork/branch of translations. 26 | -------------------------------------------------------------------------------- /docs/dev/front/trip_labelling_options.md: -------------------------------------------------------------------------------- 1 | ## Trip labelling options 2 | 3 | As of e-mission-phone v3.2.3, there are two survey options available for trip labelling: 4 | 5 | - [MULTILABEL](https://user-images.githubusercontent.com/2423263/170624159-ada650ce-1c3e-4fad-8104-9fcfad4fc322.mov): displays two buttons under each trip, one for travel mode and the other for purpose. Note that, only one response can be selected in each button. 6 | - [ENKETO](https://user-images.githubusercontent.com/2423263/170631795-5ca2f330-1626-4709-aa7a-0f63370d5f1f.mov): displays one button that launches an Enketo survey form which allows for greater customisation of question types and display logic. 7 | 8 | By default, the `MULTILABEL` option is used. To change to `ENKETO`, find all `SurveyOptions.MULTILABEL` under the folder `www/js/diary` of your e-mission-phone and replace them with `SurveyOptions.ENKETO`. 9 | 10 | To create your own Enketo survey form, see the instruction in https://github.com/e-mission/e-mission-phone/tree/master/survey-resources. -------------------------------------------------------------------------------- /docs/dev/front/you_have_a_branch_for_your_custom_client_now_what.md: -------------------------------------------------------------------------------- 1 | # You have a branch for your custom client. Now what? 2 | --- 3 | 4 | Next steps after I have created a branch (e.g. `custom-ui`) for your custom client. The instructions here are very basic and are intended to be pointers to what you can look up further. You should be familiar with git and the standard github workflow, including forks, branches, pull requests and reviews. 5 | 6 | - Fork the e-mission-phone repo (e.g. create `https://github.com/username/e-mission-phone.git`) 7 | - Clone your fork 8 | - Create a new branch (potentially tracking the `custom-ui`) - e.g. 9 | ``` 10 | $ git checkout -b custom-ui -follow --track origin/custom-ui 11 | OR 12 | $ git checkout -b add-new-feature 13 | ``` 14 | - Make the changes you want to your branch, (https://github.com/e-mission/e-mission-phone/wiki/Phone-module-structure) potentially using existing branches (e.g. `cci-berkeley`, `interscity` or `opentoall`) as templates: 15 | - cherry-picking (`git cherry-pick`) the commits or sets of commits you want to use (better), OR 16 | - downloading patches from individual pull requests (https://stackoverflow.com/questions/7827002/how-to-apply-a-git-patch-when-given-a-pull-number) OR 17 | - downloading, editing and applying patches from individual commits if the commit history is badly mangled 18 | ``` 19 | $ git show > /tmp/change.patch 20 | $ patch -i /tmp/change.patch 21 | ``` 22 | - Potentially set up your own server (https://github.com/e-mission/e-mission-server/wiki/Deploying-your-own-server-to-production) and change the `www/json/connectionConfig.json` to point to it (https://github.com/e-mission/e-mission-server/wiki/Deploying-your-own-server-to-production#configuring-the-phone-app) 23 | - Commit and push changes to your branch (e.g. `add-new-feature`) on your repo https://github.com/your-username/e-mission-phone.git 24 | 25 | ``` 26 | $ git commit 27 | $ git push origin add-new-feature 28 | ``` 29 | - Generate pull request into the *`custom-ui`, not `master`* branch) 30 | - I will review for basic sanity, merge and deploy to your channel 31 | - Ask people to deploy from your channel (e.g. https://e-mission.eecs.berkeley.edu/#/client_setup?new_client=custom-ui&clear_usercache=true&clear_local_storage=true) 32 | - ???? 33 | - Knowledge! -------------------------------------------------------------------------------- /docs/dev/future/collabathon_ideas.md: -------------------------------------------------------------------------------- 1 | | Idea name | Idea summary | Link to issue with more details | Estimated effort | Skills needed | 2 | |-----------|--------------|---------------------------------|------------------|---------------| 3 | | TripEdit | End to end support to fully edit trips | UX: https://github.com/e-mission/e-mission-docs/issues/374 backend: https://github.com/e-mission/e-mission-docs/issues/476 | 3 person days for each skill | UX designer, front-end developer, backend developer | 4 | 5 | -------------------------------------------------------------------------------- /docs/dev/future/foundations.md: -------------------------------------------------------------------------------- 1 | Here's a list of potential open source foundations we can join and the related projects in each. 2 | 3 | | Foundation | Related Projects | Notes | 4 | |------------|------------------|-------| 5 | | [Apache](http://incubator.apache.org/) | Open Climate Workbench | Most other projects are middleware related, need to find a champion + mentors | 6 | | [Linux UC](https://uc.foundation/#) | [Mapzen](https://opensourceforu.com/2019/01/open-source-mapping-platform-mapzen-joins-linux-foundation/), [CARTO](https://carto.com/blog/urban-computing-foundation/) | Most of the TAC seems to be from Big Tech companies, except UCSD | 7 | | [Eclipse openMobility Working Group](https://openmobility.eclipse.org/) | Eclipse SUMO, Eclipse MUSTANG | Mustang recently joined (Oct 2019), both seem to be simulation-based | 8 | | [Open Mobility Foundation](https://www.openmobilityfoundation.org/) | Open spec (MDS) focuse on micro-mobility, want to expand to other areas | Lots of cities involved, and want to broaden scope beyond micro-mobility | 9 | | [Open Transit Software Foundation](https://opentransitsoftwarefoundation.org) | One Bus Away | One Bus Away developers and agencies | 10 | | [Zephyr Foundation](https://zephyrtransport.org/) | ??? | They have a list of useful tools, but I don't see anything that they have directly built | 11 | | [Software Freedom Conservancy](https://sfconservancy.org/) | Open Trip Planner | ??? | 12 | | [Urban Data Science Toolkit](https://github.com/UDST) | UrbanSim | Foundation under consideration | 13 | 14 | Looks like the general trend in the transportation world is to make a new non-profit for each piece of open source software, instead of working with umbrella organizations like in computing. Maybe this is because open source projects are so novel in the domain that people wait until there is sufficient critical mass to create the nonprofit. 15 | -------------------------------------------------------------------------------- /docs/dev/future/more_custom_auto_config.md: -------------------------------------------------------------------------------- 1 | # Issue Template Form -> Config File PR 2 | 3 | ## GitHub Actions 4 | 5 | The config generation process has now been (at least partially) automated through the use of a GitHub workflow. The following steps will occur: 6 | 7 | - partners fill out the issue template "form" 8 | - upon submission of an issue in the `nrel-openpath-deploy-configs` repo, if it is an instance of the form, a worflow is launched to: 9 | - read the issue 10 | - convert it into a JSON file, formatted properly 11 | - submit a pull request or update an existing pull request to create or update the file 12 | 13 | If this workflow executes sucessfully, some configs should be ready to merge immidiately. 14 | 15 | Others may require additional steps like getting surveys uploaded or creating a notification schedule. 16 | 17 | ## Guidance before merge 18 | 19 | Obvious things to check for before merging is that it is a known deployment, that there are no missing fields, and than any surveys or custom labels have been provided 20 | 21 | ## Changes to the Configs 22 | 23 | As OpenPATH continues to grow, there may be instances where we need to update the configs - this will require updating the automation. 24 | 25 | The issue template is stored in `nrel-openpath-deploy-configs/.github/ISSUE_TEMPLATE/add-new-config.yml`. You can edit that file to add fields to the form. My convention has been to provide the `id` of form fields to match the key in the config JSON file. 26 | 27 | The bulk of processing happens in `nrel-openpath-deploy-configs/.github/actions/convertIssue/parse-issue-body.js`. Update this file to add the new key/value pair where appropriate. 28 | -------------------------------------------------------------------------------- /docs/dev/future/overview.md: -------------------------------------------------------------------------------- 1 | ## Overview: ## 2 | e-mission is an open-source, extensible platform for instrumenting human travel 3 | data. By understanding where, when, how and why people travel, we can, at a 4 | personal level, nudge people towards sustainable modes of transportation, and 5 | at a structural level, plan our urban areas better. 6 | 7 | You might want to start with this overview paper: 8 | http://cs.berkeley.edu/~shankari/emission_trb_2017_paper.pdf 9 | 10 | There are three main areas where we need to improve the system going forward. While it is possible to file issues for each of them, having one overview document with all the research links makes it easier to get an overall picture of what needs to be done. 11 | 1. [Privacy](privacy.md) 12 | 1. [Analysis](analysis.md) 13 | 1. [Scalability](scalability.md) 14 | 15 | -------------------------------------------------------------------------------- /docs/dev/future/privacy.md: -------------------------------------------------------------------------------- 1 | ## Privacy Improvements ## 2 | Location data is very privacy sensitive 3 | (http://www.nature.com/doifinder/10.1038/srep01376). This is particularly true 4 | for a location _timeline_, since anonymization techniques fail if there are 5 | repeated patterns in the data. For example, identifying a users' most common 6 | location at night and during the day can help us identify their home and work 7 | locations. 8 | 9 | But these repeated patterns are also very useful. At the personal level, we 10 | want to give people recommendations for their most common trips, or use their 11 | patterns to infer the characteristics of their travel. At the structural 12 | level, we want to aggregate the data from citizens to see repeated problems 13 | that can be targeted for fixing. 14 | 15 | The primary difference between the intrusive and useful privacy-sensitive analyses 16 | is control and ownership. 17 | 18 | Users should own their data, the analysis that runs on it, and the way 19 | in which the results are used. While users do currently get to control what raw 20 | data is collected from their smartphones, once the data has been collected, 21 | they lose control over it, and it can be re-shared in ways that they do not 22 | expect. This argues for a more mediated sharing experience at a higher level in 23 | the data stack. Some examples are 24 | ![Bad privacy examples](../../assets/future_work/privacy_examples_bad.png) 25 | 26 | 27 | Note that this high level overview recurs in many scenarios, and there is ongoing work, 28 | including in the RISE lab, around addressing some of these issues. We may be able to use 29 | some of these in our own work. For example, check out 30 | [this talk from Dawn Song](https://keystone-enclave.org/files/dawn-nsf-2018-v5.pdf). 31 | 32 | This level of control involves integrating two existing research areas. An overall system diagram is 33 | ![Privacy system diagram](../../assets/future_work/privacy_final_design.png) 34 | 35 | ### Computation on encrypted data/secure execution ### 36 | Conceptually, for users to own their own data, they need to run a server that 37 | they maintain and that collects their data. They can then choose to install 38 | the analysis scripts that they want on the server, and see the results 39 | themselves. 40 | 41 | Since few users run their own servers, this will typically involve storing 42 | encrypted data on shared infrastructure. But then how can users run algorithms 43 | against this encrypted data? 44 | 45 | - One option is to run algorithms directly against encrypted data. There has 46 | been prior work on [searching encrypted text directly](https://people.eecs.berkeley.edu/~raluca/mylar.pdf) and decrypting it in the browser to display the text. However, for maximum flexibility and performance, we typically want to run analysis on servers instead of user phones, and to have access to the results to participate in aggregate queries. There has been more recent work in this area from the RISE lab, some potentially unpublished, which we should explore. 47 | - [Opaque: An Oblivious and Encrypted Distributed Analytics Platform](https://people.eecs.berkeley.edu/~wzheng/opaque.pdf) 48 | - [Machine Learning Classification Over Encrypted Data](https://eprint.iacr.org/2014/331) 49 | 50 | - A second option is to use stored keys to decrypt the data before processing, 51 | and run the processing in a secure execution environment (e.g. SGX). 52 | Unfortunately, in order to decrypt the data, we need to store private keys, and 53 | have them accessible on the server for decryption. But then if the private 54 | keystore manager is compromised, then the attacker has access to your private 55 | key and all your data. Do web of trust/delegated authorization schemes (such as WAVE)[https://docs.google.com/document/d/1MGoL5wgVPyIdDJnEwA3We9USi3ZDKNy2pou_sppx6w8/edit?usp=sharing] 56 | solve this problem? And once the data is decrypted, root on the server where it 57 | is running will have access to it, which is why it needs to run in a hardware 58 | enclave(https://keystone-enclave.org/). Does it make sense to run full algorithms in a hardware enclave? It looks like it is currently used primarily for contract checking. Will the overhead be too high? 59 | - https://keystone-enclave.org/ 60 | - https://en.wikipedia.org/wiki/Trusted_execution_environment 61 | 62 | ### Privacy-preserving aggregation ### 63 | For structural analysis, we need to see results across a wide range of users. 64 | Conceptually, users can also choose to participate in aggregate queries for 65 | results, with controls for which results to share and how much aggregation they 66 | want to participate in. 67 | 68 | #### Related work: #### 69 | - [OpenPDS: Protecting the Privacy of Metadata through SafeAnswers](https://doi.org/10.1371/journal.pone.0098790) 70 | - [PDVLoc: A Personal Data Vault for Controlled Location Data Sharing](https://doi.org/10.1145/2523820) 71 | 72 | This kind of aggregate query is typically handled using differential privacy. 73 | However, differential privacy is challenging for timeseries data since it has 74 | so much structure. However, there has been prior work on differential privacy 75 | for certain kinds of aggregate queries against timeseries data. 76 | 77 | #### Related work: #### 78 | - [Privacy-Preserving Aggregation of Time-Series Data](https://amplab.cs.berkeley.edu/publication/privacy-preserving-aggregation-of-time-series-data/) 79 | - [Differentially private aggregation of distributed time-series with transformation and encryption](http://dl.acm.org/citation.cfm?id=1807247) 80 | 81 | 82 | #### Types of queries: #### 83 | 84 | We would like to support the following types of queries. 85 | - Point queries: These represent the kinds of queries that could be answered by a sufficiently complex sensor embedded in the infrastructure. For example: 86 | - How many people travelled on road segment `x` (where `x` is the OSM id such as https://www.openstreetmap.org/way/242298339) 87 | - What is the mode share on road segment `x` (e.g. 25% walk, 25% bike, 50% car)? 88 | - Both of the above queries over various time ranges (e.g. 3pm - 5pm on weekdays in the summer, etc) 89 | - Trajectory queries: These are still count queries, but they represent information that you can only find out through trajectories. For example: 90 | - Of the people passing through the intersection of Castro and the train tracks, how many are turning left? 91 | - How many people are turning onto Shoreline right after that? 92 | - Does this vary by mode? 93 | - Where do people who come to the train station come from? 94 | - counts on each of the access roads? 95 | - for each access road, counts along blocks that are the origins of the trips? 96 | - for each of the access roads, how long did it take for people to reach the train station along that route? 97 | - Where do people who leave Mountain View City Hall between 3pm and 5pm on weekdays in the summer go (can be a polygon)? 98 | - What is the distribution of travel times for travelers between Mountain View City Hall and the google campus in North Bayshore? 99 | - Model queries: In some ways, these are the easiest because they use standard machine learning, and there is a lot of existing work on federated databases. We should be able to do this for some subset of 100 | - Extract features of interest (e.g. time, cost, etc) from the aggregate of all the trips 101 | - Create a logistic regression model and determine the population-level coefficients for the features 102 | - Ideally, you would be able to do this for subsets of the population that can be chosen by the previous two methods - e.g. 103 | - find the time and cost coefficients for all people who travel along Castro street. 104 | - find the time and cost coefficients for all people who arrive at the Mountain View train station 105 | -------------------------------------------------------------------------------- /docs/dev/future/scalability.md: -------------------------------------------------------------------------------- 1 | ## Scalability Improvements ## 2 | This is the most straightforward task, so if you are new to research, you 3 | should consider starting with this. The first task on the task list, in particular, 4 | is relatively well defined for a research project, although it still has some 5 | ambiguity around the metrics to use. 6 | 7 | Current e-mission scalability, at least for a fully functioning server, is poor. Before I 8 | set up the multi-tier system 9 | (https://github.com/e-mission/e-mission-server/issues/530), we were running on 10 | an m3.xlarge server (64-bit, 4 vCPU, 15 GB RAM). At that time, the analysis pipeline for ~ 50 active 11 | users would take more than a day to run. Even worse, the response time for 12 | users when the pipeline was running was pathetic, on the order of minutes 13 | rather than seconds. Users would give up the app because they assumed that it 14 | was not working. 15 | 16 | This is pretty surprising, since the pipeline only looks at unprocessed 17 | entries, so should be `O(nActiveUsers)`. The only explanation for this is that 18 | the underlying storage layer is not scaling linearly, and stored data from 19 | inactive users is causing queries, even on only active users, to be slow. It is 20 | true that some of the inactive users had a lot of entries - for example, some 21 | of them were test phones that were reading data at the maximum frequency 22 | possible. However, the total number of users, both active and inactive, was 23 | under 5000, which seems low to have a minutes-long response time. 24 | 25 | ### Improve current scalability ### 26 | The first goal of this task is to improve the performance of the server so that 27 | projects can run a reasonable study without spending too much money. 28 | I'm going to define a reasonable study as: 29 | - 10,000 people 30 | - location data every 30sec on android/15m on iOS 31 | - 6 months of data collection 32 | 33 | I'm going to define a reasonable amount of computing and storage resources as: 34 | - *compute*: `m5.xlarge` instance (4 vCPU, 16 GB) 35 | - *storage*: 10 GB I/O optimized EB2 instance 36 | 37 | This means that the compute + storage for such a study should cost ~ $150 for compute + $30 for storage ~ $180 total. 38 | Ideally, we would be able to specify the users + duration supported for a bunch 39 | of discrete instance points - e.g. 40 | - `m5.large` instance + 1 GB GP2 EB2 instance (~$70 for compute + ~ 10 cents for storage = $70 total) 41 | - ... 42 | 43 | ### Explore new techniques ### 44 | Once the first task is done, we should explore other techniques for not just 45 | performance but elasticity. For example, can we use serverless (e.g. lambda) 46 | calculations to run the analysis? If so, we technically need only storage, no 47 | dedicated compute resources. Can we offload old raw data into lower cost, low 48 | performance storage to reduce costs?... 49 | 50 | We should also work closely with the privacy team to adapt to their proposed 51 | architecture. For example, should we split out the storage into separate 52 | independent units so that they can encrypt them separately? Or should we 53 | continue storing all data into one database, just encrypted with different 54 | keys? Is the storage choice for multiple separate datastores different from the 55 | choice for a single datastore? etc 56 | 57 | ### Task list ### 58 | 1. Create a workload generator. The work generator should consist of three parts: 59 | - _Synthetic data generator_: There is an existing trip generator 60 | (https://github.com/e-mission/e-mission-server/tree/master/emission/simulation), 61 | however it is obsolete. We need to replace it with a generator that creates 62 | artificial data in the new format. 63 | - There won't be existing solutions for this since the data is specific 64 | to e-mission. You can try to see if there is something similar that you 65 | can adapt, but set a time limit on the search since it is unlikely. 66 | - _Workload simulator_: A harness that generates API calls at a pre-defined 67 | rate against the system (e.g. x `get` calls/sec, y `put` calls/sec, z 68 | `getTimeline` calls/sec) and records the response time. 69 | - Note that this can be combined with the synthetic data generator - the 70 | generated data can be inserted into the system using API calls. This 71 | mode is closer to the operation of a real system. 72 | - Note that there are probably existing solutions for this. Look around 73 | first. Don't fall into the Not Invented Here trap. 74 | - _Server Launcher_: This is optional, but it will probably make your life 75 | easier, and will make end-users' lives easier as well. Write a script 76 | that, given a particular deployment configuration, creates an e-mission 77 | server instance with that configuration, installs the server, configures the 78 | server properly and launches the cronjobs. 79 | - There are approximately 1 million solutions for this. Use them. One of 80 | the current users (in Australia) supposedly worked on a Docker 81 | configuration for e-mission. You can contact https://github.com/asiripanich to see if you 82 | can start with that. 83 | 1. Run various workloads against various deployment configurations and figure 84 | out how the server scales 85 | 1. If the performance is sub-optimal, look at the logs to figure out what is 86 | slow and fix it. 87 | - I suspect that the first fix will involve swapping out the database 88 | server from mongodb to a real timeseries database. Or maybe there is a 89 | better way to use mongodb to store intermittent timeseries data? 90 | - This may not solve the scalability issues, in which case you need to 91 | figure out what the next bottleneck is, and rinse and repeat 92 | 1. Once you get to the limits of performance with the current architecture, 93 | explore other techniques, primarily for elasticity 94 | - Serverless computations, particularly if they can be combined with secure 95 | execution, could allow us to simply store user data and spin up 96 | computation on demand 97 | - Supporting longer response times for older data would allow us to support 98 | long-term data collection (decades or more). The standard techniques for 99 | this is tiered storage; and it has been implemented already, primarily 100 | for photo retrieval at Facebook/Google. Look for an existing OSS solution 101 | before rolling your own. 102 | 103 | ### A note on Not Invented Here ### 104 | Note that for several of these, there are existing solutions within academia. 105 | They will almost certainly not be as polished as commercial solutions, but 106 | please try to use them anyway. Grad students will be excited to have you use 107 | their work, and should be supportive as long as you are not asking them basic 108 | questions (e.g. how do I look at a log file?). 109 | 110 | And note that this is a *research* project, not a commercial one. This means 111 | that in addition to getting things done, which you must do, we want to enter 112 | into a dialog about how systems should be built. Using academic projects allows 113 | us to have that conversation with other systems researchers, and to hopefully, 114 | discover underlying principles that can be generalized and reused. 115 | -------------------------------------------------------------------------------- /docs/dev/future/unanswered_questions.md: -------------------------------------------------------------------------------- 1 | ## Privacy ## 2 | All papers assume that everybody has data for every time period. Is that true for e-mission? 3 | - can't you just assume that people have a value, but it is zero? 4 | - no because what about averages? If you assume 0 values for people who are not participating, averages will be off by a lot 5 | - for max and min, may be an issue for all neg/pos values 6 | - user doesn't respond or responds with "no data" 7 | - violates aggregator obliviousness 8 | - is that important? 9 | - if too many missing users, noise is incorrect? unless key dealer fixes it 10 | - noise is bounded (O(1) aggregation error) 11 | - still high but better than re-distributing 12 | - can we make this even better for a small subset of functioning users? 13 | - are there any concerns about a small subset? 14 | - describe your approach, (aggregator asks key dealer for noise corresponding to subset of non-functioning users) and get feedback 15 | - compare naive approach of redistributing keys, currently published approach of key dealer adding noise or proposed approach of aggregator requesting noise 16 | 17 | ## Secure execution ## 18 | - is keystone a combination of sanctum (software) and some other project in hardware (aegis/ascend)? 19 | - sanctum assumes trusted target software. so basically that the software running in the enclave is not malicious. Nick, do you have any questions around this? 20 | - what does it take to start experimenting with keystone? what is the keystone v1 release date? 21 | - is the workshop material online? 22 | - what would you recommend? computation against encrypted data or secure execution? is there a link to the pros and cons? 23 | - open source hardware - what is the state of tooling? 24 | - how do we access keystone hardware? do we have to actually burn an FPGA? 25 | - model is: decrypt, process, encrypt. how do get the secret key to decrypt? 26 | - store data in datastore encrypted with user's SK 27 | - user stores SK on unshared device (e.g. phone) 28 | - user and enclave negotiate symmetric key exchange right after enclave verification 29 | - user sends data to enclave encrypted with SK 30 | - Challenge: only user has SK, data is inaccessible without device storing SK (phone) being online. Conceptually, the enclave is like a little bit of the phone that can run cloud code. 31 | - consistent with signal and WhatsApp. Is this what they do under the hood? 32 | - how will this work for processing, e.g. re-running the pipeline? 33 | - users will need to launch **their** pipeline in an enclave on every sync 34 | - how will this work for aggregate queries? 35 | - users may be offline when the query is received, or may not want to participate over data plans 36 | - One idea: the query need not be instantaneous. Instead, it can include a time limit. The aggregator can poll users for data as long as the query is active. This means that users who come online only intermittently can also participate. Queries may take a day to run, but that is still faster than recruiting users and conducting a study. 37 | - We can improve performance by allowing the queries to include the number of users to participate. In that case, the aggregator can return as soon as the number of users has been reached, instead of waiting for the query to expire. This may also have implications in terms of the noise calculations and the protocol, but it is consistent with the previous issue of having data for only k/n users for a particular query. 38 | - how will the user retrieve data to be decrypted? Note that all data is stored encrypted, so nobody other than the user can see anything about it. Now assume that the user is trying to retrive some subset of the data, potentially to run algorithms against it, or to return it for a query. The naive approach is for the user to retrieve all the data, decrypt it, load it into a timeseries, and query against it. That also seems pretty inefficient. Options we discussed were: 39 | - storing the query keys without encryption (e.g. store the timestamps unencrypted). But this will leak information because we only store data when the user is in motion, so knowing when we have data will leak information on when we took trips. Also, we expect that aggregate queries will typically be gated by not just time but also by geolocation, and if you expose both time and geolocation then what is left 40 | - storing the data in subsets indexed by time blocks (e.g. each month). This may also leak some information (e.g. on the amount of travel per month) but maybe that is OK. Doesn't have a solution for the geolocation though 41 | - we can also estimate the data and see how bad the naive solution will be. Maybe we start with the naive solution and build in a performance optimization later. 42 | 43 | ## Experiment to set up a skeleton of secure execution for aggregate queries ## 44 | This is an outline of an experiment to set up a skeleton of secure execution for aggregate 45 | queries. This will allow us to verify that all the assumptions needed for the 46 | enclave based aggregation of queries before we delve deeper into the 47 | implementation. 48 | 49 | Terminology: 50 | - `Q`: query script 51 | - `e_a`: enclave running the aggregation script `s_a` 52 | - `e_u1 ... e_un`: enclaves running the user script 53 | 54 | Setup: 55 | - Create two possible aggregator programs `s_a_valid` and `s_a_invalid`. Both 56 | programs will expect to receive messages from user enclaves, and will return 57 | a count of the number of messages received to the querier. 58 | - Change the scripts slightly (potentially through included static dummy strings) to ensure 59 | that their hashes are slightly different. 60 | - Record the hashes `h_a_valid` and `h_a_invalid` 61 | - Create user programs that will receive an (IP address, valid hash) pair. The 62 | program should contact the enclave at the specified IP address and check its 63 | hash. If the hash is valid, send it the enclave an encrypted message. If the hash is 64 | invalid, ignore. 65 | - Create a query script that takes three inputs: 66 | - `n`: number of users 67 | - `aggregator`: aggregator program 68 | - `valid hash` 69 | 70 | The script should spin up `e_a` with the specified aggregator program, and 71 | spin up `n` `e_u` with the (only, hardcoded) user program. It should then 72 | send messages to all the `e_u` with the IP of the aggregator and the 73 | specified valid hash. In the real world, the querier is untrusted and will not 74 | submit the valid hash, but let's do it this way for now to make it easy to 75 | test. 76 | - Note that none of these programs have to be in python. They can be in C++ for 77 | now and we can figure out the integration with python once we know how to run 78 | python in an enclave 79 | 80 | Tests: 81 | - Run the query script with `s_a_valid` and `h_a_valid`. Expect the aggregator to return `n` 82 | - Run the query script with `s_a_invalid` and `h_a_valid`. Expect the aggregator to return 0 83 | -------------------------------------------------------------------------------- /docs/dev/publicdashboard/Working_with_public_dashboard.md: -------------------------------------------------------------------------------- 1 | # Note on execution of public dashboard to launch the notebook 2 | - First execute `mapping_dictionaries.ipynb` before launching any other notebooks. 3 | 4 | # Test Scenario for Public Dashboard changes: 5 | 1. Front end testing 6 | - Make sure the dropdown menu have valid list of metrics. 7 | - Make sure the default charts are loaded at the first instance of loading the website. 8 | - Make sure all the charts are being launched or the error table is displayed. 9 | 2. Test with loading and unloading different program/study dataset. 10 | - Load different datasets, run the scripts to execute the notebooks to make sure the charts are being generate properly. 11 | - Between datasets, drop the old dataset and load the new dataset. 12 | - Also check with empty MongoDB. 13 | 3. After the code has been merged, validate the changes on the staging environment. 14 | - Generate a snapshot of the charts from the staging website. 15 | - Request for the dataset for staging-program. 16 | - Validate the changes for both the charts and the Front end aspect. 17 | 18 | # Different docker-compose yml files? 19 | We have two docker-compose yml files, one specifically to aid in running in development mode. Another to run in production - alike mode. 20 | 21 | 1. About the `docker-compose.dev.yml` file: 22 | - This is useful if you want to launch the Jupyter notebook and execute the code block and explore output of each code blocks. 23 | 24 | To launch docker-compose.dev.yml: 25 | ``` 26 | docker-compose -f docker-compose.dev.yml up 27 | ``` 28 | 29 | 2. About the `docker-compose.yml` file: 30 | - This is useful if you want to execute the scripts to run all the required notebooks, without making any changes in the existing notebooks. 31 | - Note: We use cron jobs to execute the different notebooks in a scheduled way daily to generate the charts. 32 | 33 | To launch docker-compose.yml: 34 | ``` 35 | docker-compose -f docker-compose.yml up 36 | ``` 37 | 38 | Key difference amongst these two docker-compose file is the `CRON_MODE=TRUE` in case of `docker-compose.yml` while its disabled for `docker-compose.dev.yml`. 39 | 40 | - `docker-compose.yml`: 41 | ``` 42 | services: 43 | notebook-server: 44 | environment: 45 | - .... 46 | - CRON_MODE= 47 | - .... 48 | ``` 49 | 50 | - `docker-compose.dev.yml` 51 | ``` 52 | services: 53 | notebook-server: 54 | environment: 55 | - .... 56 | - CRON_MODE=TRUE 57 | - .... 58 | ``` 59 | 60 | # Changes required in docker-compose.dev.yml and/or docker-compose.yml to load different dataset 61 | 62 | The default `docker-compose.yml` with DB_HOST=db will load the dataset in MongoDB stored as `Stage_database`. 63 | 64 | What if we want to load a different dataset than staging, and test for a different program/study? 65 | Let us consider the example below: 66 | - (Note: This might need to updated once https://github.com/e-mission/e-mission-docs/issues/1048 issue is checked-in.) 67 | ``` 68 | services: 69 | notebook-server: 70 | environment: 71 | - DB_HOST=mongodb://db/openpath_prod_usaid_laos_ev 72 | - .... 73 | - STUDY_CONFIG=usaid-laos-ev 74 | ``` 75 | 1. For testing different config: Change the STUDY_CONFIG, as `STUDY_CONFIG=`. 76 | Example: `STUDY_CONFIG=usaid-laos-ev` for the `usaid-laos-ev` study. List of other configs for study/program are accessible from: https://github.com/e-mission/nrel-openpath-deploy-configs/tree/main/configs 77 | 1. Similarly, change the `DB_HOST=`. 78 | Example: `DB_HOST=mongodb://db/openpath_prod_usaid_laos_ev` to load the dataset `openpath_prod_usaid_laos_ev` which has been loaded into the MongoDB. 79 | 80 | # Execution of different public dashboard notebooks automatically (without launching Jupyter notebook) 81 | --- 82 | 83 | ``` 84 | $ docker-compose -f docker-compose.yml build 85 | $ docker-compose -f docker-compose.yml up 86 | $ docker ps 87 | $ docker exec -it em-public-dashboard-notebook-server-1 /bin/bash 88 | $ root@06f07def2c0e:/usr/src/app# source setup/activate.sh 89 | $ (emission) root@06f07def2c0e:/usr/src/app# cd saved-notebooks 90 | ``` 91 | - Note: Make sure the `Year` and `Month` parameter for the executing notebook is not `None` while executing the `generate_plots.py` script. If it's set to None, it will not generate the charts. 92 | Execution of notebooks 93 | ``` 94 | $ (emission) root@06f07def2c0e:/usr/src/app/saved-notebooks# PYTHONPATH=.. python bin/update_mappings.py mapping_dictionaries.ipynb 95 | $ (emission) root@06f07def2c0e:/usr/src/app/saved-notebooks# PYTHONPATH=.. python bin/generate_plots.py generic_metrics.ipynb default 96 | ``` 97 | Refer to additional notebooks from https://github.com/e-mission/em-public-dashboard/blob/main/viz_scripts/docker/crontab 98 | 99 | Then go to http://localhost:3274/ 100 | 101 | - If you want to test for specific configuration (program/study): 102 | For example, if you want to test for usaid-laos-ev: 103 | http://localhost:3274/?study_config=usaid-laos-ev 104 | and test. 105 | 106 | # Launch and explore the database 107 | 108 | - Through the Server code 109 | 110 | ``` 111 | $ docker-compose -f docker-compose.yml up 112 | $ docker ps 113 | $ docker exec -it em-public-dashboard-notebook-server-1 /bin/bash 114 | $ cd emission 115 | ``` 116 | 117 | Use scripts and relevant usecases available here to access/update the database. 118 | 119 | # Testing with staging 120 | - Load the snapshot of the dataset into MongoDB. 121 | You may want to use `reset_snapshot_to_ts.py` to reset the timestamp of the dataset to a particular date/time. 122 | - Execute different required notebooks as mentioned above. 123 | - Load the http://localhost:3274/ 124 | - Access the staging website and validate changes against: https://openpath-stage.nrel.gov/public/ 125 | 126 | Test the dropdown of metrics using different `study_config` as show below for `ebikegj` 127 | https://openpath-stage.nrel.gov/public/?study_config=ebikegj 128 | 129 | # Existing issue with browser caching the old chart data 130 | - While testing the changes, it's a good idea to clear up the cache and reload the webpage to make sure the latest charts are loaded. 131 | - Always better to validate the changes via. the timestamp under the charts and the timestamp of generation of charts. -------------------------------------------------------------------------------- /docs/dev/tutorial/android-debugging.md: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | 3 | Everything required for this can be installed through the 4 | prereq\_android\_sdk\_install.sh script in e-mission-phone. The most important 5 | packages are the emulator, platform-tools, and a system-image for your 6 | architecture. Use `sdkmanager --list_installed` to check installed android 7 | packages. 8 | 9 | Add `$ANDROID_HOME/emulator` and `$ANDROID_HOME/platform-tools` to your `$PATH` 10 | if they aren't already added. 11 | 12 | # Setting Up the Target 13 | 14 | Setup the android emulator or connect a phone over usb/wifi. 15 | 16 | ## Android Emulator 17 | 18 | Note that the android emulator does not simulate the geofence so trips need to 19 | be started manually in the developer options. 20 | 21 | Create an Android Virtual Device: 22 | 23 | ``` 24 | avdmanager avd create \ 25 | -k 'system-images;android-34;google_apis_playstore;x86_64' \ 26 | -n 27 | ``` 28 | 29 | The default hardware profile can be used. 30 | 31 | Make sure to replace x86_64 if you're on a different host architecture. Use 32 | `sdkmanager --list` to see available architectures to install. 33 | 34 | Start the emulator: 35 | 36 | ``` 37 | emulator -avd 38 | ``` 39 | 40 | `adb` should connect to the emulator automatically, run `adb devices` to verify. 41 | 42 | ## Real Phone 43 | 44 | Turn on developer options and enable USB debugging then plug the phone into your 45 | computer. 46 | 47 | # Loading the Test Application 48 | 49 | Once `adb` is connected to the test phone you can use `adb install ` 50 | to install the test application. 51 | 52 | # Java Debugger 53 | 54 | Make sure developer options are enable (repeatedly tap the build number in 55 | settings). Go to "Select debug app" in developer options and choose the test 56 | app. This will start a jdwp port any time the app is launched. Run `adb jdwp` to 57 | get the active jdwp port. 58 | 59 | Forward the jdwp connection: 60 | 61 | ``` 62 | adb forward tcp: jdwp: 63 | ``` 64 | 65 | Anytime the app is relaunched it will create another jdwp connection so these 66 | two commands need to be run again. 67 | 68 | ## CLI 69 | 70 | ``` 71 | jdb -attach localhost: -sourcepath /platforms/android/app/src/main/java 72 | ``` 73 | 74 | [Tutorial for jdb](https://m.opnxng.com/@dmytro.ch/the-java-debugger-how-to-debug-java-without-ide-how-to-use-jdb-ef732d79f915) 75 | 76 | ## VSCode 77 | 78 | Install the [Language Support for Java](https://marketplace.visualstudio.com/items?itemName=redhat.java) and the [Debugger for Java](https://marketplace.visualstudio.com/items?itemName=vscjava.vscode-java-debug) 79 | extensions. 80 | 81 | Configure the `launch.json` file to attach to the forwarded jdwp port. 82 | 83 | ```json 84 | { 85 | "version": "0.2.0", 86 | "configurations": [ 87 | { 88 | "type": "java", 89 | "name": "Current File", 90 | "request": "attach", 91 | "hostName": "localhost", 92 | "port": , 93 | } 94 | ] 95 | } 96 | ``` 97 | 98 | Start the debugger from the "Run and Debug" tab. 99 | 100 | ## Emacs 101 | 102 | Install [lsp-mode](https://emacs-lsp.github.io/lsp-mode/), [dap-mode](https://github.com/emacs-lsp/dap-mode), and [lsp-java](https://github.com/emacs-lsp/lsp-java). 103 | 104 | Required config: 105 | 106 | ```elisp 107 | (use-package lsp-mode 108 | :init 109 | (setq lsp-keymap-prefix "C-c l") 110 | :commands lsp) 111 | 112 | (use-package dap-mode) 113 | 114 | (use-package lsp-java 115 | :config (add-hook 'java-mode-hook 'lsp)) 116 | ``` 117 | 118 | Navigate to a java file in the phone repo. Lsp-mode will ask to set the project 119 | root, make sure to set it to the root of the java code 120 | (`platforms/android/app/src/main/java`) and not the root of the phone repo. 121 | 122 | Verify that `jdtls` is installed by running `lsp-install-servers` and selecting 123 | `jdtls` 124 | 125 | Run `dap-debug` and select "Java Attach" then enter the port for the jdwp 126 | connection. 127 | 128 | -------------------------------------------------------------------------------- /docs/dev/tutorial/hands-on-overview.md: -------------------------------------------------------------------------------- 1 | ## e-mission workshop: understanding and influencing human travel behavior (hands-on overview) ## 2 | 3 | Read these papers for context. 4 | 5 | - [TRB paper describing how to use e-mission functionality](https://people.eecs.berkeley.edu/~shankari/emission_trb_2017_paper.pdf) 6 | - [The in-review paper on the e-mission architecture, please do not distribute](https://people.eecs.berkeley.edu/~shankari/em-arch.pdf) 7 | 8 | In this session, we will: 9 | 10 | - [Set up the e-mission development environment](small_ui_changes/quickstart.md) 11 | - [Download your own data](#download-your-own-data) 12 | - [Explore the data](#explore-the-data) 13 | - [Change the UI](../small_ui_changes/) 14 | - Change some existing analysis 15 | - Integrate (with OSM?) 16 | 17 | ### Download your own data ### 18 | 19 | - Download your data by emailing it to yourself (Profile -> Download JSON dump) 20 | - Save the data (`*.timeline`) locally 21 | - Open the data in a text editor (e.g. vim/sublime) 22 | - Figure out what kinds of data has been collected 23 | 24 | ### Explore the data ### 25 | 26 | - [Load the data](https://github.com/e-mission/e-mission-server#loading-test-data) (say as user `meme`) 27 | - Run the single user pipeline for user `meme` 28 | - Login to the phone app as `meme` and confirm that you can see the data 29 | - Launch the `Timeseries_Sample.ipynb` and set the user to `meme`'s UUID 30 | - Explore your data 31 | - Share one insight with the rest of us 32 | 33 | -------------------------------------------------------------------------------- /docs/dev/tutorial/small_ui_changes/change_trip_end_prompt.md: -------------------------------------------------------------------------------- 1 | Change the trip-end prompt 2 | --- 3 | 4 | e-mission supports generating notifications on the phone at various transitions of the trip state machine. These are currently: 5 | 6 | - trip start 7 | - trip end 8 | - tracking off 9 | - tracking on 10 | 11 | The transitions are generated on the phone using the [transitionnotify](https://github.com/e-mission/e-mission-transition-notify.git) plugin. They do not need an internet connection or even a server to be running. 12 | 13 | All the changes here assume that you have finished the install, and your installation satisfies *all* the success criteria. 14 | 15 | ### Registering for trip notifications ### 16 | 17 | 1. Open `www/js/incident/post-trip-prompt.js` 18 | 1. Check the last few lines of the file, and ensure that they say 19 | ``` 20 | $ionicPlatform.ready().then(function() { 21 | ptap.registerTripEnd(); 22 | ptap.registerUserResponse(); 23 | }); 24 | ``` 25 | 1. If the lines say `unregisterTripEnd` and `unregisterUserResponse` instead, change `unregister` -> `register` 26 | 1. If there is no `$ionicPlatform.ready()` callback, add it as shown above 27 | 1. Save the file 28 | 1. The change is detected and the UI reloads. 29 | 1. Force-kill the app, reload and reconnect. The notifications are not registered properly otherwise. 30 | ``` 31 | [phonegap] file changed /Users/shankari/e-mission/e-mission-phone/www/templates/intro/summary.html 32 | [phonegap] Running command: /Users/shankari/e-mission/e-mission-phone/hooks/after_prepare/010_add_platform_class.js /Users/shankari/e-mission/e-mission-phone 33 | ... 34 | [phonegap] [console.log] Ending config 35 | [phonegap] [console.log] Starting run 36 | [phonegap] [console.log] onLaunch method from factory called 37 | [phonegap] [console.log] Ending run 38 | ... 39 | [phonegap] [console.log] Running calorieData with 0 and 0 40 | [phonegap] [console.log] Running calculation with NaN and NaN 41 | [phonegap] 200 /__api__/autoreload?appID=39ec34dbbee54c9df1a13aa626c467ca 42 | ``` 43 | 44 | #### Testing #### 45 | In order to test this, you need to start or end a trip. The most accurate option, and the one with the least setup, to install the app onto a physical device and take a trip. But it is hard to make changes to the prompts in that case. To test the notifications in an emulator: 46 | - Turn on locations 47 | - on android: Settings -> Location -> High Accuracy. Then open the location extended controls for the emulator and "send" a location to the emulator. 48 | - on iOS: Debug -> Location -> Freeway Drive 49 | - Start/stop the trip 50 | - on both: (Developer Zone -> Start/Stop trip) 51 | - Extra things to note 52 | - the android emulator does not generate locations by default. You need to open the location extended controls and "send" a location. If you don't send the location, the app will remain stuck in the `start` state and you will never see any notifications. You can also upload a GPX or KML file for more realism. 53 | - iOS delivers all notifications to the app when it is in the foreground. so if you want to see a visual notification on iOS, connect safari debugger and put a breakpoint in `collect-settings.js` at line `cch.forceTransition(button.transition);`. When it is hit, lock the emulator screen and resume execution. Now the notification will be visible. Alternatively, use android. 54 | 55 | ### Configuring notifications ### 56 | 57 | Notifications are configured using the format from the [local notifications plugin](https://github.com/shankari/cordova-plugin-local-notifications.git). We use a fork that supports inline buttons. Note that in order to see the modified notifications in the devapp during development, you need to force kill and restart the app and re-download the UI. 58 | 59 | #### Configuring notification text #### 60 | 61 | 1. Open `www/js/incident/post-trip-prompt.js` 62 | 1. Find the definition of the `reportNotifyConfig` e.g. 63 | 1. Change the title to `How much do you love this app` 64 | 1. Change the text to `This is super awesome` 65 | 66 | ``` 67 | var reportNotifyConfig = { 68 | id: REPORT, 69 | title: "How much do you love this app?", 70 | text: "This is super awesome", 71 | icon: 'file://img/icon.png', 72 | smallIcon: 'res://ic_mood_question.png', 73 | sound: null, 74 | actions: actions, 75 | category: REPORT_INCIDENT_TEXT, 76 | autoClear: true 77 | }; 78 | ``` 79 | 80 | 1. Save the file 81 | 1. The change is detected and the UI reloads. 82 | 1. Force kill the app, restart and reconnect 83 | 1. Manually start/stop a trip from the developer zone 84 | 1. The notification now asks you whether the app is awesome, right? 85 | 86 | #### Configuring notification actions #### 87 | 88 | 1. Find the button configurations 89 | 1. Change `MUTE` -> `5_STARS` 90 | 1. Change `SNOOZE` -> `MEH` 91 | 1. Change `REPORT` -> `SUCKS` 92 | e.g. 93 | ``` 94 | var actions = [{ 95 | identifier: '5_STARS', 96 | title: '5 Stars', 97 | icon: 'res://ic_moreoptions', 98 | activationMode: 'background', 99 | destructive: false, 100 | authenticationRequired: false 101 | }, { 102 | identifier: 'MEH', 103 | title: 'Meh', 104 | icon: 'res://ic_moreoptions', 105 | activationMode: 'background', 106 | destructive: false, 107 | authenticationRequired: false 108 | }, { 109 | identifier: 'SUCKS', 110 | title: 'Sucks', 111 | icon: 'res://ic_signin', 112 | activationMode: 'foreground', 113 | destructive: false, 114 | authenticationRequired: false 115 | }]; 116 | ``` 117 | 1. Change the checks for the buttons `if (data.identifier === 'REPORT')` -> `if (data.identifier === 'SUCKS')` 118 | e.g. 119 | ``` 120 | if (data.identifier === 'SUCKS') { 121 | ... 122 | } else if (data.identifier == '5_STARS') { 123 | ... 124 | } else if (data.identifier === 'MEH') { 125 | ... 126 | ``` 127 | 1. Add a `console.log` or `Logger.log` statement to each check - e.g. 128 | 129 | ``` 130 | } else if (data.identifier == 'MEH') { 131 | Logger.log("User hit the meh button") 132 | .... 133 | ``` 134 | 135 | 1. Note that the first two actions are background actions, so are intended to perform operations automatically without user interaction. 136 | 1. Save the file 137 | 1. The change is detected and the UI reloads. 138 | 1. Force kill the app, restart and reconnect 139 | 1. Manually start/stop a trip from the developer zone 140 | 1. Click on the notification buttons 141 | 1. Check the console logs and verify that the correct functions are invoked (e.g. if you clicked on `MEH`, you see `User hit the meh button`. 142 | 143 | | Default prompt (iOS) | Default prompt (android) | New prompt (iOS) | New prompt (android | 144 | | ------------- | --------- | ------------ | ----------- | 145 | | ![default prompt ios](../../../assets/tutorials/change_trip_end_prompt/default_prompt_ios.png) | ![default prompt android](../../../assets/tutorials/change_trip_end_prompt/default_prompt_android.png) | ![new prompt ios)](../../../assets/tutorials/change_trip_end_prompt/new_prompt_ios.png) | ![new prompt android)](../../../assets/tutorials/change_trip_end_prompt/new_prompt_android.png) | 146 | | ![default prompt expand view ios](../../../assets/tutorials/change_trip_end_prompt/default_prompt_expand_view_ios.png) | | ![new prompt expand view ios)](../../../assets/tutorials/change_trip_end_prompt/new_prompt_expand_view_ios.png) | | 147 | | ![default prompt expand ios](../../../assets/tutorials/change_trip_end_prompt/default_prompt_expand_ios.png) | | | | 148 | -------------------------------------------------------------------------------- /docs/dev/tutorial/small_ui_changes/changes_needed_when_the_set_of_tabs_changes.md: -------------------------------------------------------------------------------- 1 | # Changes needed when the set of tabs changes 2 | --- 3 | 4 | An example of all these changes is at: https://github.com/e-mission/e-mission-phone/pull/304 5 | 6 | In general, to change the set of tabs shown in the UI, it is sufficient to comment out unwanted tabs, or restore required tabs in `www/templates/main.html` 7 | e.g. add a new tab 8 | 9 | ``` 10 | 11 | 12 | 13 | 14 | ``` 15 | 16 | or remove an existing one 17 | 18 | ``` 19 | 20 | 23 | ``` 24 | 25 | But there are a few considerations to keep in mind if you change the set of tabs significantly. 26 | 27 | ### What should the initial tab be? ### 28 | The logic about the first tab shown when the user launches the app is in `startprefs.getNextState`. 29 | The default (after onboarding is complete, there is no redirect, and there is no referral) is 30 | 31 | ``` 32 | 194 } else { 33 | 195 return 'root.main.metrics'; 34 | 196 } 35 | ``` 36 | 37 | Change this to the state you want (e.g. `root.main.diary` if you want to start with the diary). 38 | 39 | ### What to do if the user disagrees ### 40 | First, you need to ensure that there is a default state to go to if the user does not want to consent (e.g. hits "Disagree" on the consent screen). By default, as you can see from `www/js/intro.js`, 41 | 42 | ``` 43 | $scope.disagree = function() { 44 | $state.go('root.main.heatmap'); 45 | }; 46 | ``` 47 | 48 | we go to the heatmap screen. But if you have commented out the heatmap, you need to have a different screen for the redirect. 49 | 50 | #### Simplest option #### 51 | If you can use an existing state, you can just replace the state name, e.g. `$state.go('root.intro')` will take the user back to the start of the onboarding. But if you do want to add a new screen, maybe one that has information on answering questions about the data collection, you need to add a new state. 52 | 53 | #### More customization #### 54 | You add a new state with two steps: 55 | - a new HTML file in `www/templates` with the information to be shown on rejection (similar to `www/templates/main-heatmap.html`, e.g. `www/templates/main-reject.html`). Note that this is a top level screen, and needs to be in an `ion-view` - e.g. 56 | ``` 57 | 58 | 59 | 60 | 61 | ``` 62 | - a new state config in `www/js/main.js` that maps to an existing view in 63 | `main.html` and an existing controller - e.g. 64 | 65 | ``` 66 | .state('root.main.refuse', { 67 | url: '/refuse', 68 | views: { 69 | 'main-cci-about': { 70 | templateUrl: 'templates/intro/refuse.html', 71 | controller: 'CCIAboutCtrl' 72 | } 73 | } 74 | }); 75 | ``` 76 | An example of such a change is https://github.com/e-mission/e-mission-phone/pull/281, with some corrections in https://github.com/e-mission/e-mission-phone/pull/284 77 | 78 | ### Check the list of states that display personalized information ### 79 | If a particular state displays personalized information, and the user tries to navigate to it without having consented, we should pop up the consent form again. Without consent, we will not collect data, and we cannot show any personalized information. The list of personalized states is in `www/js/controllers.js` and by default, it is 80 | 81 | ``` 82 | var personalTabs = ['root.main.common.map', 83 | 'root.main.control', 84 | 'root.main.metrics', 85 | 'root.main.goals', 86 | 'root.main.diary'] 87 | ``` 88 | 89 | If any personalized tabs have been added, they need to be added to this list as well. Note that the reverse is not true - if any personalized tabs have been commented out, they do not need to be removed from this list, although there is no harm in removing them. -------------------------------------------------------------------------------- /docs/dev/tutorial/small_ui_changes/easiest_way_to_change_text.md: -------------------------------------------------------------------------------- 1 | # Changing text through the GitHub UI # 2 | 3 | In general, e-mission UI changes are made by [creating a 4 | fork](https://help.github.com/articles/fork-a-repo/), [installing the 5 | devapp](https://github.com/e-mission/e-mission-devapp#installing), [testing the 6 | changes in the emulator](https://github.com/e-mission/e-mission-phone#updating-the-ui-only), and and if using the emTripLog 7 | base app, [submitting a pull request to the branch for your project](https://help.github.com/articles/about-pull-requests/). 8 | 9 | But sometimes, you want to make some quick changes without that overhead. The 10 | most common use case is probably a deployer who hired a programmer to make a 11 | custom UI. After the contract is over, she realizes that she needs to make 12 | minor changes to the text - e.g. 13 | 14 | - to the consent, once the final IRB approval is in place, 15 | - to prompts or other visual elements, based on feedback from a pilot 16 | 17 | This document outlines how she can make the changes herself directly online. 18 | easy as editing a google doc! 19 | 20 | ## Safe changes ## 21 | I would not recommend this for anything other than: 22 | 23 | - direct text substitutions (e.g. "foo" -> "bar") 24 | - adding non-visual tags (e.g. adding `alt` tags to images) 25 | 26 | Anything else has the potential for unexpected breakage and should really be 27 | tested before deployment. 28 | 29 | ## Steps for making changes ## 30 | 31 | 1. Find the file to change 32 | - If your changes will be on master, you can search directly in github 33 | - If your changes will be on a branch other than master 34 | - clone the repo to your local computer 35 | ``` 36 | $ git clone https://github.com/e-mission/e-mission-phone.git 37 | ``` 38 | - switch to the branch you want to change 39 | ``` 40 | $ git checkout --track origin/ 41 | ``` 42 | - search for the text using the standard tools for your computer (e.g. using `grep`, opening in an IDE and searching in the project...) 43 | 1. Select your branch in the github UI 44 | ![branch selection github ui](../../../assets/overview/easiest_way_to_change_text/branch_selection_github_ui.png) 45 | 1. Navigate to the file(s) you found in step 1 46 | ![navigate to file and edit](../../../assets/overview/easiest_way_to_change_text/navigate_to_file_and_edit.png) 47 | 1. Edit the file(s) directly and fill out the "Propose file change" message at the bottom so I know what you are doing. A message about write access is expected. "You’re editing a file in a project you don’t have write access to. Submitting a change to this file will write it to a new branch in your fork" 48 | 1. Submit the pull request with the change 49 | 1. See the change in your UI channel 50 | -------------------------------------------------------------------------------- /docs/dev/tutorial/small_ui_changes/quickstart.md: -------------------------------------------------------------------------------- 1 | 7 steps to an end-to-end development environment for UI-only changes 2 | --- 3 | 4 | 1. Install the server, including all [dependencies](https://github.com/e-mission/e-mission-server#dependencies) and the [e-mission code](https://github.com/e-mission/e-mission-server#installupdate) 5 | 1. Set up the devapp, by [downloading it](https://github.com/e-mission/e-mission-devapp#download) and [installing it in an emulator](https://github.com/e-mission/e-mission-devapp#installing) 6 | 1. Install the UI, including all [dependencies](https://github.com/e-mission/e-mission-phone#dependencies) and the [e-mission code](https://github.com/e-mission/e-mission-phone#installation) 7 | 1. [Start the server components](https://github.com/e-mission/e-mission-server#development), including mongod and the e-mission-server. 8 | 1. Load [sample data for a test user](https://github.com/e-mission/e-mission-server#quick-start). At the end of this step, you should be able to see this user in your local database. The server is now able to respond to API requests for this user. 9 | 10 | ``` 11 | (emission) e-mission-server $ ./e-mission-ipy.bash 12 | Python 3.6.1 | packaged by conda-forge | (default, May 11 2017, 18:00:28) 13 | Type "copyright", "credits" or "license" for more information. 14 | 15 | IPython 5.3.0 -- An enhanced Interactive Python. 16 | ... 17 | 18 | In [1]: import emission.core.get_database as edb 19 | storage not configured, falling back to sample, default configuration 20 | Connecting to database URL localhost 21 | 22 | In [2]: list(edb.get_uuid_db().find()) 23 | Out[2]: 24 | [{'_id': ObjectId('5b04d8582e26642a3d58d21e'), 25 | 'update_ts': datetime.datetime(2018, 5, 22, 19, 56, 24, 9000), 26 | 'user_email': 'test_july_22', 27 | 'uuid': UUID('908eb622-be3f-4cf4-bf04-1b7e610bea1c')}] 28 | ``` 29 | 30 | 1. Start the [UI development server](https://github.com/e-mission/e-mission-phone#running) and connect the devapp to it. The devapp will go through the "Downloading" and "Extracting" stages and then load the main e-mission UI. 31 | 1. Go through the e-mission onboarding process. When prompted for "dummy-dev auth", use `test_july_22`. 32 | - On android, you should not get any errors during the onboarding. 33 | - On iOS, you should get exactly one error saying that push notifications are not supported in the emulator 34 | After the onboarding is complete, you can load data for July 22 **2015** from the diary tab. 35 | 36 | #### Success checks #### 37 | 38 | | Login screen | Expected error (only on iOS) | Diary for 22 July | 39 | |--------------|-------------------------------------|-------------------| 40 | | ![login screen test user](../../../assets/overview/quickstart/login_screen_test_user.png) | ![expected ios error](../../../assets/overview/quickstart/expected_ios_error.png) | ![Diary view success](../../../assets/overview/quickstart/diary_view_success.png) | 41 | 42 | ##### Server logs ##### 43 | 44 | ``` 45 | START 2018-05-23 11:35:40.056231 POST /profile/create 46 | END 2018-05-23 11:35:40.115472 POST /profile/create 0.056568145751953125 47 | START 2018-05-23 11:35:40.535397 POST /result/metrics/timestamp 48 | END 2018-05-23 11:35:40.569467 POST /result/metrics/timestamp 908eb622-be3f-4cf4-bf04-1b7e610bea1c 0.03396773338317871 49 | ``` 50 | 51 | ##### UI logs ##### 52 | 53 | ``` 54 | [phonegap] [console.log] Consented in local storage, no need to show consent 55 | [phonegap] [console.log] DEBUG:changing state to root.main.metrics 56 | [phonegap] [console.log] loading root.main.metrics 57 | [phonegap] [console.log] Sending data {"freq":"D","start_time":1525824000,"end_time":1526947200,"metric":""} 58 | ... 59 | [phonegap] [console.log] twoWeeksAgoDuration = 60 | [phonegap] [console.log] twoWeeksAgoMedianSpeed = 61 | [phonegap] [console.log] twoWeeksAgoDistance = 62 | [phonegap] [console.log] Running calorieData with 0 and 0 63 | [phonegap] [console.log] Running calculation with NaN and NaN 64 | ``` 65 | 66 | #### Congratulations, you have now successfully set up the phone and server and connected them to each other #### 67 | -------------------------------------------------------------------------------- /docs/dev/tutorial/small_ui_changes/remove_tab.md: -------------------------------------------------------------------------------- 1 | Remove tab 2 | --- 3 | 4 | Several of the e-mission projects so far have removed some subset of the tabs. The naive way to remove a tab is to simply comment it out from the UI. 5 | 6 | ### Tab removal ### 7 | 8 | 1. Open `www/templates/main.html` 9 | 1. Comment out the game tab by changing 10 | ``` 11 | 12 | 13 | 14 | 15 | ``` 16 | to 17 | ``` 18 | 22 | ``` 23 | 1. Save the file 24 | 1. The change is detected and the UI reloads. 25 | ``` 26 | [phonegap] file changed /www/templates/intro/summary.html 27 | [phonegap] Running command: /hooks/after_prepare/010_add_platform_class.js 28 | ... 29 | [phonegap] [console.log] Ending config 30 | [phonegap] [console.log] Starting run 31 | [phonegap] [console.log] onLaunch method from factory called 32 | [phonegap] [console.log] Ending run 33 | ... 34 | [phonegap] [console.log] Running calorieData with 0 and 0 35 | [phonegap] [console.log] Running calculation with NaN and NaN 36 | [phonegap] 200 /__api__/autoreload?appID=39ec34dbbee54c9df1a13aa626c467ca 37 | ``` 38 | 1. Game tab is gone! 39 | 40 | 41 | ### Caveats ### 42 | 43 | 1. This is the most naive approach to removing the tab - just hide it from the user. This leaves background operations (e.g. invoked using `$ionicPlatform.ready`) in place. In order to fully remove them, you would want to remove the corresponding HTML and javascript as well. That is outside the scope of a *simple* change, use the ionic1 or angular1 documentation to experiment further. 44 | 1. Removing the heatmap tab is problematic because, since it is aggregate, and not personal, we redirect to it if the user does not consent. If you want to remove the heatmap tab, you need to change the disagree logic (https://github.com/e-mission/e-mission-phone/wiki/Changes-needed-when-the-set-of-tabs-changes), which involves a javascript change. 45 | 46 | -------------------------------------------------------------------------------- /docs/dev/tutorial/small_ui_changes/scheduling-trip-end-prompt.md: -------------------------------------------------------------------------------- 1 | # Schedule a trip-end notification 2 | Currently, the app prompts users to confirm their mode and purpose for each trip. While this is good, we might want, for example, to have them at the end of the day. 3 | 4 | Follow the following steps to schedule a trip-end notification. 5 | 6 | ### Step 1 — Disable the old trip-end notification. 7 | 8 | Comment out the content of the `registerTripEnd` method found in `www/js/tripconfirm/post-trip-prompt.js` to disable the trip-end notification, . 9 | 10 | ### Step 2 - Setup cordova notification plugin 11 | 12 | First, install the plugin 13 | 14 | ```shell 15 | $ cordova plugin add cordova-plugin-local-notification 16 | $ cordova prepare 17 | ``` 18 | 19 | Then add the following code snippet to `www/js/constant.js`. 20 | 21 | ```json 22 | CONSTANTS = { 23 | "notification": { 24 | "debug": false, 25 | "id": 6724531, 26 | "day": 3, 27 | "hour": 19, 28 | "every": "week" 29 | } 30 | } 31 | ``` 32 | 33 | > Create the `constant.js` file if it does not exit. 34 | 35 | The parameters are as follows: 36 | 37 | - **id**: The identifier of the notification. Allow to be unique in the notification stack and avoid conflicts with other notifications. Make sure that this id is not used by another notification. 38 | 39 | - **debug**: if it is true, it will repeat itself every minute from the connection to the application. 40 | If it is false it will take into account the parameters "day", "hour", "minute" and "every". 41 | 42 | - **hour**: The hour of the first notification (0-23) 43 | 44 | - **minute**: The minute of the first notification (0-59) 45 | 46 | - **day**: The day of the first notification: 47 | - 0 => Sunday 48 | - 1 => Monday 49 | - 2 => Tuesday 50 | - 3 => Wednesday 51 | - 4 => Thursday 52 | - 5 => Friday 53 | - 6 => Saturday 54 | 55 | - **every**: the interval at which to reschedule the reminder notification. The possible values ​​(text) are as follows: 56 | - second 57 | - minute 58 | - hour 59 | - day 60 | - week 61 | - month 62 | - year 63 | 64 | Checkout the [official plugin documentation](https://github.com/katzer/cordova-plugin-local-notifications) for more details. 65 | 66 | ### Step 3 - Create a new method for the notification scheduling 67 | 68 | To begin, add the following content to `www/js/splash/startprefs.js`. 69 | 70 | ```js 71 | startprefs.setReminderNotification = function () { 72 | // Set weekly notification 73 | Logger.log("_NOTIF: SET WEEKLY NOTIF"); 74 | Logger.log("_NOTIF: " + localStorage.getItem("reminderNotification")); 75 | if (!localStorage.getItem("reminderNotification")) { 76 | var notif = CONSTANTS.notification; 77 | var first_at, every; 78 | if (notif.debug === true) { 79 | first_at = new Date(new Date().getTime() + 20 * 1000); 80 | every = "minute"; 81 | } else { 82 | first_at = startprefs.getNextDayOfWeek(new Date(), notif.day); 83 | first_at.setHours(notif.hour, notif.minute, 0); 84 | every = notif.every; 85 | } 86 | $window.cordova.plugins.notification.local.schedule([ 87 | { 88 | id: notif.id, 89 | text: $translate.instant("trips-reminder"), 90 | firstAt: first_at, 91 | every: every, 92 | }, 93 | ]); 94 | 95 | $window.cordova.plugins.notification.local.on("click", function ( 96 | notification, 97 | state, 98 | data 99 | ) { 100 | $state.go("root.main.diary"); 101 | }); 102 | $window.cordova.plugins.notification.local.on("trigger", function ( 103 | notification, 104 | state, 105 | data 106 | ) { 107 | var reminderPopup = $ionicPopup 108 | .alert({ 109 | template: $translate.instant("trips-reminder"), 110 | }) 111 | .then(function (res) { 112 | $state.go("root.main.diary"); 113 | }); 114 | }); 115 | Logger.log("_NOTIF: END SET WEEKLY NOTIF"); 116 | localStorage.setItem("reminderNotification", 1); 117 | } 118 | }; 119 | ``` 120 | Then invoke it anywhere you like based on your use case. In our case the 121 | `startprefs.getNextState()` is a suitable place. 122 | -------------------------------------------------------------------------------- /docs/dev/tutorial/small_ui_changes/simple_client_changes.md: -------------------------------------------------------------------------------- 1 | 6 examples of common but simple client changes 2 | --- 3 | 4 | e-mission UI uses the open source [Ionic1](https://ionicframework.com/docs/v1/) Framework, which in turn is built on the [Angular1](https://angularjs.org/) Framework. There are multiple ionic1 and angular1 tutorials on the web, please go through them and familiarize yourself with their model-view-controller concepts. In particular, you want to get a sense of how `$scope` works and how it is used to display dynamic values in a HTML view. You might also want to read up on how to use CSS to change the UI look and feel. 5 | 6 | Here are some initial tutorials to get you started on [angular1](https://www.tutorialspoint.com/angularjs/angularjs_mvc_architecture.htm) and [ionic1](https://ccoenraets.github.io/ionic-tutorial/start-node-server.html). For the ionic1 tutorial, if you are using the devapp, you can skip module 1 (installing ionic). 7 | 8 | As you look for your own tutorials, make sure they are using version 1 of the UI frameworks. We do not support the most recent version of the frameworks due to lack of porting resources. If you would like to port the UI from ionic1 to the most recent ionic, please let me know. 9 | 10 | 1. [Change summary and consent](summary_and_consent.md) 11 | 2. [Remove a tab](remove_tab.md) 12 | 3. [Change colors](change_colors_to_your_theme.md) 13 | 4. [Display a customized survey](customized_survey.md) 14 | 5. [Change the trip end prompt](change_trip_end_prompt.md) 15 | 6. [Schedule the trip end prompt](scheduling-trip-end-prompt.md) 16 | -------------------------------------------------------------------------------- /docs/dev/tutorial/small_ui_changes/summary_and_consent.md: -------------------------------------------------------------------------------- 1 | Change summary and consent 2 | --- 3 | 4 | All e-mission projects so far have wanted to change the summary and consent to reflect their use of the platform. The summary and consent are part of the onboarding module. The HTML for the module is at `www/templates/intro` and the corresponding javascript is at `www/js`. Summary and consent changes replace the displayed text, and are HTML-only changes. 5 | 6 | All the changes here assume that you have finished the install, and your installation satisfies *all* the success criteria. 7 | 8 | ### Summary changes ### 9 | 10 | 1. Open `www/templates/intro/summary.html` 11 | 1. Change `E-Mission: Data driven carbon emission reduction` to `My Cool App: Get suggestions on your commute` 12 | 1. Save the file 13 | 1. The change is detected and the UI reloads. 14 | ``` 15 | [phonegap] file changed /Users/shankari/e-mission/e-mission-phone/www/templates/intro/summary.html 16 | [phonegap] Running command: /Users/shankari/e-mission/e-mission-phone/hooks/after_prepare/010_add_platform_class.js /Users/shankari/e-mission/e-mission-phone 17 | ... 18 | [phonegap] [console.log] Ending config 19 | [phonegap] [console.log] Starting run 20 | [phonegap] [console.log] onLaunch method from factory called 21 | [phonegap] [console.log] Ending run 22 | ... 23 | [phonegap] [console.log] Running calorieData with 0 and 0 24 | [phonegap] [console.log] Running calculation with NaN and NaN 25 | [phonegap] 200 /__api__/autoreload?appID=39ec34dbbee54c9df1a13aa626c467ca 26 | ``` 27 | 1. Log out (Profile -> Email) to restart the onboarding process 28 | 1. The summary title is now `My Cool App` 29 | 30 | ### Consent changes ### 31 | 32 | 1. Open `www/templates/intro/consent.html` 33 | 1. Change `E-Mission: Data driven carbon emission reduction` to `My Cool App: Get suggestions on your commute` 34 | 1. Change 35 | ``` 36 | This app is part of a travel behavior research platform developed at the 37 | University of California, Berkeley. The platform development is being led by K. Shankari under the 39 | supervision of Prof. David 40 | Culler. Our goal is to collect information from users about travel behavior 41 | across all modes, and use the aggregate information for better transportation 42 | infrastructure planning. 43 | ``` 44 | to 45 | ``` 46 | This app is part of a study conducted by my cool research institute in 47 | Terminus, First Foundation. The platform development is being led by 48 | Arkady Darrell under the supervision of Prof. Susan Calvin. Our goal 49 | is to see if robotic scheduling can mitigate the unsustainable travel 50 | behavior of the citizens of Trantor, in the presence of mind control by the 51 | inhabitants of the Second Foundation. 52 | ``` 53 | 1. Save the file 54 | 1. The change is detected and the UI reloads. 55 | 1. Log out (Profile -> Email) to restart the onboarding process 56 | 1. The consent page is now for the Trantor travel study. 57 | 58 | | How to logout | New title (my cool app) | New consent (trantor study) | 59 | | ------------- | --------- | ------------ | 60 | | ![how to logout](../../../assets/tutorials/simple_client_changes/how_to_logout.png) | ![new title (my cool app)](../../../assets/tutorials/simple_client_changes/new_title_my_cool_app.png) | ![new consent (trantor study)](../../../assets/tutorials/simple_client_changes/consent_for_trantor_study.png) | 61 | -------------------------------------------------------------------------------- /docs/install/deploying_production_docker.md: -------------------------------------------------------------------------------- 1 | e-mission has a fairly standard docker installation. 2 | However, I've found myself setting up servers from scratch a couple of times recently, and thought that it would be useful to write down the steps in case it is useful to have them listed in a single location. 3 | 4 | 1. [Install docker on ubuntu](https://docs.docker.com/engine/install/ubuntu/) 5 | 6 | ``` 7 | sudo docker run hello-world 8 | Unable to find image 'hello-world:latest' locally 9 | latest: Pulling from library/hello-world 10 | 0e03bdcc26d7: Pull complete 11 | Digest: sha256:49a1c8800c94df04e9658809b006fd8a686cab8028d33cfba2cc049724254202 12 | Status: Downloaded newer image for hello-world:latest 13 | 14 | Hello from Docker! 15 | This message shows that your installation appears to be working correctly. 16 | ``` 17 | 18 | 1. [Install docker-compose on ubuntu](https://docs.docker.com/compose/install/) 19 | 20 | ``` 21 | $ docker-compose --version 22 | docker-compose version 1.26.2, build eefe0d31 23 | ``` 24 | 25 | 1. Check out the [e-mission-docker repo](https://github.com/e-mission/e-mission-docker) 26 | 27 | ``` 28 | ls -a1 e-mission-docker/ 29 | . 30 | .. 31 | bin 32 | clone_and_start_server.sh 33 | Dockerfile 34 | Dockerfile.dev.server-only 35 | Dockerfile.dev.ui-only 36 | Dockerfile.nomkl 37 | examples 38 | .git 39 | LICENSE 40 | README.md 41 | start_devapp_serve.sh 42 | start_script.sh 43 | ``` 44 | 45 | 1. Copy out the multi-tier sample to a directory for this particular instance 46 | 47 | ``` 48 | $ cp -r e-mission-docker/examples/em-server-multi-tier-cronjob emdemo 49 | ``` 50 | 51 | 1. Change all the `CHANGEME` fields in the example + any other config changes needed 52 | 53 | ``` 54 | ~/emdemo$ grep -r CHANGEME . 55 | ./README.md:- you should look through all the `CHANGEME` locations and fill them out 56 | ./README.md:$ grep -r CHANGEME . 57 | ~/emdemo$ 58 | ``` 59 | 60 | 1. Launch the server! 61 | 62 | ``` 63 | ~/emdemo$ sudo docker-compose up 64 | ... 65 | Creating network "emdemo_emission" with the default driver 66 | Creating volume "emdemo_mongo-data" with default driver 67 | ... 68 | Building analysis-server 69 | ... 70 | Successfully tagged emdemo_analysis-server:latest 71 | Building web-server 72 | ... 73 | Successfully tagged emdemo_web-server:latest 74 | ... 75 | web-server_1 | } 76 | db_1 | 2020-08-12T05:27:21.126+0000 I FTDC [initandlisten] Initializing full-time diagnostic data capture with directory '/data/db/diagnostic.data' 77 | ... 78 | ``` 79 | 80 | 1. Test that it works ✔️ 81 | 82 | ``` 83 | $ curl http://localhost: 84 | 85 | ... 86 | 87 | ``` 88 | -------------------------------------------------------------------------------- /docs/install/manual_install.md: -------------------------------------------------------------------------------- 1 | # Manual installation # 2 | 3 | If you don't want to use docker, or you want to contribute to server development, you can also install the server directly. We use a virtual environment for the installation, so it will not mess up your regular python install. 4 | 5 | The installation instructions below are generally targeted towards OSX and \*nix shells such as bash. If you want to use Windows, we recomend using a POSIX compliant shell such as [gitbash](https://openhatch.org/missions/windows-setup/install-git-bash), which provides similarly rich commands. If you really want to use the Command Prompt, most commands should work, but you may need to convert `/` -> `\` to make the commands work. 6 | 7 | 8 | ## Install/update: ## 9 | ------------------- 10 | 11 | ### Installation ### 12 | is as simple as cloning the github repository. 13 | 14 | - If you do not plan to make changes to the code, clone the master repository. 15 | 16 | ``` 17 | $ git clone https://github.com/e-mission/e-mission-server.git 18 | ``` 19 | 20 | - If you might make changes or develop new features, fork (https://help.github.com/articles/fork-a-repo/) and clone your fork. 21 | 22 | ``` 23 | $ git clone https://github.com//e-mission-server.git 24 | ``` 25 | 26 | ### Update ### 27 | is as simple as pulling new changes. 28 | 29 | - If you are working off the master repository 30 | 31 | ``` 32 | $ git pull origin master 33 | ``` 34 | 35 | - If you are working off your fork, you will need to sync your fork with the main repository (https://help.github.com/articles/syncing-a-fork/) and then pull from your fork. 36 | 37 | ``` 38 | $ git pull origin master 39 | ``` 40 | 41 | ## Dependencies: ## 42 | ------------------- 43 | 44 | ### Database (mongodb) ### 45 | Start [Mongodb](http://www.mongodb.org/), version 8.0.4. 46 | 47 | #### Using docker (recommended) #### 48 | 49 | ``` 50 | $ docker run -p 27017:27017 mongo:8.0.4 51 | ``` 52 | 53 | Note that the command above _does not persist_ the data if the container is restarted. Docker has additional options for persistence, composition, etc. 54 | 55 | #### Using manual install #### 56 | 57 | 58 | 2. *Windows*: mongodb appears to be installed as a service on Windows devices and it starts automatically on reboot 59 | 3. *OSX*: You want to install homebrew and then use homebrew to install mongodb. Follow these instruction on how to do so ---> (https://docs.mongodb.com/v3.4/tutorial/install-mongodb-on-ubuntu/) 60 | 4. *Ubuntu*: http://docs.mongodb.org/manual/tutorial/install-mongodb-on-ubuntu/ 61 | 62 | 2. Start it at the default port 63 | 64 | `$ mongod` 65 | 66 | ### Python distribution ### 67 | We will use a distribution of python that is optimized for scientific 68 | computing. The [anaconda](https://store.continuum.io/cshop/anaconda/) 69 | distribution is available for a wide variety of platforms and includes the 70 | python scientific computing libraries (numpy/scipy/scikit-learn) along with 71 | native implementations for performance. Using the distribution avoids native 72 | library inconsistencies between versions. 73 | 74 | The distribution also includes its own version of pip, and a separate environment 75 | management tool called 'conda'. 76 | 77 | The distribution also includes an environment management tool called 'conda'. We will set up a separate `emission` 78 | environment within anaconda to avoid conflicts with other applications. 79 | 80 | - Install miniconda 81 | 82 | ``` 83 | $ source setup/setup_conda.sh 84 | ``` 85 | IMPORTANT: this script works for MacOSX and Ubuntu based Linux distributions, the OS must be specified on the command with one of two choices: ```Linux-x86_64``` or ```MacOSX-x86_64```. The final command would be this ```source setup/setup_conda.sh Linux-x86_64``` or this ```source setup/setup_conda.sh MacOSX-x86_64```. 86 | 87 | - Setup the `emission` environment. 88 | 89 | ``` 90 | $ source setup/setup.sh 91 | ``` 92 | 93 | - If you want to use an ipython notebook, use `$ source setup/setup_notebook.sh` instead. 94 | - If you want to use a less optimized version that uses less disk space, use `$ source setup/setup_nomkl.sh` instead. Note that this has not been extensively tested. 95 | 96 | - Verify that you are in the right environment - your prompt should start with 97 | `(emission)` and the `emission` environment should be starred. 98 | 99 | ``` 100 | (emission) ...$ conda env list 101 | # conda environments: 102 | # 103 | aws /..../anaconda/envs/aws 104 | emission * /..../anaconda/envs/emission 105 | firebase /..../anaconda/envs/firebase 106 | py27 /..../anaconda/envs/py27 107 | py36 /..../anaconda/envs/py36 108 | xbos /..../anaconda/envs/xbos 109 | root /..../anaconda 110 | ``` 111 | 112 | - If you have setup the environment already and just need to switch to it, 113 | 114 | ``` 115 | $ source setup/activate.sh 116 | ``` 117 | 118 | To switch out of the emission environment, or to manipulate it in other ways, read the conda documentation on environments https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html 119 | 120 | - Remember to re-run the setup script every time you pull from the main repository because the dependencies may have changed. 121 | 122 | ``` 123 | $ source setup/setup.sh 124 | ``` 125 | 126 | - When you are done working with e-mission, you can cleanup the environment and then just delete the entire directory. 127 | 128 | ``` 129 | $ source setup/teardown.sh 130 | $ cd .. 131 | $ rm -rf e-mission-server 132 | ``` 133 | 134 | - If you are not using 135 | 136 | 137 | ### Javascript dependencies ### 138 | 139 | Note: It is required only if the user needs a web interface for the server. Otherwise one can do without it as well. 140 | 141 | Tip: Run "bower install" instead if you are prompted password for 'https://github.com' after running "bower update". 142 | 143 | $ cd webapp 144 | $ bower update 145 | 146 | ## Run ## 147 | --------- 148 | 149 | 1. Activate the `emission` conda environment 150 | 151 | ``` 152 | $ source setup/activate.sh 153 | ``` 154 | 155 | 156 | 1. For OSX manual installs, start the database (Note: mongodb appears to be installed as a service on Windows devices and it starts automatically on reboot). 157 | 158 | $ mongod 159 | 2018-05-23T11:06:07.576-0700 I CONTROL [initandlisten] MongoDB starting : pid=60899 port=27017 dbpath=/data/db 64-bit ... 160 | 2018-05-23T11:06:07.576-0700 I CONTROL [initandlisten] db version v3.6.2 161 | ... 162 | 2018-05-23T11:06:08.420-0700 I FTDC [initandlisten] Initializing full-time diagnostic data capture with directory '/data/db/diagnostic.data' 163 | 164 | 1. **Optional** Copy configuration files. The files in `conf` can be used to customize the app with custom authentication options or enable external features such as place lookup and the game integration. Look at the samples in `conf/*`, copy them over and modify as necessary - e.g. 165 | 166 | $ find conf -name \*.sample 167 | # For the location -> name reverse lookup. Client will lookup if not populated. 168 | $ cp conf/net/ext_service/nominatim.json.sample conf/net/ext_service/nominatim.json 169 | 170 | 1. Start the server 171 | 172 | $ ./e-mission-py.bash emission/net/api/cfc_webapp.py 173 | storage not configured, falling back to sample, default configuration 174 | Connecting to database URL localhost 175 | analysis.debug.conf.json not configured, falling back to sample, default configuration 176 | Finished configuring logging for 177 | Replaced json_dumps in plugin with the one from bson 178 | Changing bt.json_loads from at 0x10f5fdb70> to 179 | Running with HTTPS turned OFF - use a reverse proxy on production 180 | Bottle v0.13-dev server starting up (using CherootServer())... 181 | Listening on http://0.0.0.0:8080/ 182 | Hit Ctrl-C to quit. 183 | 184 | 1. Test your connection to the server 185 | * Using a web browser, go to [http://localhost:8080](http://localhost:8080) 186 | * Using safari in the iOS emulator, go to [http://localhost:8080](http://localhost:8080) 187 | * Using chrome in the android emulator: 188 | * change `server.host` in `conf/net/api/webserver.conf` to 0.0.0.0, and 189 | * go to the special IP for the current host in the android emulator - [10.0.2.2](https://developer.android.com/tools/devices/emulator.html#networkaddresses) 190 | 191 | -------------------------------------------------------------------------------- /docs/manage/adding_limesurvey_support.md: -------------------------------------------------------------------------------- 1 | # Adding LimeSurvey Support 2 | --- 3 | 4 | In addition to the normal survey support, it is possible to configure a LimeSurvey support to use with the Survey screen. 5 | 6 | ## Dependencies 7 | 8 | - Python **LimeSurvey RC2 API**, you can find it on GitHub by following this link : 9 | https://github.com/lindsay-stevens/limesurveyrc2api, 10 | - [LimeSurvey](https://www.limesurvey.org/). 11 | 12 | ## Configuring LimeSurvey 13 | 14 | Before configuring `e-mission-server`, it is needed to configure a bit your LimeSurvey server. 15 | 16 | On your LimeSurvey admin interface : 17 | 18 | 1. Go to *Configuration -> Global settings*, 19 | 2. In the *Interfaces* tab, activate **JSON-RPC** by clicking on it, 20 | 3. Optionally, modify the RemoteControl URL which will be used by **e-mission-server**. 21 | 22 | ## Configuring e-mission-server 23 | 24 | 1. Go to `e-mission-server/bin/conf/net/ext_service`, 25 | 2. Add a `limesurvey.json` by copying `limesurvey.json.sample` and adapting it according to your server. 26 | 27 | ```json 28 | { 29 | "url": "RemoteControl API URL", 30 | "username": "Admin account of limesurvey", 31 | "password": "password", 32 | "url_surveys": "URL of the surveys" 33 | } 34 | ``` 35 | 36 | The `url_surveys` will be used to build an URL to be sent when the user will retrieve his/her surveys with the phone app. 37 | 38 | ## Sending a LimeSurvey survey 39 | 40 | For each new survey you will want to send, it will be necessary to quickly configure it. 41 | 42 | 1. After clicking on *Create a new survey*, go to the *Participant settings* tab, 43 | 2. Change the **default token length to 32** instead of 15 (by passing the length to 32, this will allow us to create an invitation token based on the `uuid` of the user), 44 | 3. Activate *Enable token-base response persistence*, 45 | 4. Optionally, if you want to allow the user to change his response you can enable *Allow multiple responses or update responses with one tokens*, 46 | 5. After the creation of the survey, into the settings of the survey go into *Survey menu -> Survey participants* and click on "Initialise participant table". 47 | 48 | After customising your survey, you can now send it to the user ! 49 | To send a survey, you can check [Pushing Surveys from the Server to the Phone](pushing_surveys_from_the_server_to_the_phone.md). However, use `limesurvey.sample` into the `emission/net/ext_service/push/sample.specs/push` as example. The *survey id* (`sid`) can be found either at the end of the survey url or in the LimeSurvey interface in *Survey settings -> Overview*. 50 | 51 | ```json 52 | { 53 | "alert_type": "survey", 54 | "title": "Did you rideshare ?", 55 | "message": "3 questions - Approx. 1 min", 56 | "image": "icon", 57 | "force-start": 1, 58 | "spec": { 59 | "url": "https://fabmob.limequery.com/964722", 60 | "uuidElementId": "answer886519X3X3", 61 | "type": "limesurvey", 62 | "sid": "964722" 63 | } 64 | } 65 | ``` 66 | > **WARNING**: Before sending the notification to users, do not forget to **Activate** the survey by clicking on `Activate this survey`. 67 | 68 | 69 | -------------------------------------------------------------------------------- /docs/manage/pushing_surveys_from_the_server_to_the_phone.md: -------------------------------------------------------------------------------- 1 | # Pushing Surveys from the Server to the Phone 2 | --- 3 | 4 | In addition to periodic surveys that are generated through local processing on the phone (https://github.com/e-mission/e-mission-phone/wiki/How-to-embed-an-external-survey-in-the-app), the server can also push surveys to the phone. These pushes can be periodic (e.g. every `x` hours, at certain times, e.g. at 21:00 UTC every day, or based on travel patterns from users). 5 | 6 | The basic components of a server-generated survey push are: 7 | 1. the selection of users who receive the survey 8 | 2. the survey 9 | 10 | Once the selection and the survey specs have been determined, the survey can be pushed to users using `bin/push/send_survey.py` 11 | 12 | ### Selection ### 13 | The `bin/push/send_survey.py` help message lists the ways to select users. In addition to the standard email and uuid options, you can also specify a `query_spec` that identifies users based on their travel patterns. The existing codebase includes three sample queries (`emission/net/ext_service/push/sample.specs/query`): 14 | 15 | 1. `platform`: which matches queries for a particular platform (`android` v/s `ios`) 16 | 2. `point_count`: which counts the number of times a particular user has traversed a georegion 17 | 3. `trip_metrics`: which uses trip related information such as distance, count and duration 18 | 19 | The code corresponding to these specs is at `emission/net/ext_service/push/query` - we welcome contributions of other query specs that you have found useful. 20 | 21 | ### Survey ### 22 | The survey spec is used to specify the survey to push. The spec is similar to the one used on the phone (https://github.com/e-mission/e-mission-docs/blob/master/docs/e-mission-phone/how_to_embed_an_external_survey_in_the_app.md) although a fun fact is that this came first :) Again, you specify the URL of the survey and the field that the uuid is populated into. 23 | As before, the examples are at `emission/net/ext_service/push/sample.specs/push` 24 | -------------------------------------------------------------------------------- /docs/manage/requesting_data_as_a_collaborator.md: -------------------------------------------------------------------------------- 1 | # Requesting Data as a Collaborator 2 | --- 3 | 4 | The consent document for e-mission (https://e-mission.eecs.berkeley.edu/consent) allows the platform owner (@shankari in this case) to share **de-linked** raw data with other collaborators for research. 5 | 6 | > Time-delayed subsets of individual trajectory data, associated with their UUIDs but not email addresses, may by shared with collaborators, or released as research datasets to the community from time to time. If this is done, the time delay for sharing with collaborators will be at least one month, and the time delay for releasing to the community will be at least one year. Both collaborators and researchers will be asked to agree that they will publish only aggregate, non personally identifiable results, and will not re-share the data with others. 7 | 8 | It also allows other researchers to use it to conduct studies. In this case, all data, including the **link** between the email address and the UUID will be made available to the researcher. 9 | 10 | > If this platform is being used to collect data for a study conducted by another researcher, for example, from a Transportation Engineering Department, then you will be asked to assent to a separate document outlining the data association, retention and sharing policies for that study, **in addition to the policies above**. We will make all data, including the mapping between the email address and the UUID, directly available to the lead researcher for the main study. This will allow them to associate the automatically gathered information with demographic data, and any pre and post surveys that they conduct as part of their study. The other researcher may also choose to compensate you for your time, as described in the protocol document for that study. 11 | 12 | This document provides the procedure to request access to such kinds of data. Most of the procedure is common; differences between them are labelled **linked** and **de-linked**. 13 | 14 | ## Setup GPG ## 15 | 16 | We will send and receive data encrypted/signed using GPG. 17 | 1. The steps for creating a GPG keypair are at https://www.gnupg.org/gph/en/manual/c14.html. 18 | 1. Create a keypair and export it. 19 | 1. Send me (@shankari, shankari@eecs.berkeley.edu) the public key via email. 20 | 21 | ## Data request ## 22 | 23 | ### De-linked ### 24 | Next, you need to formally request access by filling out a pdf form. 25 | 26 | 1. I will send you an encrypted version of the form you need to fill out and a copy of *my* public key. 27 | 1. Decrypt it using https://www.gnupg.org/gph/en/manual/x110.html. 28 | 1. Fill it out and sign it physically. 29 | 1. Also sign it electronically https://www.gnupg.org/gph/en/manual/x135.html 30 | 1. Encrypt it using my public key https://www.gnupg.org/gph/en/manual/x110.html and send it to me 31 | 32 | If all of this works, we know that we have bi-directional encrypted communication over email. Make sure to encrypt any privacy sensitive information (e.g. subsets of data for debugging) that you send to me in the future. 33 | 34 | ### Linked ### 35 | You need to send me a copy of your IRB approval and your consent document to ensure that you have permission to collect data. 36 | 37 | ## Data retrieval ## 38 | 39 | ### De-linked ### 40 | 1. As you can see from the consent document, you can get access to data that is time-delayed by 1 months. 41 | 1. I will upload an encrypted zip file with ~ 3 months of data to google drive and send you a link. 42 | 43 | Note that this data is very privacy-sensitive, so think through the answers carefully on the request form carefully and make sure that you follow them. Treat the data as you would like your data to be treated. 44 | 45 | ### Linked ### 46 | 1. I will upload an encrypted zip file with all your data to google drive and send you a link. 47 | 48 | 49 | ### Both ### 50 | 1. You need to decrypt it just like you decrypted the pdf form https://www.gnupg.org/gph/en/manual/x110.html. 51 | 1. When unzipped, the data consists of multiple json files, one per user. 52 | 1. The data will typically contain both raw sensed data (e.g. `background/location`) and processed data (e.g. `analysis/cleaned_trip`) 53 | 1. Data formats for the json objects are at `emission/core/wrapper` (e.g. `emission/core/wrapper/location.py` and `emission/core/wrapper/cleanedtrip.py`) 54 | 55 | ## Data analysis ## 56 | 57 | While it is possible to analyse the raw data, it is large, so you may want to load it into a database to work with. That will also allow you to write code that is compatible with the server, so that we can more easily incorporate your analysis into the standard e-mission server. 58 | 59 | ### Install the server ### 60 | Follow the README and install e-mission server locally on your own laptop. 61 | 62 | ### Load the data ### 63 | Load the data into your local database. Since this data contains information from mutiple users, and you presumably want to retain the uuids, to correlate with other surveys that you might have performed, you should use the `load_multi_timeline_for_range.py` script. Since there are multiple files, the timeline will typically be a directory, and you should pass in the prefix. For example, if the user files are `all_users_sep_dec_2016/dump_0109c47b-e640-411e-8d19-e481c52d7130`, `all_users_sep_dec_2016/dump_026f8d13-4d7a-4f8f-8d35-0ec22b0f8f8b, ...,` you should run the following command line. 64 | 65 | ``` 66 | ./e-mission-py.bash bin/debug/load_multi_timeline_for_range.py all_users_sep_dec_2016/dump_ 67 | ``` 68 | 69 | #### Linked data ONLY #### 70 | You can load the email <-> UUID mapping also into the database by using the `-m MAPFILE` option (see below). In this case, the mapping will be loaded into the `get_uuid_db()` similar to the regular server. If you don't specify the `-m` option, then the users will be loaded with dummy names - `user001`, `user002` ... - and you can manipulate the mapping json file directly to find out their email addresses. 71 | 72 | ``` 73 | ./e-mission-py.bash bin/debug/load_multi_timeline_for_range.py -m all_users.json all_users_sep_dec_2016/dump_ 74 | ``` 75 | 76 | ### Run the pipeline ### 77 | As discussed earlier, downloaded timelines typically contain analysis results, so there is no need to re-run the intake pipeline. The script will automatically detect this and print the following line. 78 | 79 | ``` 80 | timeline contains analysis results, no need to run the intake pipeline 81 | ``` 82 | 83 | If they do not contain analysis results, the script will print the line 84 | 85 | ``` 86 | all entries in the timeline contain only raw data, need to run the intake pipeline 87 | ``` 88 | 89 | In that case, you should [run the intake pipeline](https://github.com/e-mission/e-mission-docs/blob/master/docs/e-mission-server/deploying_your_own_server_to_production.md#the-analysis-pipeline) (`bin/intake_multiprocess.py`) to get the processed analysis results. 90 | 91 | ### Clean up the loaded data ### 92 | You can also remove the data by using `bin/purge_database_json.py`, which will delete everything, or `bin/debug/purge_multi_timeline_for_range.py`, which will only delete the entries in that timeline - e.g. 93 | 94 | ``` 95 | ./e-mission-py.bash bin/debug/purge_multi_timeline_for_range.py all_users_sep_dec_2016 96 | ``` 97 | 98 | ### Play with the data ### 99 | 100 | An example ipython notebook that shows data access parameters is at 101 | https://github.com/e-mission/e-mission-server/blob/master/Timeseries_Sample.ipynb 102 | 103 | It has examples on how to access raw data, processed data, and plot points. 104 | Please use the timeseries interfaces as opposed to direct mongodb queries wherever possible. 105 | That will make it easier to migrate to other, more scalable timeseries later. 106 | 107 | Again, data formats are at 108 | https://github.com/e-mission/e-mission-server/tree/master/emission/core/wrapper 109 | 110 | Let me (@shankari) know if you have any further questions... 111 | -------------------------------------------------------------------------------- /docs/project_committee.md: -------------------------------------------------------------------------------- 1 | The members of the e-mission project committee are drawn from a diverse set of backgrounds, institutional affiliations and geographic regions. The current list, in alphabetical order by last name, is below. If you would like to be on the committee and help shape the direction of the project, please email shankari@eecs.berkeley.edu 2 | 3 | 4 | | Name | Affiliation | Sector | Country | 5 | |--------------|------------------|-------------|-----------| 6 | | Eleftherios Avramidis | DFKI | Academia | Germany | 7 | | Anat Caspi | University of Washington | Academia/Nonprofit | USA | 8 | | Kyle Fitzsimmons | Concordia | Academia | Canada | 9 | | Jonathan Fürst | NEC Labs Europe | Industry | Germany | 10 | | Patrick Gendre | FabMob | Nonprofit | France | 11 | | Stephanie Huerre | Design consultant | Industry | Spain | 12 | | Vaiju Khode | The Red Maple | Small business | USA | 13 | | Gabriel Plassat | ADEME/FabMob| Government | France | 14 | | Zachary Patterson | Concordia | Academia | Canada | 15 | | K. Shankari | UC Berkeley | Academia | USA | 16 | | Amarin Siripanich | UNSW Sydney | Academia | Australia | 17 | -------------------------------------------------------------------------------- /docs/use/data_monitoring.md: -------------------------------------------------------------------------------- 1 | # How to monitor data from an OpenPATH deployment 2 | 3 | ## Public dashboard 4 | For basic summaries of the data, the public dashboard will be the easiest access point. The public dashboard is accessible to anyone on the internet, and contains a wide set of data visualizations updated daily. To view the dashboard, navigate to https://[your-deployment]-openpath.nrel.gov/public/. To remove charts from the display click the "x". To add charts, select the chart and month from the menu then click "add". Charts can be resized by dragging the lower right corner. Click the "more info" button where present to view the data shown in the chart in a table. Charts can be saved by right clicking and copying or saving the image to your local computer. 5 | 6 | ## Admin dashboard 7 | For more detailed representations and data export, the admin dashboard is provided to program administrators. You should have received and email inviting you to activate your account when the deployment was initialized, and you will be required to use two factor authentication. The dashboard may take a while to load, depending on the size of the deployment, but we are actively working on optimizations. 8 | There are a few sections in the administrator dashboard: 9 | - landing page: shows recent data regarding onboarding and active users 10 | - data: page which supports viewing and export of raw data, split into several tabs: 11 | - users: shows user data, including first and most recent trip date, number of trips, and number of labeled trips - very useful for monitoring progress and participation 12 | - trips: shows trip data, including distance, duration, and user labels - useful for trip analysis (see notes on data model) 13 | - trajectories: shows segments of trips - useful for highly detailed analysis including routes (see notes on data model) 14 | - heat maps: shows areas of frequent travel on a map 15 | 16 | ## Manually 17 | Data exported from the admin dashboard or from an individual's phone (json dump) can be analyzed independently. Feel free to look through the public dashboard repo for examples of working with the csvs from the admin dashboard, though there is not yet formalized support for this process. 18 | 19 | ## Notes on data model 20 | Below are a few notes on the data model highlighting the meaning and purpose of columns from frequently asked questions. 21 | 22 | What are the different modes? 23 | Note that user provided modes are likely to be the most accurate, but other mode types can be useful in filling data gaps. 24 | - sensed mode: algorithmic mode determination based on data from phone sensors, includes walk, bike, in vehicle, and air/hsr 25 | - predicted mode: algorithmic mode determination based on data from phone sensors and map matching, includes more specific vehicles such as bus, car, and train 26 | - ble sensed mode: only applies to fleet deployments which have configured BLE beacons and shows the name associated with the beacon detected for the trip 27 | - inferred mode/purpose: predicted mode and purpose based on an individual user's previous behavior (ie the user usually labels this location and speed as bike to work, so infer bike to work) 28 | - confirmed mode/purpose: mode and purpose provided by the user -------------------------------------------------------------------------------- /docs/use/deployments.md: -------------------------------------------------------------------------------- 1 | Organization | Region | Primary contact | on committee | Use case summary | 2 | -------------|--------|-----------------|--------------|------------------| 3 | UC Berkeley (RISE) | Public facing app | K. Shankari | Y | kitchen sink showcase app | 4 | Center for Community Innovation (CCI) | Richmond, USA | Miriam Zuk | N | Travel survey for low-income households | 5 | Taskar Center for Accessible Technology (TCAT) | Seattle, USA | Anat Caspi | Y | Crowdsourcing accessibility barriers | 6 | UC Berkeley (TripAware) | SF Bay Area, USA | Jesse Zhang | N | Motivating travel behavior change | 7 | UC Berkeley (Urban Planning) | Asuncion, Paraguay | Megumi Yamanaka | N | Capturing travel behavior before an infrastructure change | 8 | Indian Institute of Science (IISc) | Bangalore, India | Abdul Rawoof Pinjari | N | Travel survey for commute patterns | 9 | University of New South Wales (UNSW) | Sydney, Australia | Taha Rashidi | N | Travel survey in the Sydney area | 10 | Institute for Applied Sustainability to the Built Environment | Switzerland | Francesca Cellina | N | Automatic mobility tracking (planned) | 11 | Lawrence Berkeley National Laboratory | Berkeley, USA | C. Anna Spurlock | N | Unknown (custom version) | 12 | La Fabrique des Mobilités | France | Patrick Gendre | Y | Pilot deployment to iron out bugs and understand barriers to deployment by transport/mobility authorities and operators | 13 | Open Source Labs | Berlin, Germany | Julia Ebert | Y | Carbon footprint calculation | 14 | City of Heidelberg | Heidelberg, Germany | Jonathan Fürst | Y | Data for urban planning (pilot) | 15 | -------------------------------------------------------------------------------- /docs/use/end_user.md: -------------------------------------------------------------------------------- 1 | # USE THE SOFTWARE (for the end-user) 2 | ## how to install and use the app 3 | See the web site : [https://e-mission.eecs.berkeley.edu/](https://e-mission.eecs.berkeley.edu/) 4 | Also, there is an [app doc in French](https://docs.google.com/document/d/1X_FwiXjmWEFCLNhEXNa3-cD0FCjOURlLClCUiUoQ6PM/) 5 | 6 | ## use cases 7 | - list of project reusing e-mission (to be added here) 8 | - running instances (to be added here) 9 | 10 | -------------------------------------------------------------------------------- /docs/use/start_a_project.md: -------------------------------------------------------------------------------- 1 | # Getting Started with an NREL Hosted Deployment of OpenPATH 2 | 3 | The first step is to contact NREL to draw up an agreement, if you have not contacted us yet please reach out to openpath@nrel.gov to get started. We look forward to hearing about how you plan to use OpenPATH! 4 | 5 | ## Customizing Your Project 6 | 7 | ### Fill Out the Form 8 | 9 | In order to provide you with a deployment suited to your project's specific needs, we use a special JSON file to configure your app. To create that file, please fill out the form: [HERE](https://github.com/e-mission/nrel-openpath-deploy-configs/issues/new?assignees=&labels=new+config&projects=&template=add-new-config.yml&title=New+Project+Configuration+-+%5BPROJECT%5D). Please follow all instructions carefully, as this allows the automated workflow to function as intended and your file to be ready faster! 10 | 11 | You will be asked to provide information such as the name of your project, the purpose behind it, project-specific translations, and preferences for app configurations like the onboarding survey, trip labeling, and reminder schemes. It might help to gather the information before beginning to fill out the form. 12 | 13 | ### View Issue and Workflow 14 | 15 | After you fill out the form and click submit, you will see a page with a GitHub Issue containing the information that you input into the form. Please save the link to this page, we may need you to provide it to us. 16 | 17 | If everything went well, you should see a message at the bottom of the page saying that a bot referenced this issue - this means your config file has been created sucessfully! 18 | 19 | To see the generated pull request, click on "Pull requests" in the toolbar at the top of the page and find the one with the same number as the issue you opened in the title. This Pull Request will add the configuration for your project to our repo. If you're curious, you can see the generated file by clicking "Files changed" in the top of the pull request. 20 | 21 | ## Approval and Merge 22 | 23 | Once the file is ready, a developer will check it over and if everything looks good will merge the new file into the repository. 24 | 25 | Please note that any additonal files, like custom label options or surveys, will also need to be submitted before we can get your project completly ready for testing. 26 | 27 | ## Something Went Wrong 28 | 29 | If you do not see a message at the end of your issue form from a bot within a few minutes of clicking submit, something probably went wrong with the automatic file generation process. 30 | 31 | Don't worry! You can edit the issue to fix the issue. The most likely cause is that you did not follow the instructions on the form. To learn more about the problem, click "actions" in the GitHub toolbar at the top of the page, and then click on the one with [Your Project Name], if it has a red X to the left of the name, there was a problem. Click on the Action and look for an idication of the error, this may help you know what to edit in your issue to fix the problem. 32 | 33 | A green check next to the workflow means everything went smoothly, if you still don't see the bot message please contact us with the link to your issue and we will investigate. 34 | 35 | ## How do I Edit After Submitting? 36 | 37 | To edit your form after you submit, from the issue page that showed up when you clicked submit, click the 3 dots in the right hand corner of the box containing your form, then click edit. 38 | 39 | The box with your form will now be a text editor. Make updates as needed, ensuring that you keep the formatting intact. The rows starting with "###" are labels from the form, do not change them. 40 | 41 | When you're done, click "Update Comment" at the bottom right of the box. 42 | 43 | ## Additional Files? 44 | 45 | If you are choosing a more highly custom route, you might have to provide us with additonal files. Please use the files linked in the descriptions of those fields on the form as examples. Since this is slightly more complex than relying on the default options, there are some manual steps involved. You can submit the files using a GitHub pull request. If you are new to GitHub, this is a good [guide on pull requests ](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request). If you do create pull requests for additional files, it would help us out if you could link them in a comment in your issue. 46 | 47 | ### Custom Onboarding Survey 48 | 49 | We provide a default onboarding survey, but if your project needs specialized demographic information, you can write your own survey. The best way to do this is to use KoboToolbox to create the survey, then download the files and submit a pull request to add them to our repository. 50 | 51 | ### Custom Trip Labels 52 | 53 | We have a default set of trip mode and purpose labels, but you can also choose to provide your own. Please create a file like the example for a [program](https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-program-label-options.json) or a [study](https://github.com/e-mission/nrel-openpath-deploy-configs/blob/main/label_options/example-study-label-options.json) and submit it as a pull request. 54 | 55 | Modes are somewhat complex. The `value` is what's used behind the scenes. For the `baseMode`, you must choose from [those in our code base](https://github.com/e-mission/e-mission-phone/blob/922a62b7c2601f195bfe8df54654986135e99b25/www/js/diary/diaryHelper.ts#L20), these control the icon and color associated with the mode. The `met` or `met_equivalent` relates to the "metabolic equivalent of task" for each mode, you can specify a custom value, or choose an equivalent from [WALKING, BICYCLING, UNKNOWN, or IN_VEHICLE]. Make sure you specify all needed translations, failing to do so could result in poor UI in your version of the app. 56 | 57 | ### Custom Trip Surveys 58 | 59 | If your data-gathering needs extend beyond travel mode and purpose, you can ask participants to fill out a survey for each trip. Like the onboarding survey, it is best if you create this with KoboToolbox and submit the files with a pull request. If you do this, the dashboard screen will not be avaliable to participants in the app, as it relies too heavily on traditional labels to be insightful without them. 60 | 61 | ## Custom Reminder Schemes 62 | 63 | If you want to customize the schedule on which notificiations are sent to participants to open the app and label their trips, or what those notifications say, please indicate your preferences in the appropriate field on the form and a developer will translate that into the appropriate JSON in your config. -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: e-mission Docs 2 | 3 | nav: 4 | - Home: 'README.md' 5 | - Issues: 'https://github.com/e-mission/e-mission-docs/issues' 6 | - 1. Install: 7 | - '1. Manual install': 'install/manual_install.md' 8 | - '2. Deploying your own server to production': 'install/deploying_your_own_server_to_production.md' 9 | - '3. Configuring authentication': 'install/configuring_authentication.md' 10 | - '4. Multi-layer stack': 'install/multi_layer_stack.md' 11 | - 2. Use: 12 | - 'Use the app': 'use/end_user.md' 13 | - 3. Manage: 14 | - '1. Troubleshooting the app': 'manage/troubleshooting_app_faq.md' 15 | - '2. Troubleshooting the server': 'manage/troubleshooting_server_faq.md' 16 | - '3. Evaluation procedure': 'manage/evaluation_procedure.md' 17 | - '4. Requesting data as a collaborator': 'manage/requesting_data_as_a_collaborator.md' 18 | - '5. Pushing surveys from server to phone': 'manage/pushing_surveys_from_the_server_to_the_phone.md' 19 | - '5b. Adding LimeSurvey': 'manage/adding_limesurvey_support.md' 20 | - 4. Develop: 21 | - Tutorial: 22 | - 'Workshop intro': 'dev/tutorial/hands-on-overview.md' 23 | - 'Workshop fall 2018': 'dev/tutorial/logistics.md' 24 | - Small UI changes: 25 | - '1. Quickstart': 'dev/tutorial/small_ui_changes/quickstart.md' 26 | - '2. Easiest way to change text': 'dev/tutorial/small_ui_changes/easiest_way_to_change_text.md' 27 | - '3. Simple client changes': 'dev/tutorial/small_ui_changes/simple_client_changes.md' 28 | - '4. Summary and consent': 'dev/tutorial/small_ui_changes/summary_and_consent.md' 29 | - '5. Change trip end prompt': 'dev/tutorial/small_ui_changes/change_trip_end_prompt.md' 30 | - '6. Remove tabs': 'dev/tutorial/small_ui_changes/remove_tab.md' 31 | - '7. Changes needed when the set of tabs changes': 'dev/tutorial/small_ui_changes/changes_needed_when_the_set_of_tabs_changes.md' 32 | - Architecture: 33 | - '1. Data model': 'dev/archi/data_model.md' 34 | - '2. Module structure': 'dev/archi/module_structure.md' 35 | - '3. Mode inference pipeline': 'dev/archi/mode_inference_pipeline_design.md' 36 | - '4. Pipeline details': 'dev/archi/pipeline_details.md' 37 | - Front (Web/Mobile app): 38 | - '1. Phone module structure': 'dev/front/phone_module_structure.md' 39 | - '2. App UI dev FAQ': 'dev/front/high_level_faq.md' 40 | - '3. Create a new custom client': 'dev/front/create_a_new_custom_client.md' 41 | - '4. You have a branch for you custom client, now what?': 'dev/front/you_have_a_branch_for_your_custom_client_now_what.md' 42 | - '5a. Guidelines to i18n ': 'dev/front/guidelines_to_support_i18n.md' 43 | - '5b. How to add a language ': 'dev/front/how_to_add_a_language.md' 44 | - '6. How to embed an external survey in the app': 'dev/front/how_to_embed_an_external_survey_in_the_app.md' 45 | - '7. Accessing public heatmap and incident info': 'dev/front/accessing_public_heatmap_and_incident_information.md' 46 | - Back (Server): 47 | - '1. Adding a new data type': 'dev/back/adding_a_new_data_type.md' 48 | - '2. Supporting user inputs': 'dev/back/supporting_user_inputs.md' 49 | - Future: 50 | - '1. Overview': 'dev/future/overview.md' 51 | - '2. Privacy': 'dev/future/privacy.md' 52 | - '3. privacy litt. review': 'dev/future/differential_privacy_deep_dive_lit_review.md' 53 | - '4. Secure execution': 'dev/future/secure_execution_options.md' 54 | - '5. Analysis': 'dev/future/analysis.md' 55 | - '6. Scalability': 'dev/future/scalability.md' 56 | - '7. Unanswered questions': 'dev/future/unanswered_questions.md' 57 | - 5. Documenting: 58 | - 'Contributing': 'contribute_to_the_doc/CONTRIBUTING.md' 59 | - 'Docs reorganisation' : 'contribute_to_the_doc/migration_reason.md' 60 | 61 | markdown_extensions: 62 | - markdown.extensions.admonition 63 | - markdown.extensions.codehilite: 64 | guess_lang: true 65 | - markdown.extensions.def_list 66 | - markdown.extensions.footnotes 67 | - markdown.extensions.meta 68 | - markdown.extensions.toc: 69 | permalink: true 70 | - pymdownx.arithmatex 71 | - pymdownx.betterem: 72 | smart_enable: all 73 | - pymdownx.caret 74 | - pymdownx.critic 75 | - pymdownx.details 76 | - pymdownx.inlinehilite 77 | - pymdownx.keys 78 | - pymdownx.magiclink: 79 | repo_url_shorthand: true 80 | user: e-mission 81 | repo: e-mission-docs 82 | - pymdownx.mark 83 | - pymdownx.smartsymbols 84 | - pymdownx.superfences 85 | - pymdownx.tasklist: 86 | custom_checkbox: true 87 | - pymdownx.tilde -------------------------------------------------------------------------------- /readthedocs.yml: -------------------------------------------------------------------------------- 1 | python: 2 | install: requirements.txt 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs>=1 2 | Pygments>=2.2 3 | pymdown-extensions>=4.11 --------------------------------------------------------------------------------