Sorry, something broke when you tried to access this page. The web application has restarted, and no damage has
20 | been done. Please email hdawood@hsph.harvard.edu and describe what
21 | happened; if possible, please include the time that this page appeared, so that we can look at the error logs
22 | and figure out what broke.
23 |
24 |
25 | {% endblock %}
--------------------------------------------------------------------------------
/constants/security_constants.py:
--------------------------------------------------------------------------------
1 | import string
2 |
3 | ## Password Check Regexes
4 | SYMBOL_REGEX = "[^a-zA-Z0-9]"
5 | LOWERCASE_REGEX = "[a-z]"
6 | UPPERCASE_REGEX = "[A-Z]"
7 | NUMBER_REGEX = "[0-9]"
8 | PASSWORD_REQUIREMENT_REGEX_LIST = [SYMBOL_REGEX, LOWERCASE_REGEX, UPPERCASE_REGEX, NUMBER_REGEX]
9 |
10 | EASY_ALPHANUMERIC_CHARS = string.ascii_lowercase + '123456789' # intentionally does not have 0
11 |
12 | BASE64_GENERIC_ALLOWED_CHARACTERS = string .ascii_lowercase + string.ascii_uppercase + string.digits + "/+"
13 | OBJECT_ID_ALLOWED_CHARS = string.ascii_uppercase + string.ascii_lowercase + string.digits
14 |
15 | ASYMMETRIC_KEY_LENGTH = 2048 # length of private/public keys
16 |
17 | # this is a set of integers (bytes, technically), it is faster than testing bytes
18 | URLSAFE_BASE64_CHARACTERS = set(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-=")
19 |
20 | MFA_CREATED = "mfa_created"
--------------------------------------------------------------------------------
/manage.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | import multiprocessing
3 | import os
4 | import sys
5 |
6 |
7 | if __name__ == "__main__":
8 | try:
9 | command = sys.argv[1]
10 | except IndexError:
11 | command = "help"
12 |
13 | if command == "test" and sys.platform == "darwin":
14 | # Workaround for https://code.djangoproject.com/ticket/31169
15 | if os.environ.get("OBJC_DISABLE_INITIALIZE_FORK_SAFETY", "") != "YES":
16 | print("Set OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES in your" +
17 | " environment to work around use of forking in Django's" + " test runner.")
18 | sys.exit(1)
19 | multiprocessing.set_start_method("fork")
20 |
21 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django_settings")
22 | from django.core.management import execute_from_command_line
23 | execute_from_command_line(sys.argv)
24 |
--------------------------------------------------------------------------------
/database/migrations/0144_dataprocessingstatus.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.2.6 on 2025-09-18 22:20
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0143_researcher_bad_login_attempts_and_more'),
10 | ]
11 |
12 | operations = [
13 | migrations.CreateModel(
14 | name='DataProcessingStatus',
15 | fields=[
16 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
17 | ('created_on', models.DateTimeField(auto_now_add=True)),
18 | ('last_updated', models.DateTimeField(auto_now=True)),
19 | ('last_run', models.DateTimeField(blank=True, null=True)),
20 | ],
21 | options={
22 | 'abstract': False,
23 | },
24 | ),
25 | ]
26 |
--------------------------------------------------------------------------------
/database/migrations/0097_auto_20221220_0041.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.16 on 2022-12-20 00:41
2 |
3 | import django.core.validators
4 | from django.db import migrations, models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0096_alter_researcher_password_force_reset'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='researcher',
16 | name='password_min_length',
17 | field=models.SmallIntegerField(default=8, validators=[django.core.validators.MinValueValidator(8)]),
18 | ),
19 | migrations.AlterField(
20 | model_name='study',
21 | name='password_minimum_length',
22 | field=models.PositiveIntegerField(default=8, validators=[django.core.validators.MinValueValidator(8), django.core.validators.MaxValueValidator(20)]),
23 | ),
24 | ]
25 |
--------------------------------------------------------------------------------
/database/migrations/0040_fileastext.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 2.2.11 on 2020-09-29 18:24
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0039_auto_20200819_1901'),
10 | ]
11 |
12 | operations = [
13 | migrations.CreateModel(
14 | name='FileAsText',
15 | fields=[
16 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
17 | ('created_on', models.DateTimeField(auto_now_add=True)),
18 | ('last_updated', models.DateTimeField(auto_now=True)),
19 | ('tag', models.CharField(db_index=True, max_length=256)),
20 | ('text', models.TextField()),
21 | ],
22 | options={
23 | 'abstract': False,
24 | },
25 | ),
26 | ]
27 |
--------------------------------------------------------------------------------
/middleware/downtime_middleware.py:
--------------------------------------------------------------------------------
1 | from types import FunctionType
2 |
3 | from django.http.request import HttpRequest
4 | from django.http.response import HttpResponse
5 |
6 | from database.system_models import GlobalSettings
7 |
8 |
9 | class DowntimeMiddleware:
10 | """ A very quick and dirty downtime middleware. If downtime is enabled, it returns a 503. """
11 |
12 | def __init__(self, get_response: FunctionType):
13 | # just following the standard passthrough...
14 | self.get_response = get_response
15 |
16 | def __call__(self, request: HttpRequest):
17 | # if downtime is enabled, return a 503.
18 | if GlobalSettings.cached_downtime_enabled():
19 | return HttpResponse(
20 | content="This server is currently undergoing maintenance, please try again later.",
21 | status=503
22 | )
23 | return self.get_response(request)
24 |
--------------------------------------------------------------------------------
/database/migrations/0039_auto_20200819_1901.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 2.2.14 on 2020-08-19 19:01
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0038_auto_20200730_0413'),
10 | ]
11 |
12 | operations = [
13 | migrations.AlterField(
14 | model_name='archivedevent',
15 | name='response_time',
16 | field=models.DateTimeField(blank=True, db_index=True, null=True),
17 | ),
18 | migrations.AlterField(
19 | model_name='archivedevent',
20 | name='schedule_type',
21 | field=models.CharField(db_index=True, max_length=32),
22 | ),
23 | migrations.AlterField(
24 | model_name='archivedevent',
25 | name='scheduled_time',
26 | field=models.DateTimeField(db_index=True),
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/database/migrations/0090_auto_20221025_0621.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.15 on 2022-10-25 06:21
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0089_auto_20221021_0307'),
10 | ]
11 |
12 | operations = [
13 | migrations.AddField(
14 | model_name='participant',
15 | name='last_os_version',
16 | field=models.CharField(blank=True, max_length=32, null=True),
17 | ),
18 | migrations.AddField(
19 | model_name='participant',
20 | name='last_version_code',
21 | field=models.CharField(blank=True, max_length=32, null=True),
22 | ),
23 | migrations.AddField(
24 | model_name='participant',
25 | name='last_version_name',
26 | field=models.CharField(blank=True, max_length=32, null=True),
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/database/migrations/0116_remove_summarystatisticdaily_beiwe_image_survey_bytes.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 4.2.11 on 2024-03-12 13:09
2 |
3 | from django.db import migrations
4 |
5 |
6 | # image survey was a type that was never fully developed. Any instances of it are to be removed.
7 | def remove_image_survey_chunkregistry(apps, schema_editor):
8 | ChunkRegistry = apps.get_model('database', 'ChunkRegistry')
9 | ChunkRegistry.objects.filter(data_type='image_survey').delete()
10 |
11 |
12 | class Migration(migrations.Migration):
13 |
14 | dependencies = [
15 | ('database', '0115_participant_enable_extensive_device_info_tracking'),
16 | ]
17 |
18 | operations = [
19 | migrations.RunPython(remove_image_survey_chunkregistry, reverse_code=migrations.RunPython.noop),
20 | migrations.RemoveField(
21 | model_name='summarystatisticdaily',
22 | name='beiwe_image_survey_bytes',
23 | ),
24 | ]
25 |
--------------------------------------------------------------------------------
/database/migrations/0102_forestversion.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.16 on 2023-04-25 06:50
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0101_participantdeletionevent'),
10 | ]
11 |
12 | operations = [
13 | migrations.CreateModel(
14 | name='ForestVersion',
15 | fields=[
16 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
17 | ('created_on', models.DateTimeField(auto_now_add=True)),
18 | ('last_updated', models.DateTimeField(auto_now=True)),
19 | ('package_version', models.TextField(blank=True, default='')),
20 | ('git_commit', models.TextField(blank=True, default='')),
21 | ],
22 | options={
23 | 'abstract': False,
24 | },
25 | ),
26 | ]
27 |
--------------------------------------------------------------------------------
/database/migrations/0112_participantactionlog.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.23 on 2024-01-27 09:44
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0111_auto_20240118_2311'),
11 | ]
12 |
13 | operations = [
14 | migrations.CreateModel(
15 | name='ParticipantActionLog',
16 | fields=[
17 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
18 | ('timestamp', models.DateTimeField(db_index=True)),
19 | ('action', models.TextField()),
20 | ('participant', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='action_logs', to='database.participant')),
21 | ],
22 | options={
23 | 'abstract': False,
24 | },
25 | ),
26 | ]
27 |
--------------------------------------------------------------------------------
/database/migrations/0002_auto_20170923_1949.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # Generated by Django 1.11.5 on 2017-09-23 19:49
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0001_initial'),
10 | ]
11 |
12 | operations = [
13 | migrations.AlterField(
14 | model_name='survey',
15 | name='survey_type',
16 | field=models.CharField(choices=[('audio_survey', 'audio_survey'), ('tracking_survey', 'tracking_survey'), ('dummy', 'dummy')], help_text='What type of survey this is.', max_length=16),
17 | ),
18 | migrations.AlterField(
19 | model_name='surveyarchive',
20 | name='survey_type',
21 | field=models.CharField(choices=[('audio_survey', 'audio_survey'), ('tracking_survey', 'tracking_survey'), ('dummy', 'dummy')], help_text='What type of survey this is.', max_length=16),
22 | ),
23 | ]
24 |
--------------------------------------------------------------------------------
/database/migrations/0124_remove_apikey_has_tableau_api_permissions_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 4.2.11 on 2024-05-30 15:09
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0123_universal_api_keys'),
11 | ]
12 |
13 | operations = [
14 | migrations.RemoveField(
15 | model_name='apikey',
16 | name='has_tableau_api_permissions',
17 | ),
18 | migrations.AddField(
19 | model_name='apikey',
20 | name='last_used',
21 | field=models.DateTimeField(blank=True, null=True),
22 | ),
23 | migrations.AlterField(
24 | model_name='apikey',
25 | name='researcher',
26 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='api_keys', to='database.researcher'),
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/frontend/static/javascript/jqplot/plugins/jqplot.mobile.min.js:
--------------------------------------------------------------------------------
1 | /* jqPlot 1.0.8r1250 | (c) 2009-2013 Chris Leonello | jplot.com
2 | jsDate | (c) 2010-2013 Chris Leonello
3 | */(function(b){function a(e,d,c){this.bindCustomEvents=function(){this.eventCanvas._elem.bind("vclick",{plot:this},this.onClick);this.eventCanvas._elem.bind("dblclick",{plot:this},this.onDblClick);this.eventCanvas._elem.bind("taphold",{plot:this},this.onDblClick);this.eventCanvas._elem.bind("vmousedown",{plot:this},this.onMouseDown);this.eventCanvas._elem.bind("vmousemove",{plot:this},this.onMouseMove);this.eventCanvas._elem.bind("mouseenter",{plot:this},this.onMouseEnter);this.eventCanvas._elem.bind("mouseleave",{plot:this},this.onMouseLeave);if(this.captureRightClick){this.eventCanvas._elem.bind("vmouseup",{plot:this},this.onRightClick);this.eventCanvas._elem.get(0).oncontextmenu=function(){return false}}else{this.eventCanvas._elem.bind("vmouseup",{plot:this},this.onMouseUp)}};this.plugins.mobile=true}b.jqplot.postInitHooks.push(a)})(jQuery);
--------------------------------------------------------------------------------
/config/load_django.py:
--------------------------------------------------------------------------------
1 | import builtins
2 | import os
3 |
4 | import django
5 | from django.conf import settings
6 |
7 | import config.django_settings
8 |
9 |
10 | builtins = list(vars(builtins).keys())
11 |
12 | # When this file is imported we attempt to load django. If django is already loaded this action
13 | # throws a RuntimeError we catch that case and pass.
14 | try:
15 | # get all the variables declared in the django settings file, exclude builtins by _identity_.
16 | # Django 3.2 requires that all settings be upper case.
17 | django_config = {
18 | setting_name: setting_value
19 | for setting_name, setting_value in vars(config.django_settings).items()
20 | if setting_value not in builtins and setting_name.isupper()
21 | }
22 |
23 | # django setup file
24 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.django_settings")
25 | settings.configure(**django_config)
26 | django.setup()
27 |
28 | except RuntimeError as e:
29 | pass
30 |
--------------------------------------------------------------------------------
/scripts/repopulate_push_notifications.py:
--------------------------------------------------------------------------------
1 | from django.utils import timezone
2 |
3 | from database.study_models import Study
4 | from database.system_models import GlobalSettings
5 | from libs.schedules import repopulate_all_survey_scheduled_events
6 |
7 |
8 | def main():
9 | for study in Study.objects.all():
10 | repopulate_all_survey_scheduled_events(study)
11 |
12 | # This flag eneables resends. This has to be set _After_ schedules are repopulated
13 | # because... on servers where the participants have updated the app before the server has
14 | # been updated will possibly be in a position where they receive a resend of every archived
15 | # event within that period.
16 |
17 | # - repopulating logic may generate events that were missed (because of old bugs)
18 | settings = GlobalSettings().singleton()
19 | if settings.push_notification_resend_enabled is None:
20 | settings.push_notification_resend_enabled = timezone.now()
21 | settings.save()
22 |
--------------------------------------------------------------------------------
/database/migrations/0078_genericevent.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.13 on 2022-05-06 09:44
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0077_delete_fileprocesslock'),
10 | ]
11 |
12 | operations = [
13 | migrations.CreateModel(
14 | name='GenericEvent',
15 | fields=[
16 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
17 | ('created_on', models.DateTimeField(auto_now_add=True)),
18 | ('last_updated', models.DateTimeField(auto_now=True)),
19 | ('tag', models.CharField(db_index=True, max_length=256)),
20 | ('note', models.TextField()),
21 | ('stacktrace', models.TextField(blank=True, null=True)),
22 | ],
23 | options={
24 | 'abstract': False,
25 | },
26 | ),
27 | ]
28 |
--------------------------------------------------------------------------------
/constants/__init__.py:
--------------------------------------------------------------------------------
1 | # Todo: Once we upgrade to Django 3, use TextChoices(?)
2 | # we have this pattern used in a couple places, it is for choice fields.
3 | # intended for use only on classes with string values and nothing else.
4 | class DjangoDropdown:
5 |
6 | @classmethod
7 | def choices(cls) -> list[tuple[str, str]]:
8 | # this forces a dropdown to have specific choices of the same name as the attribute itself
9 | # with underscores -> spaces and title casing.
10 | return [(choice, choice.replace("_", " ").title()) for choice in cls.values()]
11 |
12 | @classmethod
13 | def values(cls) -> list[str]:
14 | # get the string VALUES of the attributes of the class for every attribute that does not start with an underscore
15 | ret = []
16 | for property_name, property_value in vars(cls).items():
17 | if not property_name.startswith("_") and isinstance(property_value, str):
18 | ret.append(property_value)
19 | return ret
20 |
--------------------------------------------------------------------------------
/database/migrations/0045_auto_20210121_0301.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 2.2.14 on 2021-01-21 03:01
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | # this migration used to use an external library called django-timezone-field it uses a charfield, it
7 | # didn't work, we got rid of it, this (should) now be manually patched out. That field type
8 | # should just be a CharField with a max_length of 63. The default value was 'America/New_York'.
9 |
10 |
11 | class Migration(migrations.Migration):
12 |
13 | dependencies = [
14 | ('database', '0044_auto_20210115_2300'),
15 | ]
16 |
17 | operations = [
18 | migrations.AddField(
19 | model_name='participant',
20 | name='push_notification_unreachable',
21 | field=models.SmallIntegerField(default=True),
22 | ),
23 | migrations.AddField(
24 | model_name='participant',
25 | name='timezone',
26 | field=models.CharField(default='America/New_York'),
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/database/migrations/0076_auto_20220420_2045.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.12 on 2022-04-20 20:45
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0075_iosedecryptionkey'),
11 | ]
12 |
13 | operations = [
14 | migrations.RenameModel('IOSEDecryptionKey', 'IOSDecryptionKey'),
15 | migrations.AlterField(
16 | model_name='iosdecryptionkey',
17 | name='base64_encryption_key',
18 | field=models.CharField(max_length=24),
19 | ),
20 | migrations.AlterField(
21 | model_name='iosdecryptionkey',
22 | name='s3_file_path',
23 | field=models.CharField(db_index=True, max_length=80, unique=True),
24 | ),
25 | migrations.RenameField(
26 | model_name='iosdecryptionkey',
27 | old_name='s3_file_path',
28 | new_name='file_name',
29 | ),
30 | ]
31 |
--------------------------------------------------------------------------------
/frontend/templates/notifications.html:
--------------------------------------------------------------------------------
1 | {% for message in messages %}
2 | {# {% for message in [ {"message": "success", "tags": "success"}, {"message": "info", "tags": "info"}, {"message": "warning", "tags": "warning"}, {"message": "error", "tags": "error"}, ] %} #}
3 | {% if message.tags != "error" %}
4 |
5 | {% else %}
6 |
7 | {% endif %}
8 |
9 | {{ message }}
10 |
11 | {% endfor %}
12 |
13 | {# don't render if this is the login page #}
14 | {% if not is_login_page %}
15 |
21 | You will be Logged out in 90 seconds due to inactivity.
22 |
23 | {% endif %}
24 |
--------------------------------------------------------------------------------
/database/migrations/0135_remove_globalsettings_earliest_possible_time_of_push_notification_resend_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 4.2.15 on 2024-12-18 15:20
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0134_devicesettings_resend_period_minutes'),
10 | ]
11 |
12 | operations = [
13 | # the original field was somewhat incorrect due to a release timing issue, any servers
14 | # running an older version have to be migrated manually to preserve correct time, but that
15 | # should just be our staging server.
16 | migrations.RemoveField(
17 | model_name='globalsettings',
18 | name='earliest_possible_time_of_push_notification_resend',
19 | ),
20 | migrations.AddField(
21 | model_name='globalsettings',
22 | name='push_notification_resend_enabled',
23 | field=models.DateTimeField(default=None, null=True, blank=True),
24 | ),
25 | ]
26 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/mano-bug-report.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Mano Bug Report
3 | about: Describe this issue template's purpose here.
4 | title: ''
5 | labels: Bug, Mano
6 | assignees: ''
7 |
8 | ---
9 |
10 | _(Note that we may change the name you assign to this issue)_
11 |
12 | ### Please fill out the following 3 items:
13 |
14 | #### I am experiencing this issue on the following operating system (include all that you have checked on, _particularly if it works on one but not on another!_):
15 | - macOS
16 | - Wandows
17 | - Linux
18 | - Wandows Subsystem for Linux (WSL)
19 |
20 | #### I am using Mano version ___________.
21 | - This can be found by running the `pip list` command and searching for Mano.
22 |
23 | #### If this issue includes an operation that sends a request to your Beiwe server, please describe steps you have taken to ensure this is not an issue with ... credentials, username, target server url, a down server, or revoked authorization to access your target study ... and things of that sort.
24 | -
25 |
26 |
27 | ### Here Begins My Report:
28 |
--------------------------------------------------------------------------------
/database/migrations/0006_auto_20180411_0453.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # Generated by Django 1.11.5 on 2018-04-11 04:53
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0005_add_admin_user'),
10 | ]
11 |
12 | operations = [
13 | migrations.AlterField(
14 | model_name='survey',
15 | name='survey_type',
16 | field=models.CharField(choices=[('audio_survey', 'audio_survey'), ('tracking_survey', 'tracking_survey'), ('dummy', 'dummy'), ('image_survey', 'image_survey')], help_text='What type of survey this is.', max_length=16),
17 | ),
18 | migrations.AlterField(
19 | model_name='surveyarchive',
20 | name='survey_type',
21 | field=models.CharField(choices=[('audio_survey', 'audio_survey'), ('tracking_survey', 'tracking_survey'), ('dummy', 'dummy'), ('image_survey', 'image_survey')], help_text='What type of survey this is.', max_length=16),
22 | ),
23 | ]
24 |
--------------------------------------------------------------------------------
/database/migrations/0035_auto_20200518_1908.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 2.2.11 on 2020-05-18 19:08
2 |
3 | from django.db import migrations
4 |
5 |
6 | def backfill_missing_survey_archives(apps, schema_editor):
7 | """ We need to fill SurveyArchive objects for all surveys predating its existence. """
8 | Survey = apps.get_model('database', 'Survey')
9 | for survey in Survey.objects.filter(archives__isnull=True):
10 | # SurveyArchives are created and tested in the save trigger.
11 | survey.save()
12 |
13 |
14 | class Migration(migrations.Migration):
15 |
16 | dependencies = [
17 | ('database', '0034_auto_20200506_2212'),
18 | ]
19 |
20 | operations = [
21 | migrations.RemoveField(
22 | model_name='surveyarchive',
23 | name='archive_end',
24 | ),
25 | migrations.RemoveField(
26 | model_name='surveyarchive',
27 | name='study',
28 | ),
29 | migrations.RunPython(backfill_missing_survey_archives, reverse_code=migrations.RunPython.noop),
30 | ]
31 |
--------------------------------------------------------------------------------
/database/migrations/0069_filetoprocess_os_type.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 2.2.25 on 2022-03-03 20:49
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | def populate_os_type(apps, schema_editor):
7 | # this does not need to be performant.
8 | FileToProcess = apps.get_model('database', 'FileToProcess')
9 | for ftp in FileToProcess.objects.all():
10 | FileToProcess.objects.filter(pk=ftp.pk).update(os_type=ftp.participant.os_type)
11 |
12 | class Migration(migrations.Migration):
13 |
14 | dependencies = [
15 | ('database', '0068_survey_name'),
16 | ]
17 |
18 | operations = [
19 | migrations.AddField(
20 | model_name='filetoprocess',
21 | name='os_type',
22 | field=models.CharField(
23 | blank=True,
24 | choices=[('IOS', 'IOS'), ('ANDROID', 'ANDROID'), ('', '')],
25 | default='',
26 | max_length=16
27 | ),
28 | ),
29 | migrations.RunPython(populate_os_type, reverse_code=migrations.RunPython.noop),
30 | ]
31 |
--------------------------------------------------------------------------------
/constants/copy_study_constants.py:
--------------------------------------------------------------------------------
1 | WEEKLY_SCHEDULE_KEY = "timings" # predates the other schedules
2 | ABSOLUTE_SCHEDULE_KEY = "absolute_timings"
3 | RELATIVE_SCHEDULE_KEY = "relative_timings"
4 | INTERVENTIONS_KEY = "interventions"
5 |
6 | # keys used if various places, pulled out into constants for consistency.
7 | DEVICE_SETTINGS_KEY = 'device_settings'
8 | STUDY_KEY = 'study'
9 | SURVEY_CONTENT_KEY = 'content'
10 | # SURVEY_SETTINGS_KEY = 'settings'
11 | SURVEYS_KEY = 'surveys'
12 | STUDY_TIME_ZONE_KEY = 'study_time_zone'
13 |
14 | # "_id" is legacy, pk shouldn't occur naturally. This list applies to Surveys and Studies.
15 | NEVER_EXPORT_THESE = ('_id', 'id', 'pk', 'created_on', 'last_updated', 'object_id', "deleted")
16 |
17 | # new stuff
18 | INTERVENTION__NAME = "intervention__name"
19 | survey_params = ("content", "survey_type", "settings")
20 | weekly_params = ("day_of_week", "hour", "minute")
21 | absolute_params = ("date", "hour", "minute")
22 | relative_params = (INTERVENTION__NAME, "days_after", "hour", "minute")
23 |
24 | BAD_DEVICE_SETTINGS_FIELDS = ("created_on", "last_updated", "study")
25 |
--------------------------------------------------------------------------------
/database/migrations/0120_devicesettings_heartbeat_message_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 4.2.11 on 2024-04-09 06:34
2 |
3 | import django.core.validators
4 | from django.db import migrations, models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0119_remove_appversionhistory_app_version_and_more'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='devicesettings',
16 | name='heartbeat_message',
17 | field=models.TextField(default='Beiwe may not be running correctly, please open the Beiwe app.'),
18 | ),
19 | migrations.AddField(
20 | model_name='devicesettings',
21 | name='heartbeat_timer_minutes',
22 | field=models.PositiveIntegerField(default=60, validators=[django.core.validators.MinValueValidator(1)]),
23 | ),
24 | migrations.AddField(
25 | model_name='participant',
26 | name='last_heartbeat_checkin',
27 | field=models.DateTimeField(blank=True, null=True),
28 | ),
29 | ]
30 |
--------------------------------------------------------------------------------
/database/migrations/0111_auto_20240118_2311.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.23 on 2024-01-18 23:11
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0110_auto_20240110_1809'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='participant',
16 | name='last_heartbeat_notification',
17 | field=models.DateTimeField(blank=True, null=True),
18 | ),
19 | migrations.CreateModel(
20 | name='AppHeartbeats',
21 | fields=[
22 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
23 | ('timestamp', models.DateTimeField(db_index=True)),
24 | ('participant', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='heartbeats', to='database.participant')),
25 | ],
26 | options={
27 | 'abstract': False,
28 | },
29 | ),
30 | ]
31 |
--------------------------------------------------------------------------------
/docker_management/nginx/prod.nginx.conf:
--------------------------------------------------------------------------------
1 | upstream beiwe-backend {
2 | server web:8000;
3 | }
4 |
5 | server {
6 | listen [::]:443 ssl ipv6only=on;
7 | listen 443 ssl;
8 |
9 | server_name my.domain;
10 |
11 | location / {
12 | proxy_pass http://beiwe-backend;
13 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
14 | proxy_set_header Host $host;
15 | proxy_redirect off;
16 | }
17 |
18 | location /static/ {
19 | autoindex on;
20 | alias /www/static/;
21 | }
22 |
23 | ssl_certificate /cert/live/my.domain/fullchain.pem;
24 | ssl_certificate_key /cert/live/my.domain/privkey.pem;
25 |
26 | # include /cert/options-ssl-nginx.conf;
27 | # ssl_dhparam /cert/ssl-dhparams.pem;
28 | }
29 |
30 | server {
31 | if ($host = my.domain) {
32 | return 301 https://$host$request_uri;
33 | } # managed by Certbot
34 |
35 | listen 80;
36 | listen [::]:80;
37 |
38 | server_name my.domain;
39 |
40 | return 404; # managed by Certbot
41 | }
42 |
--------------------------------------------------------------------------------
/frontend/static/javascript/app/survey-builder/services.js:
--------------------------------------------------------------------------------
1 | // Create an AngularJS module with a service and factory
2 | angular.module("surveyBuilder")
3 | // Create a service to generate unique IDs (e.g., UUIDs)
4 | .service("uuid", function() {
5 | this.generate = function() {
6 | var d = new Date().getTime();
7 | return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function (c) {
8 | var r = (d + Math.random() * 16) % 16 | 0;
9 | d = Math.floor(d / 16);
10 | return (c == "x" ? r : (r & 0x7 | 0x8)).toString(16); // converts to hexadecimal?
11 | });
12 | }
13 | })
14 |
15 | // Create a factory to provide logic for generating new paths
16 | .factory("logicService", function() {
17 | return {
18 | getNewPath: getNewPath
19 | };
20 |
21 | /** Generate a new path based on the provided path, type, and optional index.*/
22 | function getNewPath(base_path, type, index) {
23 | var newPath = base_path + "/" + type;
24 | if (typeof index != "undefined") {
25 | newPath = newPath + "/" + index;
26 | }
27 | return newPath;
28 | }
29 | });
30 |
--------------------------------------------------------------------------------
/database/migrations/0075_iosedecryptionkey.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.12 on 2022-04-18 04:57
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0074_auto_20220407_1840'),
11 | ]
12 |
13 | operations = [
14 | migrations.CreateModel(
15 | name='IOSEDecryptionKey',
16 | fields=[
17 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
18 | ('created_on', models.DateTimeField(auto_now_add=True)),
19 | ('last_updated', models.DateTimeField(auto_now=True)),
20 | ('s3_file_path', models.CharField(db_index=True, max_length=256, unique=True)),
21 | ('base64_encryption_key', models.CharField(max_length=256)),
22 | ('participant', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='database.participant')),
23 | ],
24 | options={
25 | 'abstract': False,
26 | },
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/frontend/static/javascript/app/survey-builder/directives/logical-block/logical-block.js:
--------------------------------------------------------------------------------
1 | angular.module("surveyBuilder")
2 | .directive("logicalBlock", function(ARITHMETIC_OPERATORS, LOGICAL_OPERATORS, _, logicService) {
3 | return {
4 | "link": function(scope) {
5 | /* Constants */
6 | scope._ = _;
7 | scope.ARITHMETIC_OPERATORS = ARITHMETIC_OPERATORS;
8 | scope.LOGICAL_OPERATORS = LOGICAL_OPERATORS;
9 |
10 | scope.$watch("surveyBuilder.currentQuestionFields.question_id", function() {
11 | scope.type = _.keys(scope.surveyBuilder.getValueAtPath(scope.path))[0];
12 | });
13 | scope.getNewPath = function(index) {
14 | return logicService.getNewPath(scope.path, scope.type, index);
15 | };
16 | },
17 | "restrict": "E",
18 | "scope": {
19 | "surveyBuilder": "=",
20 | "path": "@"
21 | },
22 | // need to add a fake variable to the end of the templateUrl to force the browser to reload the template
23 | "templateUrl": "/static/javascript/app/survey-builder/directives/logical-block/logical-block.html?n=1"
24 | };
25 | });
26 |
--------------------------------------------------------------------------------
/database/migrations/0098_researchersession.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.16 on 2022-12-21 13:32
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0097_auto_20221220_0041'),
11 | ]
12 |
13 | operations = [
14 | migrations.CreateModel(
15 | name='ResearcherSession',
16 | fields=[
17 | ('session_key', models.CharField(max_length=40, primary_key=True, serialize=False, verbose_name='session key')),
18 | ('session_data', models.TextField(verbose_name='session data')),
19 | ('expire_date', models.DateTimeField(db_index=True, verbose_name='expire date')),
20 | ('researcher', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='web_sessions', to='database.researcher')),
21 | ],
22 | options={
23 | 'verbose_name': 'session',
24 | 'verbose_name_plural': 'sessions',
25 | 'abstract': False,
26 | },
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/frontend/static/css/survey_builder.css:
--------------------------------------------------------------------------------
1 | th, td {
2 | padding: 5px;
3 | }
4 |
5 | .time-label {
6 | font-size: 1em;
7 | }
8 |
9 | .tablecell-align-top {
10 | vertical-align: top;
11 | }
12 |
13 | .add-logic-button {
14 | margin-top: 1px;
15 | margin-bottom: 2px;
16 | }
17 | @media (min-width: 768px) {
18 | .modal-dialog {
19 | width: 80%;
20 | }
21 | }.faded {
22 | opacity: 0.3;
23 | }
24 |
25 | .schedule-x { /* make the x very clickable and red */
26 | cursor: pointer;
27 | padding-top: 4pt;
28 | margin-top: -4pt;
29 | padding-bottom: 4pt;
30 | margin-bottom: -4pt;
31 | padding-right: 7pt;
32 | margin-right: -7pt;
33 | padding-left: 7pt;
34 | margin-left: -7pt;
35 | vertical-align:sub;
36 | font-size: large;
37 | color: var(--Bred-very-dark);
38 | text-shadow: 1px 1px 2px var(--Bblack-half);
39 | /* float: inline-end; */
40 | }
41 |
42 |
43 | .list-group-item {
44 | background-color: #222;
45 | color: var(--Bwhite);
46 | border-color: #222;
47 | }
48 | .list-group-item:hover {
49 | background-color: var(--Baqua-less-bright66);
50 | border-color: var(--Baqua-less-bright66);;
51 | }
52 |
--------------------------------------------------------------------------------
/database/migrations/0087_auto_20221013_1412.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.15 on 2022-10-13 14:12
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0086_alter_scheduledevent_most_recent_event'),
10 | ]
11 |
12 | operations = [
13 | migrations.AddField(
14 | model_name='devicesettings',
15 | name='ambient_audio_bitrate',
16 | field=models.PositiveIntegerField(default=24000),
17 | ),
18 | migrations.AddField(
19 | model_name='devicesettings',
20 | name='ambient_audio_off_duration_seconds',
21 | field=models.PositiveIntegerField(default=600),
22 | ),
23 | migrations.AddField(
24 | model_name='devicesettings',
25 | name='ambient_audio_on_duration_seconds',
26 | field=models.PositiveIntegerField(default=600),
27 | ),
28 | migrations.AddField(
29 | model_name='devicesettings',
30 | name='ambient_audio_sampling_rate',
31 | field=models.PositiveIntegerField(default=44100),
32 | ),
33 | ]
34 |
--------------------------------------------------------------------------------
/database/migrations/0114_devicestatusreporthistory.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.24 on 2024-02-28 09:04
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0113_appheartbeats_message'),
11 | ]
12 |
13 | operations = [
14 | migrations.CreateModel(
15 | name='DeviceStatusReportHistory',
16 | fields=[
17 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
18 | ('app_os', models.CharField(max_length=32)),
19 | ('os_version', models.CharField(max_length=32)),
20 | ('app_version', models.CharField(max_length=32)),
21 | ('endpoint', models.TextField()),
22 | ('compressed_report', models.BinaryField()),
23 | ('participant', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='device_status_reports', to='database.participant')),
24 | ],
25 | options={
26 | 'abstract': False,
27 | },
28 | ),
29 | ]
30 |
--------------------------------------------------------------------------------
/frontend/templates/dashboard/dashboard_stream_javascript.html:
--------------------------------------------------------------------------------
1 |
21 |
--------------------------------------------------------------------------------
/database/migrations/0056_auto_20210525_1920.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 2.2.19 on 2021-05-25 19:20
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0055_auto_20210524_2128'),
11 | ]
12 |
13 | operations = [
14 | migrations.AlterField(
15 | model_name='summarystatisticdaily',
16 | name='jasmine_task',
17 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='jasmine_summary_statistics', to='database.ForestTask'),
18 | ),
19 | migrations.AlterField(
20 | model_name='summarystatisticdaily',
21 | name='willow_task',
22 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='willow_summary_statistics', to='database.ForestTask'),
23 | ),
24 | migrations.AddConstraint(
25 | model_name='summarystatisticdaily',
26 | constraint=models.UniqueConstraint(fields=('date', 'participant'), name='unique_summary_statistic'),
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/database/migrations/0117_researcher_last_login_time_alter_survey_survey_type_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 4.2.11 on 2024-03-15 18:20
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0116_remove_summarystatisticdaily_beiwe_image_survey_bytes'),
10 | ]
11 |
12 | operations = [
13 | migrations.AddField(
14 | model_name='researcher',
15 | name='last_login_time',
16 | field=models.DateTimeField(blank=True, null=True),
17 | ),
18 | migrations.AlterField(
19 | model_name='survey',
20 | name='survey_type',
21 | field=models.CharField(choices=[('audio_survey', 'audio_survey'), ('tracking_survey', 'tracking_survey')], help_text='What type of survey this is.', max_length=16),
22 | ),
23 | migrations.AlterField(
24 | model_name='surveyarchive',
25 | name='survey_type',
26 | field=models.CharField(choices=[('audio_survey', 'audio_survey'), ('tracking_survey', 'tracking_survey')], help_text='What type of survey this is.', max_length=16),
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/cluster_management/pushed_files/cron_root.txt:
--------------------------------------------------------------------------------
1 | # Edit this file to introduce tasks to be run by cron.
2 | #
3 | # Each task to run has to be defined through a single line
4 | # indicating with different fields when the task will be run
5 | # and what command to run for the task
6 | #
7 | # To define the time you can provide concrete values for
8 | # minute (m), hour (h), day of month (dom), month (mon),
9 | # and day of week (dow) or use '*' in these fields (for 'any').
10 | #
11 | # Notice that tasks will be started based on the cron's system
12 | # daemon's notion of time and timezones.
13 | #
14 | # Output of the crontab jobs (including errors) is sent through
15 | # email to the user the crontab file belongs to (unless redirected).
16 | #
17 | # For example, you can run a backup of all your user accounts
18 | # at 5 a.m every week with:
19 | # 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
20 | #
21 | # For more information see the manual pages of crontab(5) and cron(8)
22 | #
23 | # m h dom mon dow command
24 |
25 | # restart supervisord once a day at 4:28am (utc?)
26 | 29 4 * * *: supervisorctl restart all
27 | 28 4 * * WED: export DEBIAN_FRONTEND=noninteractive; apt update; apt full-upgrade -yq; shutdown -r
28 |
--------------------------------------------------------------------------------
/database/migrations/0101_participantdeletionevent.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.16 on 2023-03-02 14:29
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0100_participant_last_get_latest_device_settings'),
11 | ]
12 |
13 | operations = [
14 | migrations.CreateModel(
15 | name='ParticipantDeletionEvent',
16 | fields=[
17 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
18 | ('created_on', models.DateTimeField(auto_now_add=True)),
19 | ('last_updated', models.DateTimeField(auto_now=True)),
20 | ('files_deleted_count', models.BigIntegerField(default=0)),
21 | ('purge_confirmed_time', models.DateTimeField(blank=True, db_index=True, null=True)),
22 | ('participant', models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, related_name='deletion_event', to='database.participant')),
23 | ],
24 | options={
25 | 'abstract': False,
26 | },
27 | ),
28 | ]
29 |
--------------------------------------------------------------------------------
/cluster_management/pushed_files/cron_worker.txt:
--------------------------------------------------------------------------------
1 | # Edit this file to introduce tasks to be run by cron.
2 | #
3 | # Each task to run has to be defined through a single line
4 | # indicating with different fields when the task will be run
5 | # and what command to run for the task
6 | #
7 | # To define the time you can provide concrete values for
8 | # minute (m), hour (h), day of month (dom), month (mon),
9 | # and day of week (dow) or use '*' in these fields (for 'any').#
10 | # Notice that tasks will be started based on the cron's system
11 | # daemon's notion of time and timezones.
12 | #
13 | # Output of the crontab jobs (including errors) is sent through
14 | # email to the user the crontab file belongs to (unless redirected).
15 | #
16 | # For example, you can run a backup of all your user accounts
17 | # at 5 a.m every week with:
18 | # 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
19 | #
20 | # For more information see the manual pages of crontab(5) and cron(8)
21 | #
22 | # m h dom mon dow command
23 |
24 | # see notes in cron_manager.txt
25 |
26 | @daily : daily; journalctl --vacuum-time=1w
27 | @daily : daily; cd $PROJECT_PATH; chronic /home/ubuntu/.pyenv/versions/beiwe/bin/python run_script.py upload_logs.py
28 | @daily : daily; apt autoclean
29 |
--------------------------------------------------------------------------------
/frontend/static/javascript/participants_table.js:
--------------------------------------------------------------------------------
1 | $(document).ready(function(){
2 | // Set up the main list of participants using DataTables
3 | $("#participantList").DataTable({
4 | "processing": true,
5 | "serverSide": true,
6 | "ajax": {
7 | url: "/study/" + studyId + "/get_participants_api/",
8 | type: 'POST' // Get request size ballooned past 4k characters, so use POST
9 | },
10 | "order": [1, 'asc'],
11 | "columnDefs": [
12 | // Format the second column (index 1) to be a link to the View Participant page
13 | {"targets": 2, "render": function(data, type, row, meta) {
14 | if(type === 'display') {
15 | data = '' + data + ''
16 | }
17 | return data;
18 | }},
19 | // You can sort the table on any of the first 4 columns
20 | // {"targets": [0, 1, 2, 3], "orderable": true},
21 | // You can now sort the table on any other columns (custom fields & intervention dates)
22 | {"targets": "_all", "orderable": true},
23 | ]
24 | });
25 | });
26 |
--------------------------------------------------------------------------------
/libs/utils/compression.py:
--------------------------------------------------------------------------------
1 | import pyzstd
2 |
3 | from config.settings import DATA_COMPRESSION_LEVEL
4 |
5 |
6 | # there has been substantial testing of the zstd compression modes for data produced by beiwe.
7 | # the pyzstd library implementation of zstd is the fastest one based on benchmarks.
8 |
9 | def compress(some_bytes: bytes, level: int = DATA_COMPRESSION_LEVEL) -> bytes:
10 | if level > 4:
11 | raise Exception(
12 | "the 'dfast' strategy does not work correctly with compression levels 0-4, see code in the compression_tests for more details."
13 | )
14 | return pyzstd.RichMemZstdCompressor( # type: ignore
15 | {
16 | pyzstd.CParameter.compressionLevel: level,
17 | pyzstd.CParameter.nbWorkers: -1,
18 | # positive integers may not actually result in 1 full core of utilization.
19 | # 0 and -1 seem to do some kind of auto-detection that keeps a core fully busy.
20 | # documentation of zstd generally implies that there is a limit applied to small files
21 | pyzstd.CParameter.strategy: pyzstd.Strategy.dfast,
22 | }
23 | ).compress(some_bytes)
24 |
25 |
26 | def decompress(input: bytes) -> bytes:
27 | return pyzstd.decompress(input)
28 |
--------------------------------------------------------------------------------
/database/migrations/0043_auto_20210111_2318.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 2.2.14 on 2021-01-11 23:18
2 |
3 | from django.db import migrations, models
4 | import django.utils.timezone
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0042_auto_20201204_2223'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='intervention',
16 | name='created_on',
17 | field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
18 | preserve_default=False,
19 | ),
20 | migrations.AddField(
21 | model_name='intervention',
22 | name='last_updated',
23 | field=models.DateTimeField(auto_now=True),
24 | ),
25 | migrations.AddField(
26 | model_name='interventiondate',
27 | name='created_on',
28 | field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now),
29 | preserve_default=False,
30 | ),
31 | migrations.AddField(
32 | model_name='interventiondate',
33 | name='last_updated',
34 | field=models.DateTimeField(auto_now=True),
35 | ),
36 | ]
37 |
--------------------------------------------------------------------------------
/database/migrations/0013_auto_20180530_0153.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # Generated by Django 1.11.5 on 2018-05-30 01:53
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0012_auto_20180525_0141'),
11 | ]
12 |
13 | operations = [
14 | migrations.RenameField(
15 | model_name='encryptionerrormetadata',
16 | old_name='errors_lines',
17 | new_name='error_lines',
18 | ),
19 | migrations.AddField(
20 | model_name='decryptionkeyerror',
21 | name='traceback',
22 | field=models.TextField(null=True),
23 | ),
24 | migrations.AddField(
25 | model_name='encryptionerrormetadata',
26 | name='participant',
27 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='database.Participant'),
28 | ),
29 | migrations.AddField(
30 | model_name='lineencryptionerror',
31 | name='participant',
32 | field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='database.Participant'),
33 | ),
34 | ]
35 |
--------------------------------------------------------------------------------
/database/migrations/0055_auto_20210524_2128.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 2.2.19 on 2021-05-24 21:28
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0054_auto_20210520_1931'),
11 | ]
12 |
13 | operations = [
14 | migrations.RemoveField(
15 | model_name='summarystatisticdaily',
16 | name='jasmine_tracker',
17 | ),
18 | migrations.RemoveField(
19 | model_name='summarystatisticdaily',
20 | name='willow_tracker',
21 | ),
22 | migrations.AddField(
23 | model_name='summarystatisticdaily',
24 | name='jasmine_task',
25 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='jasmine_summary_statistics', to='database.ForestParam'),
26 | ),
27 | migrations.AddField(
28 | model_name='summarystatisticdaily',
29 | name='willow_task',
30 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='willow_summary_statistics', to='database.ForestParam'),
31 | ),
32 | ]
33 |
--------------------------------------------------------------------------------
/libs/aes.py:
--------------------------------------------------------------------------------
1 | from os import urandom
2 |
3 | from Cryptodome.Cipher import AES
4 |
5 |
6 | # todo: use the buffer interface to make encryption/decryption faster/not use 2x memory (if possible)
7 |
8 |
9 | def encrypt_for_server(data: bytes, encryption_key: bytes) -> bytes:
10 | """ Encrypts config using the ENCRYPTION_KEY, prepends the generated initialization vector.
11 | Use this function on an entire file (as a bytes). """
12 | if not isinstance(encryption_key, bytes):
13 | raise Exception(f"received non-bytes object {type(encryption_key)}")
14 | if len(encryption_key) != 32:
15 | raise Exception(f"received encryption key with bad length: {len(encryption_key)}")
16 | iv: bytes = urandom(16) # bytes
17 | return iv + AES.new(encryption_key, AES.MODE_CFB, segment_size=8, IV=iv).encrypt(data)
18 |
19 |
20 | def decrypt_server(data: bytes, encryption_key: bytes) -> bytes:
21 | """ Decrypts config encrypted by the encrypt_for_server function. """
22 | if not isinstance(encryption_key, bytes):
23 | raise Exception(f"received non-bytes object {type(encryption_key)}")
24 | iv = data[:16]
25 | data = data[16:] # gr arg, memcopy operation...
26 | return AES.new(encryption_key, AES.MODE_CFB, segment_size=8, IV=iv).decrypt(data)
27 |
--------------------------------------------------------------------------------
/scripts/upload_logs.py:
--------------------------------------------------------------------------------
1 | from subprocess import check_output
2 |
3 | from django.utils import timezone
4 |
5 | from constants.common_constants import LOGS_FOLDER
6 | from database.models import S3File
7 | from libs.s3 import s3_upload_plaintext
8 | from libs.utils.compression import compress
9 |
10 |
11 | """
12 | This script runs periodically. The default system logrotate periodicity should be weekly, so as long
13 | as this script is run more frequently than that all log data in the auth log will be uploaded.
14 | """
15 |
16 | def main():
17 | with open("/var/log/auth.log", "rb") as f:
18 | auth_log = f.read()
19 |
20 | log_size_uncompressed = len(auth_log)
21 | auth_log = compress(auth_log) # compress the log file
22 | log_size_compressed = len(auth_log)
23 |
24 | now = timezone.now().isoformat()
25 | # should be something like ip-172-31-67-107
26 | hostname = check_output("hostname").strip().decode()
27 | path = f"{LOGS_FOLDER}/auth_log/{hostname}-{now}.log"
28 | # file name should sort by hostname then date
29 | s3_upload_plaintext(path, auth_log)
30 | print("uploaded auth_log")
31 |
32 | S3File.objects.create(
33 | path=path, size_uncompressed=log_size_uncompressed, size_compressed=log_size_compressed
34 | )
35 |
--------------------------------------------------------------------------------
/database/migrations/0049_auto_20210223_2005.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 2.2.15 on 2021-02-23 20:05
2 | from collections import Counter
3 |
4 | from django.db import migrations, models
5 |
6 |
7 | def purge_absolute_schedules(apps, schema_editor):
8 | """ Removes all but one entry of any duplicates in FileToProcess. """
9 | FileToProcess = apps.get_model('database', 'FileToProcess')
10 | files = Counter(FileToProcess.objects.values_list("s3_file_path", flat=True))
11 |
12 | for path, count in files.items():
13 | if count > 1:
14 | # get ids, remove the "first", delete all that remain.
15 | pks = list(FileToProcess.objects.filter(s3_file_path=path).values_list("pk", flat=True))
16 | pks.pop(0)
17 | FileToProcess.objects.filter(pk__in=pks).delete()
18 |
19 |
20 | class Migration(migrations.Migration):
21 |
22 | dependencies = [
23 | ('database', '0048_auto_20210212_1946'),
24 | ]
25 |
26 | operations = [
27 | migrations.RunPython(purge_absolute_schedules, migrations.RunPython.noop),
28 | migrations.AlterField(
29 | model_name='filetoprocess',
30 | name='s3_file_path',
31 | field=models.CharField(max_length=256, unique=True),
32 | ),
33 | ]
34 |
--------------------------------------------------------------------------------
/database/migrations/0109_auto_20231206_1116.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.23 on 2023-12-06 11:16
2 |
3 | import django.core.validators
4 | from django.db import migrations, models
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0108_auto_20231128_0142'),
11 | ]
12 |
13 | operations = [
14 | migrations.RenameField(
15 | model_name='participant',
16 | old_name='unregistered',
17 | new_name='permanently_retired',
18 | ),
19 | migrations.AlterField(
20 | model_name='participant',
21 | name='device_id',
22 | field=models.CharField(blank=True, max_length=256),
23 | ),
24 | migrations.AlterField(
25 | model_name='participant',
26 | name='os_type',
27 | field=models.CharField(blank=True, choices=[('IOS', 'IOS'), ('ANDROID', 'ANDROID'), ('', '')], max_length=16),
28 | ),
29 | migrations.AlterField(
30 | model_name='participant',
31 | name='patient_id',
32 | field=models.CharField(max_length=8, unique=True, validators=[django.core.validators.RegexValidator('^[1-9a-z]+$', message='This field can only contain characters 1-9 and a-z.')]),
33 | ),
34 | ]
35 |
--------------------------------------------------------------------------------
/database/migrations/0142_remove_s3file_compression_time_ns_and_more.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 5.2.2 on 2025-06-13 09:19
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0141_add_s3file_index_sha1'),
10 | ]
11 |
12 | operations = [
13 | migrations.RemoveField(
14 | model_name='s3file',
15 | name='compression_time_ns',
16 | ),
17 | migrations.RemoveField(
18 | model_name='s3file',
19 | name='decompression_time_ns',
20 | ),
21 | migrations.RemoveField(
22 | model_name='s3file',
23 | name='decrypt_time_ns',
24 | ),
25 | migrations.RemoveField(
26 | model_name='s3file',
27 | name='download_time_ns',
28 | ),
29 | migrations.RemoveField(
30 | model_name='s3file',
31 | name='encryption_time_ns',
32 | ),
33 | migrations.RemoveField(
34 | model_name='s3file',
35 | name='upload_time_ns',
36 | ),
37 | migrations.AddField(
38 | model_name='s3file',
39 | name='glacier',
40 | field=models.BooleanField(default=False),
41 | ),
42 | ]
43 |
--------------------------------------------------------------------------------
/frontend/static/javascript/app/survey-builder/directives/conditional-block/conditional-block.js:
--------------------------------------------------------------------------------
1 | angular.module("surveyBuilder")
2 | .directive("conditionalBlock", function(_, ARITHMETIC_OPERATORS) {
3 | return {
4 | link: function(scope) {
5 | scope._ = _;
6 | scope.ARITHMETIC_OPERATORS = ARITHMETIC_OPERATORS;
7 | scope.$watch("surveyBuilder.currentQuestionFields.question_id", function() {
8 | scope.currentQuestionId = scope.surveyBuilder.currentQuestionFields.question_id;
9 | scope.type = _.keys(scope.surveyBuilder.getValueAtPath(scope.path))[0];
10 | });
11 | scope.getQuestionNumber = function(questionId) {
12 | var index = _.indexOf(scope.surveyBuilder.questionIds, questionId);
13 | if (index >= 0) {
14 | return "Q" + ++index;
15 | } else {
16 | return "(error)"
17 | }
18 | }
19 | },
20 | replace: true,
21 | restrict: "E",
22 | scope: {
23 | surveyBuilder: "=",
24 | path: "@"
25 | },
26 | // need to add a fake variable to the end of the templateUrl to force the browser to reload the template
27 | templateUrl: "/static/javascript/app/survey-builder/directives/conditional-block/conditional-block.html?n=1"
28 | };
29 | });
30 |
--------------------------------------------------------------------------------
/scripts/script_that_deletes_problem_uploads.py:
--------------------------------------------------------------------------------
1 | from constants.common_constants import PROBLEM_UPLOADS
2 | from database.models import GenericEvent, IOSDecryptionKey, S3File
3 | from libs.s3 import s3_delete_many_versioned, s3_list_versions
4 |
5 |
6 | DELETION_PAGE_SIZE = 1000
7 |
8 |
9 | # based off the pattern over in particpiant purge
10 | def main():
11 | many_file_version_ids = []
12 | s3_prefix = PROBLEM_UPLOADS + "/"
13 |
14 | for i, key_path_version_id_tuple in enumerate(s3_list_versions(s3_prefix)):
15 | if i % DELETION_PAGE_SIZE * 10 == 0:
16 | print(i)
17 |
18 | # yield a page of files, reset page
19 | many_file_version_ids.append(key_path_version_id_tuple)
20 |
21 | if len(many_file_version_ids) % DELETION_PAGE_SIZE == 0:
22 | s3_delete_many_versioned(many_file_version_ids)
23 | many_file_version_ids = []
24 |
25 | if many_file_version_ids:
26 | s3_delete_many_versioned(many_file_version_ids)
27 |
28 | IOSDecryptionKey.objects.all().delete()
29 | # GenericEvent.objects.filter(tag__startswith="problem_upload").delete() # actually we can just clear it completely
30 | GenericEvent.objects.all().delete()
31 | S3File.objects.filter(path__startswith=PROBLEM_UPLOADS).delete()
32 |
--------------------------------------------------------------------------------
/database/migrations/0088_auto_20221019_2057.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.15 on 2022-10-19 20:57
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0087_auto_20221013_1412'),
10 | ]
11 |
12 | operations = [
13 | migrations.AddField(
14 | model_name='participant',
15 | name='last_get_latest_surveys',
16 | field=models.DateTimeField(blank=True, null=True),
17 | ),
18 | migrations.AddField(
19 | model_name='participant',
20 | name='last_register_user',
21 | field=models.DateTimeField(blank=True, null=True),
22 | ),
23 | migrations.AddField(
24 | model_name='participant',
25 | name='last_set_fcm_token',
26 | field=models.DateTimeField(blank=True, null=True),
27 | ),
28 | migrations.AddField(
29 | model_name='participant',
30 | name='last_set_password',
31 | field=models.DateTimeField(blank=True, null=True),
32 | ),
33 | migrations.AddField(
34 | model_name='participant',
35 | name='last_upload',
36 | field=models.DateTimeField(blank=True, null=True),
37 | ),
38 | ]
39 |
--------------------------------------------------------------------------------
/database/migrations/0118_filetoprocess_app_version_appversionhistory.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 4.2.11 on 2024-03-28 01:33
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0117_researcher_last_login_time_alter_survey_survey_type_and_more'),
11 | ]
12 |
13 | operations = [
14 | migrations.AddField(
15 | model_name='filetoprocess',
16 | name='app_version',
17 | field=models.CharField(blank=True, default='', max_length=16),
18 | ),
19 | migrations.CreateModel(
20 | name='AppVersionHistory',
21 | fields=[
22 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
23 | ('created_on', models.DateTimeField(auto_now_add=True)),
24 | ('last_updated', models.DateTimeField(auto_now=True)),
25 | ('app_version', models.CharField(max_length=16)),
26 | ('participant', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='app_version_history', to='database.participant')),
27 | ],
28 | options={
29 | 'abstract': False,
30 | },
31 | ),
32 | ]
33 |
--------------------------------------------------------------------------------
/database/migrations/0008_auto_20180418_1701.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # Generated by Django 1.11.5 on 2018-04-18 17:01
3 | from django.db import migrations, models
4 | import django.utils.timezone
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0007_auto_20180413_2033'),
11 | ]
12 |
13 | operations = [
14 | migrations.AlterField(
15 | model_name='chunkregistry',
16 | name='data_type',
17 | field=models.CharField(choices=[(b'accelerometer', b'accelerometer'), (b'bluetooth', b'bluetooth'), (b'calls', b'calls'), (b'gps', b'gps'), (b'identifiers', b'identifiers'), (b'app_log', b'app_log'), (b'power_state', b'power_state'), (b'survey_answers', b'survey_answers'), (b'survey_timings', b'survey_timings'), (b'texts', b'texts'), (b'audio_recordings', b'audio_recordings'), (b'wifi', b'wifi'), (b'proximity', b'proximity'), (b'gyro', b'gyro'), (b'magnetometer', b'magnetometer'), (b'devicemotion', b'devicemotion'), (b'reachability', b'reachability'), (b'ios_log', b'ios_log'), (b'image_survey', b'image_survey')], max_length=32),
18 | ),
19 | migrations.AlterField(
20 | model_name='surveyarchive',
21 | name='archive_end',
22 | field=models.DateTimeField(default=django.utils.timezone.now),
23 | ),
24 | ]
25 |
--------------------------------------------------------------------------------
/cluster_management/pushed_files/rabbitmq_configuration.txt:
--------------------------------------------------------------------------------
1 | # We use the environment-variable-like configuration for RabbitMQ.
2 | # documentation for all the environment variable options can be found at
3 | # https://www.rabbitmq.com/docs/configure#supported-environment-variables
4 |
5 | # these extra comments are from an ancient version of the default rabbitmq.conf file.
6 |
7 | # Defaults to rabbit. This can be useful if you want to run more than one node per machine -
8 | # RABBITMQ_NODENAME should be unique per erlang-node-and-machine combination. See the clustering on
9 | # a single machine guide for details:
10 | # http://www.rabbitmq.com/clustering.html#single-machine
11 | #NODENAME=rabbit
12 |
13 | # By default RabbitMQ will bind to all interfaces, on IPv4 and IPv6 if available. Set this if you
14 | # only want to bind to one network interface or# address family.
15 | #NODE_IP_ADDRESS=127.0.0.1
16 |
17 | # Defaults to 5672; we use port 50000.
18 | NODE_PORT=50000
19 |
20 | # 50001 is used by supervisord, so we will use 50002.
21 | # Task distribution works without needing to modify the security groups, not 100% clear
22 | # what this port is used for - its the erlang distribution port.
23 | # if not present you get a "invalid_dist_port_range,70000" error because by default it takes the
24 | # node port and adds 20k. Sure. Whatever.
25 | RABBITMQ_DIST_PORT=50002
26 |
--------------------------------------------------------------------------------
/.isort.cfg:
--------------------------------------------------------------------------------
1 | [settings]
2 | # documentation reference: https://pycqa.github.io/isort/docs/configuration/options.html
3 |
4 | # this configuration results in 4 separate, sorted sections of imports:
5 | # std library imports
6 | # pip-installed libraries
7 | # local project folders
8 | # imports from forest
9 | # (The names of the sections seems backwards, but the output is as desired.)
10 | KNOWN_FIRST_PARTY=libs,services,config,database,authentication,constants,middleware,cluster_management,endpoints
11 | KNOWN_LOCAL_FOLDER=forest
12 |
13 | # pulls * imports up to the top of their section (can prevent accidental overwriting of imports)
14 | star_first=True
15 |
16 | # this codebase uses a "very soft" 100 character line length limit.
17 | line_length=100
18 |
19 | # 10 = HANGING_INDENT_WITH_PARENTHESES - uses a numery mapping, see docs.
20 | multi_line_output=10
21 |
22 | # Parentheses are nicer than backslashes...
23 | use_parentheses=True
24 |
25 | # this defaults to false for a reason that is wholly worse than using an as-import.
26 | remove_redundant_aliases=True
27 |
28 | # include as-imports on the same line as others so you can find them without IDE help.
29 | combine_as_imports=True
30 |
31 | # alphabetical because it's easier to find things.
32 | order_by_type=False
33 |
34 | # This seems to be a default on most code formatters including our yapf config.
35 | lines_after_imports=2
36 |
--------------------------------------------------------------------------------
/libs/endpoint_helpers/password_validation_helpers.py:
--------------------------------------------------------------------------------
1 | import re
2 |
3 | from constants.message_strings import NEW_PASSWORD_N_LONG, NEW_PASSWORD_RULES_FAIL
4 | from constants.security_constants import PASSWORD_REQUIREMENT_REGEX_LIST
5 | from database.user_models_researcher import Researcher
6 |
7 |
8 | def check_password_requirements(researcher: Researcher, password: str) -> tuple[bool, str]:
9 | """ Runs all the password requirement tests for researcher passwords. """
10 | researcher_min = get_min_password_requirement(researcher)
11 | if len(password) < researcher_min:
12 | return False, NEW_PASSWORD_N_LONG.format(length=researcher_min)
13 |
14 | for regex in PASSWORD_REQUIREMENT_REGEX_LIST:
15 | if not re.search(regex, password):
16 | return False, NEW_PASSWORD_RULES_FAIL
17 |
18 | return True, None
19 |
20 |
21 | def get_min_password_requirement(researcher: Researcher) -> int:
22 | """ Returns the minimum required password length for a researcher. """
23 | # studies define their own minimum
24 | min_length = researcher.study_relations.order_by("-study__password_minimum_length") \
25 | .values_list("study__password_minimum_length", flat=True).first() or 0
26 | # for site admins it is 20
27 | if researcher.site_admin:
28 | min_length = 20
29 | # the absolute minimum is 8
30 | return max(min_length, 8)
--------------------------------------------------------------------------------
/database/migrations/0093_dataaccessrecord.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.16 on 2022-11-17 21:34
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0092_auto_20221103_1709'),
11 | ]
12 |
13 | operations = [
14 | migrations.CreateModel(
15 | name='DataAccessRecord',
16 | fields=[
17 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
18 | ('created_on', models.DateTimeField(auto_now_add=True)),
19 | ('last_updated', models.DateTimeField(auto_now=True)),
20 | ('query_params', models.TextField()),
21 | ('error', models.TextField(blank=True, null=True)),
22 | ('registry_dict_size', models.PositiveBigIntegerField(blank=True, null=True)),
23 | ('time_end', models.DateTimeField(blank=True, null=True)),
24 | ('bytes', models.PositiveBigIntegerField(blank=True, null=True)),
25 | ('researcher', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='data_access_record', to='database.researcher')),
26 | ],
27 | options={
28 | 'abstract': False,
29 | },
30 | ),
31 | ]
32 |
--------------------------------------------------------------------------------
/constants/celery_constants.py:
--------------------------------------------------------------------------------
1 | from os.path import join as path_join
2 |
3 | from constants import DjangoDropdown
4 | from constants.common_constants import BEIWE_PROJECT_ROOT
5 |
6 |
7 | # Celery Constants
8 | DATA_PROCESSING_CELERY_SERVICE = "services.celery_data_processing"
9 | DATA_PROCESSING_CELERY_QUEUE = "data_processing"
10 | PUSH_NOTIFICATION_SEND_SERVICE = "services.push_notification_send"
11 | PUSH_NOTIFICATION_SEND_QUEUE = "push_notifications"
12 | FOREST_SERVICE = "services.celery_forest"
13 | FOREST_QUEUE = "forest_queue"
14 | SCRIPTS_SERVICE = "services.scripts_runner"
15 | SCRIPTS_QUEUE = "scripts_queue"
16 |
17 | CELERY_CONFIG_LOCATION = path_join(BEIWE_PROJECT_ROOT, "manager_ip")
18 |
19 | # Push notification constants
20 | ANDROID_FIREBASE_CREDENTIALS = "android_firebase_credentials"
21 | IOS_FIREBASE_CREDENTIALS = "ios_firebase_credentials"
22 | BACKEND_FIREBASE_CREDENTIALS = "backend_firebase_credentials"
23 | # firebase gets the default app name unless otherwise specified, so it is necessary to have
24 | # another name for testing that will never be used to send notifications
25 | FIREBASE_APP_TEST_NAME = 'FIREBASE_APP_TEST_NAME'
26 |
27 |
28 | # Todo: make the pattern of using this object generic throughout all task processing, not just forest.
29 | class ForestTaskStatus(DjangoDropdown):
30 | queued = 'queued'
31 | running = 'running'
32 | success = 'success'
33 | error = 'error'
34 | cancelled = 'cancelled'
35 |
--------------------------------------------------------------------------------
/scripts/run_custom_ondeploy_script.py:
--------------------------------------------------------------------------------
1 | from sys import argv
2 |
3 | from constants.common_constants import CUSTOM_ONDEPLOY_SCRIPT_EB, CUSTOM_ONDEPLOY_SCRIPT_PROCESSING
4 | from libs.s3 import s3_list_files, s3_retrieve_plaintext
5 | from libs.sentry import SentryUtils
6 |
7 |
8 | def main():
9 | if "elasticbeanstalk" in argv:
10 | script_folder = CUSTOM_ONDEPLOY_SCRIPT_EB
11 | error_sentry = SentryUtils.report_webserver()
12 | elif "processing" in argv:
13 | script_folder = CUSTOM_ONDEPLOY_SCRIPT_PROCESSING
14 | error_sentry = SentryUtils.report_data_processing()
15 | else:
16 | raise Exception(f"Must supply either 'elasticbeanstalk' or 'processing' as an argument, found `{argv}`")
17 |
18 | sorted_files: list[str] = list(s3_list_files(script_folder))
19 | sorted_files.sort()
20 |
21 | for file_name in sorted_files:
22 | if file_name.endswith(".py"):
23 | # with error_sentry():
24 | python_file_contents = s3_retrieve_plaintext(file_name).decode()
25 | print(f"\nRunning {file_name}")
26 | # print("\n"*5, python_file_contents, "\n"*5)
27 | # this code needs to run in the current context with code assets loaded
28 | exec(python_file_contents, globals())
29 | print(f"\nDone running {file_name}\n")
30 |
31 | if error_sentry.errors:
32 | error_sentry.raise_errors() # stop execution if there are errors
33 |
--------------------------------------------------------------------------------
/user_scripts/kill_zombie_celery_tasks.py:
--------------------------------------------------------------------------------
1 | # add the root of the project into the path to allow cd-ing into this folder and running the script.
2 | from sys import path
3 | from os.path import abspath
4 | path.insert(0, abspath(__file__).rsplit('/', 2)[0])
5 |
6 | from psutil import process_iter
7 |
8 | for python_process in process_iter():
9 | # find python processes, identify celery push notification processes, and kill any zombies.
10 | # filter out non python3 and irrelevant celery tasks, we only care about push notifications.
11 | if "python3" != python_process.name().lower():
12 | continue
13 |
14 | # command is a list of all commandline parameters that were passed into the process.
15 | command = python_process.cmdline()
16 | if "celery" not in command or "services.celery_push_notifications" not in command:
17 | continue
18 |
19 | # we now have only have the celery processes for push notifications.
20 | # check that each one has a supervisord parent process.
21 | supervisord_parent = False
22 | supervisor_process = None
23 | for parent_process in python_process.parents():
24 | if parent_process.name().lower() == "supervisord":
25 | supervisord_parent = True
26 | supervisor_process = parent_process
27 |
28 | # If it doesn't have a supervisor process, or if it doesbut its not running, terminate it.
29 | if supervisord_parent == False or not supervisor_process.is_running():
30 | python_process.terminate()
31 |
--------------------------------------------------------------------------------
/requirements.in:
--------------------------------------------------------------------------------
1 | # core libraries
2 | Django>=5.2
3 | Django<5.3
4 | firebase-admin==5.4.0
5 | Jinja2
6 | orjson
7 |
8 | mypy==1.15.0
9 | django-stubs[1.15.0]
10 | pyzstd
11 |
12 | # django-extensions for some really nice management commands and terminal helpers.
13 | django-extensions
14 |
15 | python-dateutil
16 |
17 | # testing libraries
18 | # mock
19 | # coverage
20 | # tblib
21 | # redgreenunittest
22 | # time-machine
23 |
24 | pyqrcode
25 | pypng
26 | pyotp
27 |
28 | # This version of pycryptodome results in "from Cryptodome import ..." imports.
29 | pycryptodomex==3.23.0
30 |
31 | # error reporting
32 | cronutils==0.4.2
33 |
34 | # data sanitization
35 | bleach>=4.1.0
36 |
37 | # server requirement
38 | psycopg
39 |
40 | # ipython can exhibit weird issues that make it look like the codebase is broken. Most issues
41 | # are resolved by forcing this relatively new version of ipython.
42 | ipython>=8.12.2
43 | gunicorn
44 |
45 | # Keep these dependencies up to date
46 | boto3
47 |
48 | # we actually had a requirements conflict _involving pip-tools_ so we are removing it from the requirements...
49 | # pip-tools>=7.3.0
50 |
51 |
52 | #! YOU MUST manually update the following line to the correct git hash. it must be 40 characters.
53 | # current_forest_git_hash: f49b9898383e2a7a51302a5d45f8f2e573b2809e
54 | # manual override prefix is git+https://git@github.com/onnela-lab/forest@haaash_goooeees_heeerreee
55 | beiwe-forest==1.0
56 |
57 | celery==5.4.0
58 |
59 |
60 | rcssmin # extremely fast css minifier
61 |
--------------------------------------------------------------------------------
/database/migrations/0110_auto_20240110_1809.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.23 on 2024-01-10 18:09
2 |
3 | from django.db import migrations, models
4 |
5 |
6 | class Migration(migrations.Migration):
7 |
8 | dependencies = [
9 | ('database', '0109_auto_20231206_1116'),
10 | ]
11 |
12 | operations = [
13 | migrations.AddField(
14 | model_name='participant',
15 | name='enable_aggressive_background_persistence',
16 | field=models.BooleanField(default=False),
17 | ),
18 | migrations.AddField(
19 | model_name='participant',
20 | name='enable_beta_features',
21 | field=models.BooleanField(default=False),
22 | ),
23 | migrations.AddField(
24 | model_name='participant',
25 | name='enable_binary_uploads',
26 | field=models.BooleanField(default=False),
27 | ),
28 | migrations.AddField(
29 | model_name='participant',
30 | name='enable_developer_datastream',
31 | field=models.BooleanField(default=False),
32 | ),
33 | migrations.AddField(
34 | model_name='participant',
35 | name='enable_heartbeat',
36 | field=models.BooleanField(default=False),
37 | ),
38 | migrations.AddField(
39 | model_name='participant',
40 | name='enable_new_authentication',
41 | field=models.BooleanField(default=False),
42 | ),
43 | ]
44 |
--------------------------------------------------------------------------------
/database/migrations/0131_uuid_migration.py:
--------------------------------------------------------------------------------
1 | # created manually, 3024-10-2
2 |
3 | import uuid
4 |
5 | from django.db import migrations, models
6 | from django.db.migrations.state import StateApps
7 |
8 |
9 | def add_uuids(apps: StateApps, schema_editor):
10 | """
11 | We need to add uuids to existing Scheduledevent objects.
12 | """
13 | ScheduledEvent = apps.get_model('database', 'ScheduledEvent')
14 |
15 | new_uuids = [
16 | ScheduledEvent(id=pk, uuid=uuid.uuid4()) for pk in
17 | ScheduledEvent.objects.filter(uuid__isnull=True, deleted=False).values_list('id', flat=True)
18 | ]
19 | ScheduledEvent.objects.bulk_update(new_uuids, ['uuid'], batch_size=3000)
20 |
21 |
22 | def remove_uuids(apps: StateApps, schema_editor):
23 | """
24 | We need to remove uuids from existing Scheduledevent objects when migrating backwards (should
25 | just be for debugging? hope so!).
26 | """
27 | ScheduledEvent = apps.get_model('database', 'ScheduledEvent')
28 | ScheduledEvent.objects.all().update(uuid=None)
29 |
30 |
31 | class Migration(migrations.Migration):
32 |
33 | dependencies = [
34 | ('database', '0130_appversionhistory_os_is_ios'),
35 | ]
36 |
37 | operations = [
38 | migrations.RunPython(add_uuids, reverse_code=remove_uuids),
39 | migrations.AlterField(
40 | model_name='scheduledevent',
41 | name='uuid',
42 | field=models.UUIDField(blank=True, db_index=True, default=uuid.uuid4, null=True, unique=True),
43 | ),
44 | ]
45 |
--------------------------------------------------------------------------------
/database/migrations/0047_auto_20210208_2249.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 2.2.15 on 2021-02-08 22:49
2 |
3 | from django.db import migrations, models
4 | import django.db.models.deletion
5 |
6 |
7 | class Migration(migrations.Migration):
8 |
9 | dependencies = [
10 | ('database', '0046_auto_20210128_1632'),
11 | ]
12 |
13 | operations = [
14 | migrations.RemoveField(
15 | model_name='participant',
16 | name='push_notification_unreachable',
17 | ),
18 | migrations.AddField(
19 | model_name='archivedevent',
20 | name='status',
21 | field=models.TextField(db_index=True, default='success'),
22 | preserve_default=False,
23 | ),
24 | migrations.AddField(
25 | model_name='participant',
26 | name='push_notification_unreachable_count',
27 | field=models.SmallIntegerField(default=True),
28 | ),
29 | migrations.CreateModel(
30 | name='PushNotificationDisabledEvent',
31 | fields=[
32 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
33 | ('count', models.IntegerField()),
34 | ('timestamp', models.DateTimeField(auto_now_add=True, db_index=True)),
35 | ('participant', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='database.Participant')),
36 | ],
37 | options={
38 | 'abstract': False,
39 | },
40 | ),
41 | ]
42 |
--------------------------------------------------------------------------------
/SECURITY.md:
--------------------------------------------------------------------------------
1 | # Security Policy
2 |
3 | ## Hi, I'm Eli (@biblicabeebli), maintainer of the Beiwe Platform.
4 |
5 | I've got a good handle on security writ large, but this is a team of mostly just me so your support is appreciated. I welcome any such reports.
6 |
7 | If you have any security items to report, please email me at my dedicated contact for items related to this platform, biblicabeiwe@gmail.com. I will try to respond quickly.
8 |
9 |
10 | ## Before Reporting
11 |
12 | If your report is a suggestion about how to improve security, particularly via a configuration change to one of our supported runtime platforms, please post that as an issue on the github repository. Enhancements to existing deployments may require additional detail or safety checks as they will not necessarily be handled or applied by serious system administrators.
13 |
14 |
15 | ## Reporting a Vulnerability
16 |
17 | Please make your subject line clearly state that it is about a possible security issue on the beiwe-backend (or other) repo. If there is a CVE to look up please make that prominent in your email.
18 |
19 | I will respond to your communication as soon as possible. I will only discard communications that appear to be spam or clearly AI-generated.
20 |
21 |
22 | ## Version support
23 | The Beiwe Backend is a rolling release with no version number. There is no legacy support beyond upgrade guides, fixes and patches will be applied to the current release.
24 |
25 | ---
26 |
27 | (Beiwe is a platform for research purposes. It is developed by Onnela Lab at the Harvard T.H. Chan School of Public Health.)
--------------------------------------------------------------------------------
/database/migrations/0095_auto_20221212_1336.py:
--------------------------------------------------------------------------------
1 | # Generated by Django 3.2.16 on 2022-12-12 13:36
2 |
3 | import django.core.validators
4 | from django.db import migrations, models
5 | import django.utils.timezone
6 |
7 |
8 | class Migration(migrations.Migration):
9 |
10 | dependencies = [
11 | ('database', '0094_auto_20221205_1132'),
12 | ]
13 |
14 | operations = [
15 | migrations.AddField(
16 | model_name='researcher',
17 | name='password_force_reset',
18 | field=models.BooleanField(default=False),
19 | ),
20 | migrations.AddField(
21 | model_name='researcher',
22 | name='password_last_changed',
23 | field=models.DateTimeField(default=django.utils.timezone.now),
24 | ),
25 | migrations.AddField(
26 | model_name='study',
27 | name='password_max_age_days',
28 | field=models.PositiveIntegerField(default=365, validators=[django.core.validators.MinValueValidator(30), django.core.validators.MaxValueValidator(365)]),
29 | ),
30 | migrations.AddField(
31 | model_name='study',
32 | name='password_max_age_enabled',
33 | field=models.BooleanField(default=False),
34 | ),
35 | migrations.AddField(
36 | model_name='study',
37 | name='password_minimum_length',
38 | field=models.PositiveIntegerField(default=8, validators=[django.core.validators.MinValueValidator(8), django.core.validators.MaxValueValidator(100)]),
39 | ),
40 | ]
41 |
--------------------------------------------------------------------------------
/frontend/static/javascript/app/survey-builder/directives/add-logic-buttons/add-logic-buttons.html:
--------------------------------------------------------------------------------
1 |