├── .dockerignore ├── .env.template ├── .github └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── Dockerfile ├── Makefile ├── README.md ├── VERSION ├── contrib ├── entrypoint.sh └── start.sh ├── docker-compose.yml ├── manage.py ├── nginx.conf.sigil ├── pylama.ini ├── pytest.ini ├── requirements.txt ├── sensorsafrica ├── __init__.py ├── admin.py ├── api │ ├── __init__.py │ ├── models.py │ ├── v1 │ │ ├── __init__.py │ │ ├── filters.py │ │ ├── router.py │ │ ├── serializers.py │ │ └── views.py │ └── v2 │ │ ├── __init__.py │ │ ├── filters.py │ │ ├── router.py │ │ ├── serializers.py │ │ └── views.py ├── celeryapp.py ├── fixtures │ ├── auth.json │ ├── sensordata.json │ ├── sensors.json │ └── sensortypes.json ├── management │ └── commands │ │ ├── add_city_names.py │ │ ├── cache_lastactive_nodes.py │ │ ├── cache_static_json_data.py │ │ ├── calculate_data_statistics.py │ │ └── upload_to_ckan.py ├── migrations │ ├── 0001_sensordatastat.py │ ├── 0002_city.py │ ├── 0003_auto_20190222_1137.py │ ├── 0004_auto_20190509_1145.py │ └── __init__.py ├── openstuttgart │ ├── __init__.py │ └── feinstaub │ │ ├── __init__.py │ │ └── sensors │ │ ├── __init__.py │ │ └── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_auto_20150330_1800.py │ │ ├── 0003_auto_20150330_1805.py │ │ ├── 0004_auto_20150331_1907.py │ │ ├── 0005_auto_20150403_2041.py │ │ ├── 0006_auto_20150404_2050.py │ │ ├── 0007_auto_20150405_2151.py │ │ ├── 0008_auto_20150503_1554.py │ │ ├── 0009_auto_20150503_1556.py │ │ ├── 0010_auto_20150620_1708.py │ │ ├── 0011_auto_20150807_1927.py │ │ ├── 0012_auto_20150807_1943.py │ │ ├── 0013_auto_20151025_1615.py │ │ ├── 0014_sensor_public.py │ │ ├── 0015_sensordata_software_version.py │ │ ├── 0016_auto_20160209_2030.py │ │ ├── 0017_auto_20160416_1803.py │ │ ├── 0018_auto_20170218_2329.py │ │ ├── 0019_auto_20190125_0521.py │ │ ├── 0020_auto_20190314_1232.py │ │ ├── 0021_auto_20210204_1106.py │ │ ├── 0022_auto_20210211_2023.py │ │ └── __init__.py ├── router.py ├── settings.py ├── static │ └── v2 │ │ ├── data.1h.json │ │ ├── data.24h.json │ │ ├── data.dust.min.json │ │ ├── data.json │ │ └── data.temp.min.json ├── tasks.py ├── templates │ ├── addsensordevice.html │ └── rest_framework │ │ └── api.html ├── tests │ ├── conftest.py │ ├── settings.py │ ├── test_city_view.py │ ├── test_filter_view.py │ ├── test_full_push.py │ ├── test_large_dataset.py │ ├── test_lastactive.py │ ├── test_nested_write_sensordata_push.py │ ├── test_node_view.py │ ├── test_sensor_location_view.py │ ├── test_sensor_type_view.py │ ├── test_sensor_view.py │ ├── test_sensordata_view.py │ └── test_sensordatastats_view.py ├── urls.py └── wsgi.py └── setup.py /.dockerignore: -------------------------------------------------------------------------------- 1 | # Git 2 | .git 3 | .gitignore 4 | 5 | # Docker 6 | docker-compose.yml 7 | .docker 8 | 9 | # Byte-compiled / optimized / DLL files 10 | __pycache__/ 11 | */__pycache__/ 12 | */*/__pycache__/ 13 | */*/*/__pycache__/ 14 | *.py[cod] 15 | */*.py[cod] 16 | */*/*.py[cod] 17 | */*/*/*.py[cod] 18 | 19 | # Distribution / packaging 20 | .Python 21 | env/ 22 | build/ 23 | develop-eggs/ 24 | dist/ 25 | downloads/ 26 | eggs/ 27 | lib/ 28 | lib64/ 29 | parts/ 30 | sdist/ 31 | var/ 32 | *.egg-info/ 33 | .installed.cfg 34 | *.egg 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | # Virtual environment 60 | .env/ 61 | .venv/ 62 | venv/ 63 | 64 | # PyCharm 65 | .idea 66 | 67 | # Python mode for VIM 68 | .ropeproject 69 | */.ropeproject 70 | */*/.ropeproject 71 | */*/*/.ropeproject 72 | 73 | # Vim swap files 74 | *.swp 75 | */*.swp 76 | */*/*.swp 77 | */*/*/*.swp 78 | -------------------------------------------------------------------------------- /.env.template: -------------------------------------------------------------------------------- 1 | SENSORSAFRICA_DATABASE_URL="" 2 | SENSORSAFRICA_DEBUG="" 3 | SENSORSAFRICA_FLOWER_ADMIN_PASSWORD="" 4 | SENSORSAFRICA_FLOWER_ADMIN_USERNAME="" 5 | SENSORSAFRICA_RABBITMQ_URL="" 6 | SENSORSAFRICA_SENTRY_DSN="" 7 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. 4 | 5 | Fixes # (issue) 6 | 7 | ## Type of change 8 | 9 | Please delete options that are not relevant. 10 | 11 | - [ ] Bug fix (non-breaking change which fixes an issue) 12 | - [ ] New feature (non-breaking change which adds functionality) 13 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 14 | - [ ] This change requires a documentation update 15 | 16 | 17 | ## Screenshots 18 | 19 | 20 | ## Checklist: 21 | 22 | - [ ] My code follows the style guidelines of this project 23 | - [ ] I have performed a self-review of my own code 24 | - [ ] I have commented my code, particularly in hard-to-understand areas 25 | - [ ] I have made corresponding changes to the documentation -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/vim,venv,emacs,linux,macos,python,django,virtualenv,intellij+all,visualstudiocode 3 | # Edit at https://www.gitignore.io/?templates=vim,venv,emacs,linux,macos,python,django,virtualenv,intellij+all,visualstudiocode 4 | 5 | ### Django ### 6 | *.log 7 | *.pot 8 | *.pyc 9 | __pycache__/ 10 | local_settings.py 11 | db.sqlite3 12 | media 13 | 14 | # If your build process includes running collectstatic, then you probably don't need or want to include staticfiles/ 15 | # in your Git repository. Update and uncomment the following line accordingly. 16 | # /staticfiles/ 17 | 18 | ### Django.Python Stack ### 19 | # Byte-compiled / optimized / DLL files 20 | *.py[cod] 21 | *$py.class 22 | 23 | # C extensions 24 | *.so 25 | 26 | # Distribution / packaging 27 | .Python 28 | build/ 29 | develop-eggs/ 30 | dist/ 31 | downloads/ 32 | eggs/ 33 | .eggs/ 34 | lib/ 35 | lib64/ 36 | parts/ 37 | sdist/ 38 | var/ 39 | wheels/ 40 | share/python-wheels/ 41 | *.egg-info/ 42 | .installed.cfg 43 | *.egg 44 | MANIFEST 45 | 46 | # PyInstaller 47 | # Usually these files are written by a python script from a template 48 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 49 | *.manifest 50 | *.spec 51 | 52 | # Installer logs 53 | pip-log.txt 54 | pip-delete-this-directory.txt 55 | 56 | # Unit test / coverage reports 57 | htmlcov/ 58 | .tox/ 59 | .nox/ 60 | .coverage 61 | .coverage.* 62 | .cache 63 | nosetests.xml 64 | coverage.xml 65 | *.cover 66 | .hypothesis/ 67 | .pytest_cache/ 68 | 69 | # Translations 70 | *.mo 71 | 72 | # Django stuff: 73 | 74 | # Flask stuff: 75 | instance/ 76 | .webassets-cache 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # PyBuilder 85 | target/ 86 | 87 | # Jupyter Notebook 88 | .ipynb_checkpoints 89 | 90 | # IPython 91 | profile_default/ 92 | ipython_config.py 93 | 94 | # pyenv 95 | .python-version 96 | 97 | # SageMath parsed files 98 | *.sage.py 99 | 100 | # Environments 101 | .env 102 | .venv 103 | env/ 104 | venv/ 105 | ENV/ 106 | env.bak/ 107 | venv.bak/ 108 | 109 | # Spyder project settings 110 | .spyderproject 111 | .spyproject 112 | 113 | # Rope project settings 114 | .ropeproject 115 | 116 | # mkdocs documentation 117 | /site 118 | 119 | # mypy 120 | .mypy_cache/ 121 | .dmypy.json 122 | dmypy.json 123 | 124 | # Pyre type checker 125 | .pyre/ 126 | 127 | ### Emacs ### 128 | # -*- mode: gitignore; -*- 129 | *~ 130 | \#*\# 131 | /.emacs.desktop 132 | /.emacs.desktop.lock 133 | *.elc 134 | auto-save-list 135 | tramp 136 | .\#* 137 | 138 | # Org-mode 139 | .org-id-locations 140 | *_archive 141 | 142 | # flymake-mode 143 | *_flymake.* 144 | 145 | # eshell files 146 | /eshell/history 147 | /eshell/lastdir 148 | 149 | # elpa packages 150 | /elpa/ 151 | 152 | # reftex files 153 | *.rel 154 | 155 | # AUCTeX auto folder 156 | /auto/ 157 | 158 | # cask packages 159 | .cask/ 160 | 161 | # Flycheck 162 | flycheck_*.el 163 | 164 | # server auth directory 165 | /server/ 166 | 167 | # projectiles files 168 | .projectile 169 | 170 | # directory configuration 171 | .dir-locals.el 172 | 173 | # network security 174 | /network-security.data 175 | 176 | 177 | ### Intellij+all ### 178 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 179 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 180 | 181 | # User-specific stuff 182 | .idea/**/workspace.xml 183 | .idea/**/tasks.xml 184 | .idea/**/usage.statistics.xml 185 | .idea/**/dictionaries 186 | .idea/**/shelf 187 | 188 | # Generated files 189 | .idea/**/contentModel.xml 190 | 191 | # Sensitive or high-churn files 192 | .idea/**/dataSources/ 193 | .idea/**/dataSources.ids 194 | .idea/**/dataSources.local.xml 195 | .idea/**/sqlDataSources.xml 196 | .idea/**/dynamic.xml 197 | .idea/**/uiDesigner.xml 198 | .idea/**/dbnavigator.xml 199 | 200 | # Gradle 201 | .idea/**/gradle.xml 202 | .idea/**/libraries 203 | 204 | # Gradle and Maven with auto-import 205 | # When using Gradle or Maven with auto-import, you should exclude module files, 206 | # since they will be recreated, and may cause churn. Uncomment if using 207 | # auto-import. 208 | # .idea/modules.xml 209 | # .idea/*.iml 210 | # .idea/modules 211 | 212 | # CMake 213 | cmake-build-*/ 214 | 215 | # Mongo Explorer plugin 216 | .idea/**/mongoSettings.xml 217 | 218 | # File-based project format 219 | *.iws 220 | 221 | # IntelliJ 222 | out/ 223 | 224 | # mpeltonen/sbt-idea plugin 225 | .idea_modules/ 226 | 227 | # JIRA plugin 228 | atlassian-ide-plugin.xml 229 | 230 | # Cursive Clojure plugin 231 | .idea/replstate.xml 232 | 233 | # Crashlytics plugin (for Android Studio and IntelliJ) 234 | com_crashlytics_export_strings.xml 235 | crashlytics.properties 236 | crashlytics-build.properties 237 | fabric.properties 238 | 239 | # Editor-based Rest Client 240 | .idea/httpRequests 241 | 242 | # Android studio 3.1+ serialized cache file 243 | .idea/caches/build_file_checksums.ser 244 | 245 | ### Intellij+all Patch ### 246 | # Ignores the whole .idea folder and all .iml files 247 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 248 | 249 | .idea/ 250 | 251 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 252 | 253 | *.iml 254 | modules.xml 255 | .idea/misc.xml 256 | *.ipr 257 | 258 | ### Linux ### 259 | 260 | # temporary files which can be created if a process still has a handle open of a deleted file 261 | .fuse_hidden* 262 | 263 | # KDE directory preferences 264 | .directory 265 | 266 | # Linux trash folder which might appear on any partition or disk 267 | .Trash-* 268 | 269 | # .nfs files are created when an open file is removed but is still being accessed 270 | .nfs* 271 | 272 | ### macOS ### 273 | # General 274 | .DS_Store 275 | .AppleDouble 276 | .LSOverride 277 | 278 | # Icon must end with two \r 279 | Icon 280 | 281 | # Thumbnails 282 | ._* 283 | 284 | # Files that might appear in the root of a volume 285 | .DocumentRevisions-V100 286 | .fseventsd 287 | .Spotlight-V100 288 | .TemporaryItems 289 | .Trashes 290 | .VolumeIcon.icns 291 | .com.apple.timemachine.donotpresent 292 | 293 | # Directories potentially created on remote AFP share 294 | .AppleDB 295 | .AppleDesktop 296 | Network Trash Folder 297 | Temporary Items 298 | .apdisk 299 | 300 | ### Python ### 301 | # Byte-compiled / optimized / DLL files 302 | 303 | # C extensions 304 | 305 | # Distribution / packaging 306 | 307 | # PyInstaller 308 | # Usually these files are written by a python script from a template 309 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 310 | 311 | # Installer logs 312 | 313 | # Unit test / coverage reports 314 | 315 | # Translations 316 | 317 | # Django stuff: 318 | 319 | # Flask stuff: 320 | 321 | # Scrapy stuff: 322 | 323 | # Sphinx documentation 324 | 325 | # PyBuilder 326 | 327 | # Jupyter Notebook 328 | 329 | # IPython 330 | 331 | # pyenv 332 | 333 | # celery beat schedule file 334 | 335 | # SageMath parsed files 336 | 337 | # Environments 338 | 339 | # Spyder project settings 340 | 341 | # Rope project settings 342 | 343 | # mkdocs documentation 344 | 345 | # mypy 346 | 347 | # Pyre type checker 348 | 349 | ### Python Patch ### 350 | .venv/ 351 | 352 | ### venv ### 353 | # Virtualenv 354 | # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ 355 | [Bb]in 356 | [Ii]nclude 357 | [Ll]ib 358 | [Ll]ib64 359 | [Ll]ocal 360 | [Ss]cripts 361 | pyvenv.cfg 362 | pip-selfcheck.json 363 | 364 | ### Vim ### 365 | # Swap 366 | [._]*.s[a-v][a-z] 367 | [._]*.sw[a-p] 368 | [._]s[a-rt-v][a-z] 369 | [._]ss[a-gi-z] 370 | [._]sw[a-p] 371 | 372 | # Session 373 | Session.vim 374 | 375 | # Temporary 376 | .netrwhist 377 | # Auto-generated tag files 378 | tags 379 | # Persistent undo 380 | [._]*.un~ 381 | 382 | ### VirtualEnv ### 383 | # Virtualenv 384 | # http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ 385 | 386 | ### VisualStudioCode ### 387 | .vscode/ 388 | 389 | ### VisualStudioCode Patch ### 390 | # Ignore all local history of files 391 | .history 392 | 393 | # End of https://www.gitignore.io/api/vim,venv,emacs,linux,macos,python,django,virtualenv,intellij+all,visualstudiocode 394 | 395 | 396 | # celery beat schedule file 397 | celerybeat-schedule 398 | celerybeat.pid 399 | 400 | logs/ 401 | 402 | /static 403 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6.3 2 | 3 | ENV PYTHONUNBUFFERED 1 4 | 5 | # Create application root directory 6 | WORKDIR /src 7 | 8 | RUN mkdir media static logs 9 | VOLUME [ "/src/logs" ] 10 | 11 | # Upgrade pip and setuptools with trusted hosts 12 | RUN python -m pip install --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade pip setuptools 13 | 14 | # Copy the current directory contents into the container at sensorsafrica 15 | COPY . /src/ 16 | 17 | # Install dependencies 18 | RUN pip install -q git+https://github.com/opendata-stuttgart/feinstaub-api && \ 19 | pip install -q . 20 | 21 | # Expose port server 22 | EXPOSE 8000 23 | EXPOSE 5555 24 | 25 | COPY ./contrib/start.sh /start.sh 26 | COPY ./contrib/entrypoint.sh /entrypoint.sh 27 | 28 | ENTRYPOINT ["/entrypoint.sh"] 29 | CMD [ "/start.sh" ] 30 | 31 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | COMPOSE = docker-compose 2 | 3 | build: 4 | $(COMPOSE) build 5 | 6 | up: 7 | $(COMPOSE) up -d 8 | 9 | log: 10 | $(COMPOSE) logs -f 11 | 12 | compilescss: 13 | $(COMPOSE) exec api python manage.py compilescss 14 | $(COMPOSE) exec api python manage.py collectstatic --clear --noinput 15 | 16 | enter: 17 | $(COMPOSE) exec api bash 18 | 19 | shell: 20 | $(COMPOSE) exec api python manage.py shell 21 | 22 | migrate: 23 | $(COMPOSE) exec api python manage.py migrate 24 | 25 | test: 26 | $(COMPOSE) exec api pytest --pylama 27 | 28 | testexpr: 29 | $(COMPOSE) exec api pytest --pylama -k '$(expr)' 30 | 31 | createsuperuser: 32 | $(COMPOSE) exec api python manage.py createsuperuser 33 | 34 | down: 35 | $(COMPOSE) down 36 | 37 | clean: 38 | @find . -name "*.pyc" -exec rm -rf {} \; 39 | @find . -name "__pycache__" -delete 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sensors.AFRICA API 2 | 3 | API to save and access data from deployed sensors in cities all around Africa. 4 | 5 | ## Documentation 6 | 7 | The API is documented [here.](https://github.com/CodeForAfricaLabs/sensors.AFRICA-api/wiki/API-Documentation) 8 | 9 | ## Development 10 | 11 | Gitignore is standardized for this project using [gitignore.io](https://www.gitignore.io/) to support various development platforms. 12 | To get the project up and running: 13 | 14 | - Clone this repo 15 | 16 | ### Virtual environment 17 | 18 | - Use virtualenv to create your virtual environment; `virtualenv venv` 19 | - Activate the virtual environment; `source venv/bin/activate` 20 | - Install feinstaub; `pip install git+https://github.com/opendata-stuttgart/feinstaub-api` 21 | - Install the requirements; `pip install .` 22 | - Create a sensorsafrica database with the following sql script: 23 | 24 | ```sql 25 | CREATE DATABASE sensorsafrica; 26 | CREATE USER sensorsafrica WITH ENCRYPTED PASSWORD 'sensorsafrica'; 27 | GRANT ALL PRIVILEGES ON DATABASE sensorsafrica TO sensorsafrica; 28 | ``` 29 | 30 | - Migrate the database; `python manage.py migrate` 31 | - Run the server; `python manage.py runserver` 32 | - Create super user for admin login; `python manage.py createsuperuser` 33 | 34 | username: `` 35 | email: blank 36 | password: `` 37 | 38 | ### Docker 39 | 40 | Using docker compose: 41 | 42 | - Create a `.env` file using `.env.template` . ***docker-compose has some default values for these variables*** 43 | - Build the project; `docker-compose build` or `make build` 44 | - Run the project; `docker-compose up -d` or `make up` 45 | 46 | Docker compose make commands: 47 | 48 | - `make build` 49 | - `make up` - run docker and detach 50 | - `make log` - tail logs 51 | - `make test` - run test 52 | - `make migrate` - migrate database 53 | - `make createsuperuser` - create a super user for admin 54 | - `make compilescss` 55 | - `make enter` - enter docker shell 56 | - `make django` - enter docker django shell 57 | 58 | **NOTE:** 59 | `docker-compose` is strictly for development and testing purposes. 60 | The Dockerfile is written for production since dokku is being used and it will look for Dockerfile. 61 | 62 | ### Tests 63 | 64 | - Virtual Environment; `pytest --pylama` 65 | - Docker; `docker-compose run api pytest --pylama` 66 | 67 | **NOTE:** 68 | If entrypoint and start scripts are changed, make sure they have correct/required permissions since we don't grant permissions to the files using the Dockerfile. 69 | Run the commands: 70 | 71 | ```bash 72 | chmod +x contrib/entrypoint.sh 73 | chmod +x contrib/start.sh 74 | ``` 75 | 76 | ## Deployment 77 | 78 | ### Dokku 79 | 80 | On your local machine run: 81 | 82 | ```bash 83 | git remote add dokku dokku@dokku.me:sensorsafrica-api 84 | git push dokku master 85 | ``` 86 | 87 | For more information read [Deploying to Dokku](http://dokku.viewdocs.io/dokku/deployment/application-deployment/#deploying-to-dokku). 88 | 89 | ### Cronjob 90 | 91 | This project uses celery to create cronjobs and flower to monitor the cron jobs as a web admin. 92 | To create your jobs, add the task to the `tasks.py` and `CELERY_BEAT_SCHEDULE` in `settings.py`. 93 | 94 | Everything starts automatically as setup in the `contrib/start.sh`: 95 | 96 | ```bash 97 | celery -A sensorsafrica beat -l info &> /src/logs/celery.log & 98 | celery -A sensorsafrica worker -l info &> /src/logs/celery.log & 99 | celery -A sensorsafrica flower --basic_auth=$SENSORSAFRICA_FLOWER_ADMIN_USERNAME:$SENSORSAFRICA_FLOWER_ADMIN_PASSWORD &> /src/logs/celery.log & 100 | ``` 101 | 102 | Note: If you run the project in the virtualenv you will have to start rabbitmq and pass that link to settings by the env variable `SENSORSAFRICA_RABBITMQ_URL` 103 | 104 | 105 | ## Monitoring 106 | 107 | ### Flower 108 | 109 | It starts up in in the `contrib/start.sh`: 110 | 111 | ```bash 112 | ... 113 | celery -A sensorsafrica flower --basic_auth=$SENSORSAFRICA_FLOWER_ADMIN_USERNAME:$SENSORSAFRICA_FLOWER_ADMIN_PASSWORD &> /src/logs/celery.log & 114 | ``` 115 | 116 | ### Slack 117 | 118 | Provide channel webhook as an enivronment variable `SENSORSAFRICA_CELERY_SLACK_WEBHOOK`. The default options are used: 119 | 120 | ``` 121 | DEFAULT_OPTIONS = { 122 | "slack_beat_init_color": "#FFCC2B", 123 | "slack_broker_connect_color": "#36A64F", 124 | "slack_broker_disconnect_color": "#D00001", 125 | "slack_celery_startup_color": "#FFCC2B", 126 | "slack_celery_shutdown_color": "#660033", 127 | "slack_task_prerun_color": "#D3D3D3", 128 | "slack_task_success_color": "#36A64F", 129 | "slack_task_failure_color": "#D00001", 130 | "slack_request_timeout": 1, 131 | "flower_base_url": None, 132 | "show_celery_hostname": False, 133 | "show_task_id": True, 134 | "show_task_execution_time": True, 135 | "show_task_args": True, 136 | "show_task_kwargs": True, 137 | "show_task_exception_info": True, 138 | "show_task_return_value": True, 139 | "show_task_prerun": False, 140 | "show_startup": True, 141 | "show_shutdown": True, 142 | "show_beat": True, 143 | "show_broker": False, 144 | "use_fixed_width": True, 145 | "include_tasks": None, 146 | "exclude_tasks": None, 147 | "failures_only": False, 148 | "webhook": None, 149 | "beat_schedule": None, 150 | "beat_show_full_task_path": False, 151 | } 152 | ``` 153 | 154 | ### Sentry 155 | 156 | Set the enivronment variable `SENSORSAFRICA_SENTRY_DSN`. 157 | 158 | ### Archiving 159 | 160 | Archives are sent to CKAN and require environment configuration: 161 | 162 | ``` 163 | - CKAN_ARCHIVE_API_KEY=.. 164 | - CKAN_ARCHIVE_OWNER_ID=... 165 | - CKAN_ARCHIVE_URL= 166 | ``` 167 | 168 | ## Contributing 169 | 170 | [opendata-stuttgart/feinstaub-api](https://github.com/opendata-stuttgart/feinstaub-api) prefer generating and applying migration to the database at the point of deployment (probably to reduce the number of changes to be applied). 171 | We, on the other hand, prefer the Django recommended approach of creating and reviewing migration files at the development time, and then applying the same migration files to different environments; dev, staging and eventually production. 172 | 173 | Hence, with any contribution, include both `sensors.AFRICA-api` and `opendata-stuttgart/feinstaub-api` migration files by running `python manage.py makemigrations` command before creating a PR. 174 | 175 | ## License 176 | 177 | GNU GPLv3 178 | 179 | Copyright (C) 2018 Code for Africa 180 | 181 | This program is free software: you can redistribute it and/or modify 182 | it under the terms of the GNU General Public License as published by 183 | the Free Software Foundation, either version 3 of the License, or 184 | (at your option) any later version. 185 | 186 | This program is distributed in the hope that it will be useful, 187 | but WITHOUT ANY WARRANTY; without even the implied warranty of 188 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 189 | GNU General Public License for more details. 190 | 191 | You should have received a copy of the GNU General Public License 192 | along with this program. If not, see . 193 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.1 -------------------------------------------------------------------------------- /contrib/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | cmd="$@" 4 | 5 | function postgres_ready(){ 6 | python << END 7 | import sys 8 | import psycopg2 9 | try: 10 | conn = psycopg2.connect("$SENSORSAFRICA_DATABASE_URL") 11 | except psycopg2.OperationalError: 12 | sys.exit(-1) 13 | sys.exit(0) 14 | END 15 | } 16 | 17 | until postgres_ready; do 18 | >&2 echo "Postgres is unavailable - sleeping" 19 | sleep 1 20 | done 21 | 22 | >&2 echo "Postgres is up - continuing..." 23 | exec $cmd 24 | -------------------------------------------------------------------------------- /contrib/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | python manage.py migrate --noinput # Apply database migrations 3 | python manage.py collectstatic --clear --noinput # Collect static files 4 | 5 | # Prepare log files and start outputting logs to stdout 6 | touch /src/logs/celery.log 7 | touch /src/logs/gunicorn.log 8 | touch /src/logs/access.log 9 | tail -n 0 -f /src/logs/*.log & 10 | 11 | celery -A sensorsafrica beat -l info &> /src/logs/celery.log & 12 | celery -A sensorsafrica worker --hostname=$DOKKU_APP_NAME -l info &> /src/logs/celery.log & 13 | celery -A sensorsafrica flower --basic_auth=$SENSORSAFRICA_FLOWER_ADMIN_USERNAME:$SENSORSAFRICA_FLOWER_ADMIN_PASSWORD &> /src/logs/celery.log & 14 | 15 | # Start Gunicorn processes 16 | echo Starting Gunicorn. 17 | exec gunicorn \ 18 | --bind 0.0.0.0:8000 \ 19 | --timeout 180 \ 20 | --workers 5 \ 21 | --worker-class gevent \ 22 | --log-level=info \ 23 | --log-file=/src/logs/gunicorn.log \ 24 | --access-logfile=/src/logs/access.log \ 25 | --name sensorsafrica --reload sensorsafrica.wsgi:application \ 26 | --chdir sensorsafrica/ 27 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | rabbitmq: 5 | image: rabbitmq:3.12.7-management 6 | ports: 7 | - "5672:5672" 8 | # GUI port 9 | - "15672:15672" 10 | environment: 11 | - RABBITMQ_DEFAULT_USER=sensorsafrica 12 | - RABBITMQ_DEFAULT_PASS=sensorsafrica 13 | healthcheck: 14 | test: [ "CMD-SHELL", "rabbitmq-diagnostics -q ping" ] 15 | interval: 10s 16 | timeout: 5s 17 | retries: 2 18 | 19 | postgres: 20 | image: postgres:13.7 21 | ports: 22 | - "54321:5432" 23 | environment: 24 | - POSTGRES_USER=sensorsafrica 25 | - POSTGRES_PASSWORD=sensorsafrica 26 | - POSTGRES_DB=sensorsafrica 27 | volumes: 28 | - postgres_data:/var/lib/postgresql/data/ 29 | api: 30 | build: 31 | context: . 32 | environment: 33 | SENSORSAFRICA_DATABASE_URL: ${SENSORSAFRICA_DATABASE_URL:-postgres://sensorsafrica:sensorsafrica@postgres:5432/sensorsafrica} 34 | SENSORSAFRICA_RABBITMQ_URL: ${SENSORSAFRICA_RABBITMQ_URL:-amqp://sensorsafrica:sensorsafrica@rabbitmq/} 35 | SENSORSAFRICA_FLOWER_ADMIN_USERNAME: ${SENSORSAFRICA_FLOWER_ADMIN_USERNAME:-admin} 36 | SENSORSAFRICA_FLOWER_ADMIN_PASSWORD: ${SENSORSAFRICA_FLOWER_ADMIN_PASSWORD:-password} 37 | DOKKU_APP_NAME: ${DOKKU_APP_NAME:-sensorsafrica} 38 | CKAN_ARCHIVE_API_KEY: ${CKAN_ARCHIVE_API_KEY} 39 | CKAN_ARCHIVE_OWNER_ID: ${CKAN_ARCHIVE_OWNER_ID} 40 | CKAN_ARCHIVE_URL: ${CKAN_ARCHIVE_URL} 41 | depends_on: 42 | - postgres 43 | - rabbitmq 44 | links: 45 | - postgres 46 | volumes: 47 | - .:/src 48 | ports: 49 | - "8000:8000" 50 | - "5555:5555" 51 | 52 | volumes: 53 | postgres_data: 54 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'sensorsafrica.settings') 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | assert django 16 | except ImportError: 17 | raise ImportError( 18 | "Couldn't import Django. Are you sure it's installed and " 19 | "available on your PYTHONPATH environment variable? Did you " 20 | "forget to activate a virtual environment?" 21 | ) 22 | raise 23 | execute_from_command_line(sys.argv) 24 | -------------------------------------------------------------------------------- /nginx.conf.sigil: -------------------------------------------------------------------------------- 1 | {{ range $port_map := .PROXY_PORT_MAP | split " " }} 2 | {{ $port_map_list := $port_map | split ":" }} 3 | {{ $scheme := index $port_map_list 0 }} 4 | {{ $listen_port := index $port_map_list 1 }} 5 | {{ $upstream_port := index $port_map_list 2 }} 6 | 7 | {{ if eq $scheme "http" }} 8 | server { 9 | listen [::]:{{ $listen_port }}; 10 | listen {{ $listen_port }}; 11 | {{ if $.NOSSL_SERVER_NAME }}server_name {{ $.NOSSL_SERVER_NAME }}; {{ end }} 12 | access_log /var/log/nginx/{{ $.APP }}-access.log; 13 | error_log /var/log/nginx/{{ $.APP }}-error.log; 14 | 15 | location / { 16 | 17 | gzip on; 18 | gzip_min_length 1100; 19 | gzip_buffers 4 32k; 20 | gzip_types text/css text/javascript text/xml text/plain text/x-component application/javascript application/x-javascript application/json application/xml application/rss+xml font/truetype application/x-font-ttf font/opentype application/vnd.ms-fontobject image/svg+xml; 21 | gzip_vary on; 22 | gzip_comp_level 6; 23 | 24 | ## Start CORS here. 25 | 26 | # Django already returns header 'Access-Control-Allow-Origin' '*' 27 | # add_header 'Access-Control-Allow-Origin' '*'; 28 | add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; 29 | add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; 30 | add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; 31 | 32 | 33 | ##End CORS 34 | 35 | proxy_pass http://{{ $.APP }}-{{ $upstream_port }}; 36 | proxy_http_version 1.1; 37 | proxy_set_header Upgrade $http_upgrade; 38 | proxy_set_header Connection "upgrade"; 39 | proxy_set_header Host $http_host; 40 | proxy_set_header X-Forwarded-Proto $scheme; 41 | proxy_set_header X-Forwarded-For $remote_addr; 42 | proxy_set_header X-Forwarded-Port $server_port; 43 | proxy_set_header X-Request-Start $msec; 44 | } 45 | include {{ $.DOKKU_ROOT }}/{{ $.APP }}/nginx.conf.d/*.conf; 46 | 47 | 48 | } 49 | {{ else if eq $scheme "https"}} 50 | server { 51 | listen [::]:{{ $listen_port }} ssl {{ if eq $.HTTP2_SUPPORTED "true" }}http2{{ else if eq $.SPDY_SUPPORTED "true" }}spdy{{ end }}; 52 | listen {{ $listen_port }} ssl {{ if eq $.HTTP2_SUPPORTED "true" }}http2{{ else if eq $.SPDY_SUPPORTED "true" }}spdy{{ end }}; 53 | {{ if $.SSL_SERVER_NAME }}server_name {{ $.SSL_SERVER_NAME }}; {{ end }} 54 | {{ if $.NOSSL_SERVER_NAME }}server_name {{ $.NOSSL_SERVER_NAME }}; {{ end }} 55 | access_log /var/log/nginx/{{ $.APP }}-access.log; 56 | error_log /var/log/nginx/{{ $.APP }}-error.log; 57 | 58 | ssl_certificate {{ $.APP_SSL_PATH }}/server.crt; 59 | ssl_certificate_key {{ $.APP_SSL_PATH }}/server.key; 60 | ssl_protocols TLSv1.2; 61 | ssl_prefer_server_ciphers on; 62 | 63 | keepalive_timeout 70; 64 | {{ if and (eq $.SPDY_SUPPORTED "true") (ne $.HTTP2_SUPPORTED "true") }}add_header Alternate-Protocol {{ $.NGINX_SSL_PORT }}:npn-spdy/2;{{ end }} 65 | 66 | 67 | location / { 68 | 69 | gzip on; 70 | gzip_min_length 1100; 71 | gzip_buffers 4 32k; 72 | gzip_types text/css text/javascript text/xml text/plain text/x-component application/javascript application/x-javascript application/json application/xml application/rss+xml font/truetype application/x-font-ttf font/opentype application/vnd.ms-fontobject image/svg+xml; 73 | gzip_vary on; 74 | gzip_comp_level 6; 75 | 76 | ## Start CORS here. 77 | 78 | # Django already returns header 'Access-Control-Allow-Origin' '*' 79 | # add_header 'Access-Control-Allow-Origin' '*'; 80 | add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; 81 | add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; 82 | add_header 'Access-Control-Expose-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Content-Range,Range'; 83 | 84 | 85 | ##End CORS 86 | 87 | proxy_pass http://{{ $.APP }}-{{ $upstream_port }}; 88 | proxy_http_version 1.1; 89 | proxy_set_header Upgrade $http_upgrade; 90 | proxy_set_header Connection "upgrade"; 91 | proxy_set_header Host $http_host; 92 | proxy_set_header X-Forwarded-Proto $scheme; 93 | proxy_set_header X-Forwarded-For $remote_addr; 94 | proxy_set_header X-Forwarded-Port $server_port; 95 | proxy_set_header X-Request-Start $msec; 96 | } 97 | include {{ $.DOKKU_ROOT }}/{{ $.APP }}/nginx.conf.d/*.conf; 98 | 99 | } 100 | {{ end }}{{ end }} 101 | 102 | {{ if $.DOKKU_APP_LISTENERS }} 103 | {{ range $upstream_port := $.PROXY_UPSTREAM_PORTS | split " " }} 104 | upstream {{ $.APP }}-{{ $upstream_port }} { 105 | {{ range $listeners := $.DOKKU_APP_LISTENERS | split " " }} 106 | {{ $listener_list := $listeners | split ":" }} 107 | {{ $listener_ip := index $listener_list 0 }} 108 | server {{ $listener_ip }}:{{ $upstream_port }};{{ end }} 109 | } 110 | {{ end }}{{ end }} 111 | -------------------------------------------------------------------------------- /pylama.ini: -------------------------------------------------------------------------------- 1 | [pylama] 2 | ignore = E501,C901,W0611,W0401 3 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | DJANGO_SETTINGS_MODULE = sensorsafrica.tests.settings 3 | python_files = tests.py test_*.py *_tests.py 4 | addopts = -p no:warnings 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | django==1.11.29 #LTS 2 | coreapi==2.3.3 3 | dj-database-url==0.5.0 4 | timeago==1.0.10 5 | 6 | flower==0.9.5 7 | tornado<6 8 | sentry-sdk==1.14.0 9 | celery==4.3.0 10 | gevent==1.2.2 11 | greenlet==0.4.12 12 | whitenoise==4.1.2 13 | 14 | eradicate==1.0 15 | pytest==5.2.1 16 | 17 | ckanapi==4.1 18 | 19 | celery-slack==0.3.0 20 | 21 | urllib3<2 22 | 23 | django-cors-headers==3.0.2 24 | 25 | geopy==2.1.0 26 | -------------------------------------------------------------------------------- /sensorsafrica/__init__.py: -------------------------------------------------------------------------------- 1 | # This will make sure the app is always imported when 2 | # Django starts so that shared_task will use this app. 3 | from .celeryapp import app as celery_app 4 | 5 | __all__ = ['celery_app'] 6 | -------------------------------------------------------------------------------- /sensorsafrica/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.utils.html import format_html 3 | from django.conf.urls import include, url 4 | from django.template.response import TemplateResponse 5 | from .api.models import LastActiveNodes, SensorDataStat, City 6 | from django.db.models import Q 7 | 8 | from feinstaub.sensors.admin import ( 9 | SensorLocationAdmin, 10 | SensorLocation, 11 | SensorData, 12 | Node, 13 | ) 14 | 15 | import timeago 16 | import datetime 17 | import django.utils.timezone 18 | 19 | 20 | @admin.register(LastActiveNodes) 21 | class LastActiveNodesAdmin(admin.ModelAdmin): 22 | readonly_fields = ["node", "location", "last_data_received_at"] 23 | list_display = ["node", "location", "received", "previous_locations"] 24 | search_fields = ["node", "location", "last_data_received_at"] 25 | list_filter = ["node", "location", "last_data_received_at"] 26 | 27 | def get_queryset(self, request): 28 | return LastActiveNodes.objects.order_by("node_id").distinct("node_id") 29 | 30 | def received(self, obj): 31 | now = datetime.datetime.now(django.utils.timezone.utc) 32 | 33 | if not obj.last_data_received_at: 34 | return "Unknown" 35 | 36 | return "( %s ) %s" % ( 37 | timeago.format(obj.last_data_received_at, now), 38 | obj.last_data_received_at, 39 | ) 40 | 41 | def previous_locations(self, obj): 42 | prev = list( 43 | LastActiveNodes.objects.filter( 44 | Q(node_id=obj.node), ~Q(location_id=obj.location) 45 | ) 46 | ) 47 | return format_html( 48 | """ 49 |
50 |

51 | {} 52 |

53 |

{}

54 |
55 | """, 56 | len(prev), 57 | ", ".join(map(lambda n: n.location.location, prev)) 58 | ) 59 | 60 | def get_actions(self, request): 61 | actions = super(LastActiveNodesAdmin, self).get_actions(request) 62 | del actions["delete_selected"] 63 | return actions 64 | 65 | def has_add_permission(self, request): 66 | return False 67 | 68 | def has_delete_permission(self, request, obj=None): 69 | return False 70 | 71 | def save_model(self, request, obj, form, change): 72 | pass 73 | 74 | def delete_model(self, request, obj): 75 | pass 76 | 77 | def save_related(self, request, form, formsets, change): 78 | pass 79 | 80 | 81 | @admin.register(SensorDataStat) 82 | class SensorDataStatAdmin(admin.ModelAdmin): 83 | readonly_fields = [ 84 | "node", 85 | "sensor", 86 | "location", 87 | "city_slug", 88 | "value_type", 89 | "average", 90 | "maximum", 91 | "minimum", 92 | "timestamp", 93 | ] 94 | search_fields = ["city_slug", "value_type"] 95 | list_display = [ 96 | "node", 97 | "sensor", 98 | "location", 99 | "city_slug", 100 | "value_type", 101 | "average", 102 | "maximum", 103 | "minimum", 104 | "timestamp", 105 | "created", 106 | "modified", 107 | ] 108 | list_filter = ["timestamp", "node", "sensor", "location"] 109 | 110 | def get_actions(self, request): 111 | actions = super(SensorDataStatAdmin, self).get_actions(request) 112 | del actions["delete_selected"] 113 | return actions 114 | 115 | def has_add_permission(self, request): 116 | return False 117 | 118 | def has_delete_permission(self, request, obj=None): 119 | return False 120 | 121 | def save_model(self, request, obj, form, change): 122 | pass 123 | 124 | def delete_model(self, request, obj): 125 | pass 126 | 127 | def save_related(self, request, form, formsets, change): 128 | pass 129 | 130 | 131 | @admin.register(City) 132 | class CityAdmin(admin.ModelAdmin): 133 | search_fields = ["slug", "name", "country"] 134 | list_display = ["slug", "name", "country", "latitude", "longitude"] 135 | list_filter = ["name", "country"] 136 | -------------------------------------------------------------------------------- /sensorsafrica/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/api/__init__.py -------------------------------------------------------------------------------- /sensorsafrica/api/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django_extensions.db.models import TimeStampedModel 3 | from feinstaub.sensors.models import Node, Sensor, SensorLocation 4 | from django.utils.text import slugify 5 | 6 | 7 | class City(TimeStampedModel): 8 | slug = models.CharField(max_length=255, db_index=True, null=False, blank=False) 9 | name = models.CharField(max_length=255, db_index=True, null=False, blank=False) 10 | country = models.CharField(max_length=255, db_index=True, null=False, blank=False) 11 | location = models.CharField(max_length=255, db_index=True, null=False, blank=False) 12 | latitude = models.DecimalField(max_digits=14, decimal_places=11, null=True, blank=True) 13 | longitude = models.DecimalField(max_digits=14, decimal_places=11, null=True, blank=True) 14 | 15 | class Meta: 16 | verbose_name_plural = "Cities" 17 | 18 | def save(self, *args, **kwargs): 19 | self.slug = slugify(self.name) 20 | return super(City, self).save(*args, **kwargs) 21 | 22 | 23 | class SensorDataStat(TimeStampedModel): 24 | node = models.ForeignKey(Node) 25 | sensor = models.ForeignKey(Sensor) 26 | location = models.ForeignKey(SensorLocation) 27 | 28 | city_slug = models.CharField(max_length=255, db_index=True, null=False, blank=False) 29 | value_type = models.CharField(max_length=255, db_index=True, null=False, blank=False) 30 | 31 | average = models.FloatField(null=False, blank=False) 32 | maximum = models.FloatField(null=False, blank=False) 33 | minimum = models.FloatField(null=False, blank=False) 34 | 35 | # Number of data points averaged 36 | sample_size = models.IntegerField(null=False, blank=False) 37 | # Last datetime of calculated stats 38 | last_datetime = models.DateTimeField() 39 | 40 | timestamp = models.DateTimeField() 41 | 42 | def __str__(self): 43 | return "%s %s %s avg=%s min=%s max=%s" % ( 44 | self.timestamp, 45 | self.city_slug, 46 | self.value_type, 47 | self.average, 48 | self.minimum, 49 | self.maximum, 50 | ) 51 | 52 | 53 | class LastActiveNodes(TimeStampedModel): 54 | node = models.ForeignKey(Node) 55 | location = models.ForeignKey(SensorLocation) 56 | last_data_received_at = models.DateTimeField() 57 | 58 | class Meta: 59 | unique_together = ['node', 'location'] 60 | -------------------------------------------------------------------------------- /sensorsafrica/api/v1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/api/v1/__init__.py -------------------------------------------------------------------------------- /sensorsafrica/api/v1/filters.py: -------------------------------------------------------------------------------- 1 | import django_filters 2 | from django.db import models 3 | 4 | from feinstaub.sensors.models import Node, SensorData 5 | 6 | class NodeFilter(django_filters.FilterSet): 7 | class Meta: 8 | model = Node 9 | fields = { 10 | "location__country": ["exact"], 11 | "location__city": ["exact"], 12 | "last_notify": ["exact", "gte", "lte"]} 13 | filter_overrides = { 14 | models.DateTimeField: { 15 | 'filter_class': django_filters.IsoDateTimeFilter, 16 | }, 17 | } 18 | 19 | 20 | class SensorFilter(django_filters.FilterSet): 21 | class Meta: 22 | model = SensorData 23 | fields = { 24 | "sensor": ["exact"], 25 | "timestamp": ["exact", "gte", "lte"] 26 | } 27 | filter_overrides = { 28 | models.DateTimeField: { 29 | 'filter_class': django_filters.IsoDateTimeFilter, 30 | }, 31 | } 32 | -------------------------------------------------------------------------------- /sensorsafrica/api/v1/router.py: -------------------------------------------------------------------------------- 1 | # The base version is entirely based on feinstaub 2 | from feinstaub.main.views import UsersView 3 | from feinstaub.sensors.views import ( 4 | SensorView, 5 | StatisticsView, 6 | SensorDataView, 7 | ) 8 | 9 | from .views import ( 10 | FilterView, 11 | NodeView, 12 | NowView, 13 | PostSensorDataView, 14 | SensorsAfricaSensorDataView, 15 | VerboseSensorDataView, 16 | ) 17 | 18 | from rest_framework import routers 19 | 20 | router = routers.DefaultRouter() 21 | router.register(r"node", NodeView) 22 | router.register(r"sensor", SensorView) 23 | router.register(r"data", VerboseSensorDataView) 24 | router.register(r"statistics", StatisticsView, basename="statistics") 25 | router.register(r"now", NowView) 26 | router.register(r"user", UsersView) 27 | router.register( 28 | r"sensors/(?P\d+)", SensorsAfricaSensorDataView, basename="sensors" 29 | ) 30 | router.register(r"filter", FilterView, basename="filter") 31 | 32 | api_urls = router.urls 33 | 34 | push_sensor_data_router = routers.DefaultRouter() 35 | push_sensor_data_router.register(r"push-sensor-data", PostSensorDataView) 36 | 37 | push_sensor_data_urls = push_sensor_data_router.urls 38 | -------------------------------------------------------------------------------- /sensorsafrica/api/v1/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | from feinstaub.sensors.models import ( 3 | Node, 4 | SensorData, 5 | SensorDataValue, 6 | SensorLocation 7 | ) 8 | from feinstaub.sensors.serializers import ( 9 | NestedSensorLocationSerializer, 10 | NestedSensorSerializer, 11 | SensorDataSerializer as PostSensorDataSerializer 12 | ) 13 | 14 | class NodeLocationSerializer(NestedSensorLocationSerializer): 15 | class Meta(NestedSensorLocationSerializer.Meta): 16 | fields = NestedSensorLocationSerializer.Meta.fields + ("latitude", "longitude", "city") 17 | 18 | class NodeSerializer(serializers.ModelSerializer): 19 | sensors = NestedSensorSerializer(many=True) 20 | location = NodeLocationSerializer() 21 | 22 | class Meta: 23 | model = Node 24 | fields = ('id', 'sensors', 'uid', 'owner', 'location', 'last_notify') 25 | 26 | class SensorLocationSerializer(serializers.ModelSerializer): 27 | class Meta: 28 | model = SensorLocation 29 | fields = '__all__' 30 | 31 | 32 | class SensorDataValueSerializer(serializers.ModelSerializer): 33 | class Meta: 34 | model = SensorDataValue 35 | fields = ['value_type', 'value'] 36 | 37 | 38 | class SensorDataSerializer(serializers.ModelSerializer): 39 | sensordatavalues = SensorDataValueSerializer(many=True) 40 | location = SensorLocationSerializer() 41 | 42 | class Meta: 43 | model = SensorData 44 | fields = ['location', 'timestamp', 'sensordatavalues'] 45 | 46 | class LastNotifySensorDataSerializer(PostSensorDataSerializer): 47 | 48 | def create(self, validated_data): 49 | sd = super().create(validated_data) 50 | # use node from authenticator 51 | successful_authenticator = self.context['request'].successful_authenticator 52 | node, pin = successful_authenticator.authenticate(self.context['request']) 53 | 54 | #sometimes we post historical data (eg: from other network) 55 | #this means we have to update last_notify only if current timestamp is greater than what's there 56 | if node.last_notify is None or node.last_notify < sd.timestamp: 57 | node.last_notify = sd.timestamp 58 | node.save() 59 | 60 | return sd 61 | -------------------------------------------------------------------------------- /sensorsafrica/api/v1/views.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import pytz 3 | import json 4 | import django_filters 5 | 6 | 7 | from django.conf import settings 8 | from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q 9 | from django.db.models.functions import Cast, TruncDate 10 | from dateutil.relativedelta import relativedelta 11 | from django.utils import timezone 12 | from rest_framework import mixins, pagination, viewsets 13 | from rest_framework.authentication import SessionAuthentication, TokenAuthentication 14 | from rest_framework.exceptions import ValidationError 15 | from rest_framework.permissions import IsAuthenticatedOrReadOnly 16 | 17 | from feinstaub.sensors.models import Node, SensorData 18 | from feinstaub.sensors.serializers import NowSerializer 19 | from feinstaub.sensors.views import SensorDataView, StandardResultsSetPagination 20 | from feinstaub.sensors.authentication import NodeUidAuthentication 21 | 22 | from .filters import NodeFilter, SensorFilter 23 | from .serializers import LastNotifySensorDataSerializer, NodeSerializer, SensorDataSerializer 24 | 25 | class FilterView(mixins.ListModelMixin, viewsets.GenericViewSet): 26 | serializer_class = SensorDataSerializer 27 | 28 | def get_queryset(self): 29 | sensor_type = self.request.GET.get("type", r"\w+") 30 | country = self.request.GET.get("country", r"\w+") 31 | city = self.request.GET.get("city", r"\w+") 32 | return ( 33 | SensorData.objects.filter( 34 | timestamp__gte=timezone.now() - datetime.timedelta(minutes=5), 35 | sensor__sensor_type__uid__iregex=sensor_type, 36 | location__country__iregex=country, 37 | location__city__iregex=city, 38 | ) 39 | .only("sensor", "timestamp") 40 | .prefetch_related("sensordatavalues") 41 | ) 42 | 43 | 44 | class NodeView( 45 | mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet 46 | ): 47 | """Show all nodes belonging to authenticated user""" 48 | 49 | authentication_classes = [SessionAuthentication, TokenAuthentication] 50 | pagination_class = StandardResultsSetPagination 51 | permission_classes = [IsAuthenticatedOrReadOnly] 52 | queryset = Node.objects.none() 53 | serializer_class = NodeSerializer 54 | filter_class = NodeFilter 55 | 56 | def get_queryset(self): 57 | if self.request.user.is_authenticated: 58 | if self.request.user.groups.filter(name="show_me_everything").exists(): 59 | return Node.objects.all() 60 | 61 | return Node.objects.filter( 62 | Q(owner=self.request.user) 63 | | Q( 64 | owner__groups__name__in=[ 65 | g.name for g in self.request.user.groups.all() 66 | ] 67 | ) 68 | ) 69 | 70 | return Node.objects.none() 71 | 72 | 73 | class NowView(mixins.ListModelMixin, viewsets.GenericViewSet): 74 | """Show all public sensors active in the last 5 minutes with newest value""" 75 | 76 | permission_classes = [] 77 | serializer_class = NowSerializer 78 | queryset = SensorData.objects.none() 79 | 80 | def get_queryset(self): 81 | now = timezone.now() 82 | startdate = now - datetime.timedelta(minutes=5) 83 | return SensorData.objects.filter( 84 | sensor__public=True, modified__range=[startdate, now] 85 | ) 86 | 87 | class PostSensorDataView(mixins.CreateModelMixin, 88 | viewsets.GenericViewSet): 89 | """ This endpoint is to POST data from the sensor to the api. 90 | """ 91 | authentication_classes = (NodeUidAuthentication,) 92 | permission_classes = tuple() 93 | serializer_class = LastNotifySensorDataSerializer 94 | queryset = SensorData.objects.all() 95 | 96 | 97 | class VerboseSensorDataView(SensorDataView): 98 | filter_class = SensorFilter 99 | 100 | class SensorsAfricaSensorDataView(mixins.ListModelMixin, viewsets.GenericViewSet): 101 | serializer_class = SensorDataSerializer 102 | 103 | def get_queryset(self): 104 | return ( 105 | SensorData.objects.filter( 106 | timestamp__gte=timezone.now() - datetime.timedelta(minutes=5), 107 | sensor=self.kwargs["sensor_id"], 108 | ) 109 | .only("sensor", "timestamp") 110 | .prefetch_related("sensordatavalues") 111 | ) 112 | 113 | -------------------------------------------------------------------------------- /sensorsafrica/api/v2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/api/v2/__init__.py -------------------------------------------------------------------------------- /sensorsafrica/api/v2/filters.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | import django_filters 3 | from feinstaub.sensors.views import SensorFilter 4 | 5 | class CustomSensorFilter(SensorFilter): 6 | class Meta(SensorFilter.Meta): 7 | fields = {"sensor": ["exact"], 8 | "sensor__public": ["exact"], 9 | "location__country": ['exact'], 10 | "location__city": ['exact'], 11 | "timestamp": ("gte", "lte"), 12 | } 13 | filter_overrides = { 14 | models.DateTimeField: { 15 | 'filter_class': django_filters.IsoDateTimeFilter, 16 | }, 17 | } 18 | -------------------------------------------------------------------------------- /sensorsafrica/api/v2/router.py: -------------------------------------------------------------------------------- 1 | from rest_framework import routers 2 | from django.conf.urls import url, include 3 | 4 | from .views import ( 5 | CitiesView, 6 | NodesView, 7 | NowView, 8 | SensorDataStatsView, 9 | SensorDataView, 10 | SensorLocationsView, 11 | SensorTypesView, 12 | SensorsView, 13 | StatisticsView, 14 | meta_data, 15 | ) 16 | 17 | router = routers.DefaultRouter() 18 | router.register(r"data", SensorDataView, basename="sensor-data") 19 | router.register(r"data/stats/(?P[air]+)", SensorDataStatsView, basename="sensor-data-stats") 20 | router.register(r"cities", CitiesView, basename="cities") 21 | router.register(r"nodes", NodesView, basename="nodes") 22 | router.register(r"now", NowView, basename="now") 23 | router.register(r"locations", SensorLocationsView, basename="sensor-locations") 24 | router.register(r"sensors", SensorsView, basename="sensors") 25 | router.register(r"sensor-types", SensorTypesView, basename="sensor-types") 26 | router.register(r"statistics", StatisticsView, basename="statistics") 27 | 28 | api_urls = [ 29 | url(r"^", include(router.urls)), 30 | url(r"^meta/", meta_data), 31 | ] 32 | -------------------------------------------------------------------------------- /sensorsafrica/api/v2/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | from feinstaub.sensors.serializers import ( 3 | NestedSensorLocationSerializer, 4 | ) 5 | from feinstaub.sensors.models import Node, Sensor, SensorType, SensorLocation 6 | from feinstaub.sensors.serializers import (VerboseSensorDataSerializer, ) 7 | 8 | 9 | class SensorDataStatSerializer(serializers.Serializer): 10 | calculated_average = serializers.FloatField() 11 | calculated_minimum = serializers.FloatField() 12 | calculated_maximum = serializers.FloatField() 13 | value_type = serializers.CharField(max_length=200) 14 | start_datetime = serializers.DateTimeField() 15 | end_datetime = serializers.DateTimeField() 16 | city_slug = serializers.CharField(max_length=200) 17 | 18 | 19 | class CitySerializer(serializers.Serializer): 20 | latitude = serializers.DecimalField(max_digits=14, decimal_places=11) 21 | longitude = serializers.DecimalField(max_digits=14, decimal_places=11) 22 | slug = serializers.CharField(max_length=255) 23 | name = serializers.CharField(max_length=255) 24 | country = serializers.CharField(max_length=255) 25 | label = serializers.SerializerMethodField() 26 | 27 | def get_label(self, obj): 28 | return "{}, {}".format(obj.name, obj.country) 29 | 30 | 31 | class SensorSerializer(serializers.ModelSerializer): 32 | class Meta: 33 | model = Sensor 34 | fields = ("id", "node", "description", "pin", "sensor_type", "public") 35 | 36 | 37 | class SensorLocationSerializer(NestedSensorLocationSerializer): 38 | class Meta(NestedSensorLocationSerializer.Meta): 39 | fields = NestedSensorLocationSerializer.Meta.fields + ( 40 | "longitude", 41 | "latitude", 42 | "altitude", 43 | "street_name", 44 | "street_number", 45 | "city", 46 | "country", 47 | "postalcode", 48 | "traffic_in_area", 49 | "oven_in_area", 50 | "industry_in_area", 51 | "owner", 52 | ) 53 | 54 | 55 | class SensorTypeSerializer(serializers.ModelSerializer): 56 | class Meta: 57 | model = SensorType 58 | fields = ("id", "uid", "name", "manufacturer") 59 | 60 | 61 | class NodeSerializer(serializers.ModelSerializer): 62 | class Meta: 63 | model = Node 64 | fields = ( 65 | "id", 66 | "uid", 67 | "owner", 68 | "location", 69 | "name", 70 | "description", 71 | "height", 72 | "sensor_position", 73 | "email", 74 | "last_notify", 75 | "indoor", 76 | "inactive", 77 | "exact_location", 78 | ) 79 | 80 | class SensorDataSensorLocationSerializer(serializers.ModelSerializer): 81 | class Meta: 82 | model = SensorLocation 83 | fields = ('id', "country", ) 84 | 85 | class SensorDataSerializer(VerboseSensorDataSerializer): 86 | location = SensorDataSensorLocationSerializer() 87 | -------------------------------------------------------------------------------- /sensorsafrica/api/v2/views.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import django_filters 3 | import pytz 4 | import json 5 | 6 | from dateutil.relativedelta import relativedelta 7 | 8 | from django.contrib.auth.models import User 9 | from django.conf import settings 10 | from django.utils import timezone 11 | from django.db import connection 12 | from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q, Count, Prefetch 13 | from django.db.models.functions import Cast, TruncHour, TruncDay, TruncMonth 14 | from django.utils.decorators import method_decorator 15 | from django.utils.text import slugify 16 | from django.views.decorators.cache import cache_page 17 | 18 | from rest_framework import mixins, pagination, viewsets 19 | from rest_framework.authentication import SessionAuthentication, TokenAuthentication 20 | from rest_framework.decorators import action 21 | from rest_framework.exceptions import ValidationError 22 | from rest_framework.response import Response 23 | from rest_framework.permissions import IsAuthenticated, AllowAny 24 | from rest_framework.decorators import api_view, authentication_classes 25 | 26 | from feinstaub.sensors.views import SensorFilter, StandardResultsSetPagination 27 | from feinstaub.sensors.serializers import NowSerializer 28 | from feinstaub.sensors.models import ( 29 | Node, 30 | Sensor, 31 | SensorData, 32 | SensorDataValue, 33 | SensorLocation, 34 | SensorType, 35 | ) 36 | 37 | 38 | from ..models import City, LastActiveNodes, SensorDataStat 39 | from .serializers import ( 40 | SensorDataStatSerializer, 41 | CitySerializer, 42 | SensorTypeSerializer, 43 | NodeSerializer, 44 | SensorSerializer, 45 | SensorLocationSerializer, 46 | SensorDataSerializer, 47 | ) 48 | 49 | from .filters import CustomSensorFilter 50 | 51 | value_types = {"air": ["P1", "P2", "humidity", "temperature"]} 52 | 53 | 54 | def beginning_of_today(): 55 | return timezone.now().replace(hour=0, minute=0, second=0, microsecond=0) 56 | 57 | 58 | def end_of_today(): 59 | return beginning_of_today() + datetime.timedelta(hours=24) 60 | 61 | 62 | def beginning_of_day(from_date): 63 | return datetime.datetime.strptime(from_date, "%Y-%m-%d").replace(tzinfo=pytz.UTC) 64 | 65 | 66 | def end_of_day(to_date): 67 | return beginning_of_day(to_date) + datetime.timedelta(hours=24) 68 | 69 | 70 | def validate_date(date_text, error): 71 | try: 72 | datetime.datetime.strptime(date_text, "%Y-%m-%d") 73 | except ValueError: 74 | raise ValidationError(error) 75 | 76 | 77 | class CustomPagination(pagination.PageNumberPagination): 78 | page_size_query_param = "page_size" 79 | max_page_size = 1000 80 | page_size = 100 81 | 82 | def get_paginated_response(self, data_stats): 83 | # If filtering from a date 84 | # We will need to have a list of the value_types e.g. { 'P1': [{}, {}] } 85 | from_date = self.request.query_params.get("from", None) 86 | interval = self.request.query_params.get("interval", None) 87 | 88 | results = {} 89 | for data_stat in data_stats: 90 | city_slug = data_stat["city_slug"] 91 | value_type = data_stat["value_type"] 92 | 93 | if city_slug not in results: 94 | results[city_slug] = { 95 | "city_slug": city_slug, 96 | value_type: [] if from_date or interval else {}, 97 | } 98 | 99 | if value_type not in results[city_slug]: 100 | results[city_slug][value_type] = [] if from_date or interval else {} 101 | 102 | values = results[city_slug][value_type] 103 | include_result = getattr( 104 | values, "append" if from_date or interval else "update" 105 | ) 106 | include_result( 107 | { 108 | "average": data_stat["calculated_average"], 109 | "minimum": data_stat["calculated_minimum"], 110 | "maximum": data_stat["calculated_maximum"], 111 | "start_datetime": data_stat["start_datetime"], 112 | "end_datetime": data_stat["end_datetime"], 113 | } 114 | ) 115 | 116 | return Response( 117 | { 118 | "next": self.get_next_link(), 119 | "previous": self.get_previous_link(), 120 | "count": len(results.keys()), 121 | "results": list(results.values()), 122 | } 123 | ) 124 | 125 | 126 | class CitiesView(mixins.ListModelMixin, viewsets.GenericViewSet): 127 | queryset = City.objects.all() 128 | serializer_class = CitySerializer 129 | pagination_class = StandardResultsSetPagination 130 | 131 | 132 | class NodesView(viewsets.ViewSet): 133 | """Create and list nodes, with the option to list authenticated user's nodes.""" 134 | authentication_classes = [SessionAuthentication, TokenAuthentication] 135 | permission_classes = [IsAuthenticated] 136 | 137 | # Note: Allow access to list_nodes for https://v2.map.aq.sensors.africa/#4/-4.46/19.54 138 | @action(detail=False, methods=["get"], url_path="list-nodes", url_name="list_nodes", permission_classes=[AllowAny]) 139 | def list_nodes(self, request): 140 | """List all public nodes with active sensors.""" 141 | now = datetime.datetime.now() 142 | one_year_ago = now - datetime.timedelta(days=365) 143 | 144 | last_active_nodes = ( 145 | LastActiveNodes.objects.filter(last_data_received_at__gte=one_year_ago) 146 | .select_related("node", "location") 147 | .prefetch_related( 148 | Prefetch( 149 | "node__sensors", 150 | queryset=Sensor.objects.filter(public=True), 151 | ) 152 | ) 153 | ) 154 | 155 | nodes = [] 156 | 157 | # Loop through the last active nodes 158 | for last_active in last_active_nodes: 159 | # Get the current node only if it has public sensors 160 | node = last_active.node 161 | if not node.sensors.exists(): 162 | continue 163 | 164 | # The last acive date 165 | last_data_received_at = last_active.last_data_received_at 166 | 167 | # last_data_received_at 168 | stats = [] 169 | # Get data stats from 5mins before last_data_received_at 170 | if last_data_received_at: 171 | last_5_mins = last_data_received_at - datetime.timedelta(minutes=5) 172 | stats = ( 173 | SensorDataValue.objects.filter( 174 | Q(sensordata__sensor__node=node.id), 175 | # Open endpoints should return data from public sensors 176 | # only in case a node has both public & private sensors 177 | Q(sensordata__sensor__public=True), 178 | Q(sensordata__location=last_active.location.id), 179 | Q(sensordata__timestamp__gte=last_5_mins), 180 | Q(sensordata__timestamp__lte=last_data_received_at), 181 | # Ignore timestamp values 182 | ~Q(value_type="timestamp"), 183 | # Match only valid float text 184 | Q(value__regex=r"^\-?\d+(\.?\d+)?$"), 185 | ) 186 | .values("value_type") 187 | .annotate( 188 | sensor_id=F("sensordata__sensor__id"), 189 | start_datetime=Min("sensordata__timestamp"), 190 | end_datetime=Max("sensordata__timestamp"), 191 | average=Avg(Cast("value", FloatField())), 192 | minimum=Min(Cast("value", FloatField())), 193 | maximum=Max(Cast("value", FloatField())), 194 | ) 195 | ) 196 | 197 | # If the last_active node location is not same as current node location 198 | # then the node has moved locations since it was last active 199 | moved_to = None 200 | if last_active.location.id != node.location.id: 201 | moved_to = { 202 | "name": node.location.location, 203 | "longitude": node.location.longitude, 204 | "latitude": node.location.latitude, 205 | "city": { 206 | "name": node.location.city, 207 | "slug": slugify(node.location.city), 208 | }, 209 | } 210 | 211 | nodes.append( 212 | { 213 | "node_moved": moved_to is not None, 214 | "moved_to": moved_to, 215 | "node": {"uid": last_active.node.uid, "id": last_active.node.id, "owner": last_active.node.owner.id}, 216 | "location": { 217 | "name": last_active.location.location, 218 | "longitude": last_active.location.longitude, 219 | "latitude": last_active.location.latitude, 220 | "city": { 221 | "name": last_active.location.city, 222 | "slug": slugify(last_active.location.city), 223 | }, 224 | }, 225 | "last_data_received_at": last_data_received_at, 226 | "stats": stats, 227 | } 228 | ) 229 | 230 | return Response(nodes) 231 | 232 | @action(detail=False, methods=["get"], url_path="my-nodes", url_name="my_nodes") 233 | def list_my_nodes(self, request): 234 | """List only the nodes owned by the authenticated user.""" 235 | if request.user.is_authenticated: 236 | queryset = Node.objects.filter( 237 | Q(owner=request.user) 238 | | Q( 239 | owner__groups__name__in=[ 240 | g.name for g in request.user.groups.all() 241 | ] 242 | ) 243 | ) 244 | serializer = NodeSerializer(queryset, many=True) 245 | return Response(serializer.data) 246 | return Response({"detail": "Authentication credentials were not provided."}, status=403) 247 | 248 | @action(detail=False, methods=["post"], url_path="register-node", url_name="register_node") 249 | def register_node(self, request): 250 | serializer = NodeSerializer(data=request.data) 251 | if serializer.is_valid(): 252 | serializer.save() 253 | return Response(serializer.data, status=201) 254 | 255 | return Response(serializer.errors, status=400) 256 | 257 | 258 | class SensorDataPagination(pagination.CursorPagination): 259 | cursor_query_param = "next_page" 260 | ordering = "-timestamp" 261 | page_size = 100 262 | 263 | 264 | class SensorDataView( 265 | mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet 266 | ): 267 | """ 268 | View for retrieving and downloading detailed sensor data records, with access controlled based on 269 | user permissions and ownership. 270 | 271 | This endpoint allows authenticated users to retrieve sensor data records, with the following access rules: 272 | - Users in the `show_me_everything` group have access to all sensor data records. 273 | - Other users can access data from sensors they own, sensors owned by members of their groups, or public sensors. 274 | - Non-authenticated users can only access public sensor data. 275 | """ 276 | 277 | 278 | 279 | authentication_classes = [SessionAuthentication, TokenAuthentication] 280 | queryset = SensorData.objects.all() 281 | pagination_class = SensorDataPagination 282 | permission_classes = [IsAuthenticated] 283 | filter_backends = (django_filters.rest_framework.DjangoFilterBackend,) 284 | filter_class = CustomSensorFilter 285 | serializer_class = SensorDataSerializer 286 | 287 | def get_queryset(self): 288 | if self.request.user.is_authenticated: 289 | if self.request.user.groups.filter(name="show_me_everything").exists(): 290 | return SensorData.objects.all() 291 | 292 | # Return data from sensors owned or 293 | # owned by someone in the same group as requesting user or 294 | # public sensors 295 | return SensorData.objects.filter( 296 | Q(sensor__node__owner=self.request.user) 297 | | Q(sensor__node__owner__groups__name__in=[g.name for g in self.request.user.groups.all()]) 298 | | Q(sensor__public=True) 299 | ) 300 | 301 | return SensorData.objects.filter(sensor__public=True) 302 | 303 | 304 | class SensorDataStatsView(mixins.ListModelMixin, viewsets.GenericViewSet): 305 | """ 306 | View to retrieve summarized statistics for specific sensor types (e.g., air quality) within a defined date range, 307 | filtered by city and grouped by specified intervals (hourly, daily, or monthly). 308 | 309 | **URL Parameters:** 310 | - `sensor_type` (str): The type of sensor data to retrieve (e.g., air_quality). 311 | 312 | **Query Parameters:** 313 | - `city` (str, optional): Comma-separated list of city slugs to filter data by location. 314 | - `from` (str, optional): Start date in "YYYY-MM-DD" format. Required if `to` is specified. 315 | - `to` (str, optional): End date in "YYYY-MM-DD" format. Defaults to 24 hours before `to_date` if unspecified. 316 | - `interval` (str, optional): Aggregation interval for results - either "hour", "day", or "month". Defaults to "day". 317 | - `value_type` (str, optional): Comma-separated list of value types to filter (e.g., "PM2.5, PM10"). 318 | 319 | **Caching:** 320 | - Results are cached for 1 hour (`@cache_page(3600)`) to reduce server load. 321 | 322 | **Returns:** 323 | - A list of sensor data statistics, grouped by city, value type, and specified interval. 324 | - Each entry includes: 325 | - `value_type` (str): Type of sensor value (e.g., PM2.5). 326 | - `city_slug` (str): City identifier. 327 | - `truncated_timestamp` (datetime): Timestamp truncated to the specified interval. 328 | - `start_datetime` (datetime): Start of the aggregated time period. 329 | - `end_datetime` (datetime): End of the aggregated time period. 330 | - `calculated_average` (float): Weighted average of sensor values. 331 | - `calculated_minimum` (float): Minimum recorded value within the period. 332 | - `calculated_maximum` (float): Maximum recorded value within the period. 333 | """ 334 | queryset = SensorDataStat.objects.none() 335 | serializer_class = SensorDataStatSerializer 336 | pagination_class = CustomPagination 337 | authentication_classes = [SessionAuthentication, TokenAuthentication] 338 | permission_classes = [IsAuthenticated] 339 | 340 | @method_decorator(cache_page(3600)) 341 | def dispatch(self, request, *args, **kwargs): 342 | return super().dispatch(request, *args, **kwargs) 343 | 344 | def get_queryset(self): 345 | sensor_type = self.kwargs["sensor_type"] 346 | 347 | city_slugs = self.request.query_params.get("city", None) 348 | from_date = self.request.query_params.get("from", None) 349 | to_date = self.request.query_params.get("to", None) 350 | interval = self.request.query_params.get("interval", None) 351 | 352 | if to_date and not from_date: 353 | raise ValidationError({"from": "Must be provide along with to query"}) 354 | if from_date: 355 | validate_date(from_date, {"from": "Must be a date in the format Y-m-d."}) 356 | if to_date: 357 | validate_date(to_date, {"to": "Must be a date in the format Y-m-d."}) 358 | 359 | value_type_to_filter = self.request.query_params.get("value_type", None) 360 | 361 | filter_value_types = value_types[sensor_type] 362 | if value_type_to_filter: 363 | filter_value_types = set(value_type_to_filter.upper().split(",")) & set( 364 | [x.upper() for x in value_types[sensor_type]] 365 | ) 366 | 367 | if not from_date and not to_date: 368 | to_date = timezone.now().replace(minute=0, second=0, microsecond=0) 369 | from_date = to_date - datetime.timedelta(hours=24) 370 | interval = "day" if not interval else interval 371 | elif not to_date: 372 | from_date = beginning_of_day(from_date) 373 | # Get data from_date until the end 374 | # of day yesterday which is the beginning of today 375 | to_date = beginning_of_today() 376 | else: 377 | from_date = beginning_of_day(from_date) 378 | to_date = end_of_day(to_date) 379 | 380 | queryset = SensorDataStat.objects.filter( 381 | value_type__in=filter_value_types, 382 | timestamp__gte=from_date, 383 | timestamp__lte=to_date, 384 | ) 385 | 386 | if interval == "month": 387 | truncate = TruncMonth("timestamp") 388 | elif interval == "day": 389 | truncate = TruncDay("timestamp") 390 | else: 391 | truncate = TruncHour("timestamp") 392 | 393 | if city_slugs: 394 | queryset = queryset.filter(city_slug__in=city_slugs.split(",")) 395 | 396 | return ( 397 | queryset.values("value_type", "city_slug") 398 | .annotate( 399 | truncated_timestamp=truncate, 400 | start_datetime=Min("timestamp"), 401 | end_datetime=Max("timestamp"), 402 | calculated_average=ExpressionWrapper( 403 | Sum(F("average") * F("sample_size")) / Sum("sample_size"), 404 | output_field=FloatField(), 405 | ), 406 | calculated_minimum=Min("minimum"), 407 | calculated_maximum=Max("maximum"), 408 | ) 409 | .values( 410 | "value_type", 411 | "city_slug", 412 | "truncated_timestamp", 413 | "start_datetime", 414 | "end_datetime", 415 | "calculated_average", 416 | "calculated_minimum", 417 | "calculated_maximum", 418 | ) 419 | .order_by("city_slug", "-truncated_timestamp") 420 | ) 421 | 422 | 423 | class SensorLocationsView(viewsets.ViewSet): 424 | """ 425 | View for retrieving and creating sensor entries. 426 | """ 427 | authentication_classes = [SessionAuthentication, TokenAuthentication] 428 | permission_classes = [IsAuthenticated] 429 | pagination_class = StandardResultsSetPagination 430 | 431 | def list(self, request): 432 | queryset = SensorLocation.objects.all() 433 | serializer = SensorLocationSerializer(queryset, many=True) 434 | return Response(serializer.data) 435 | 436 | def create(self, request): 437 | serializer = SensorLocationSerializer(data=request.data) 438 | if serializer.is_valid(): 439 | serializer.save() 440 | return Response(serializer.data, status=201) 441 | 442 | return Response(serializer.errors, status=400) 443 | 444 | 445 | class SensorTypesView(viewsets.ViewSet): 446 | """ 447 | View for retrieving and creating sensor type entries. 448 | """ 449 | authentication_classes = [SessionAuthentication, TokenAuthentication] 450 | permission_classes = [IsAuthenticated] 451 | pagination_class = StandardResultsSetPagination 452 | 453 | def list(self, request): 454 | queryset = SensorType.objects.all() 455 | serializer = SensorTypeSerializer(queryset, many=True) 456 | return Response(serializer.data) 457 | 458 | def create(self, request): 459 | serializer = SensorTypeSerializer(data=request.data) 460 | if serializer.is_valid(): 461 | serializer.save() 462 | return Response(serializer.data, status=201) 463 | 464 | return Response(serializer.errors, status=400) 465 | 466 | 467 | class SensorsView(viewsets.ViewSet): 468 | authentication_classes = [SessionAuthentication, TokenAuthentication] 469 | permission_classes = [IsAuthenticated] 470 | pagination_class = StandardResultsSetPagination 471 | 472 | def get_permissions(self): 473 | if self.action == "create": 474 | permission_classes = [IsAuthenticated] 475 | else: 476 | permission_classes = [AllowAny] 477 | 478 | return [permission() for permission in permission_classes] 479 | 480 | def list(self, request): 481 | queryset = Sensor.objects.all() 482 | serializer = SensorSerializer(queryset, many=True) 483 | return Response(serializer.data) 484 | 485 | def create(self, request): 486 | serializer = SensorSerializer(data=request.data) 487 | if serializer.is_valid(): 488 | serializer.save() 489 | return Response(serializer.data, status=201) 490 | 491 | return Response(serializer.errors, status=400) 492 | 493 | 494 | @api_view(['GET']) 495 | @authentication_classes([SessionAuthentication, TokenAuthentication]) 496 | def meta_data(request): 497 | nodes_count = Node.objects.count() 498 | sensors_count = Sensor.objects.count() 499 | sensor_data_count = SensorData.objects.count() 500 | 501 | database_size = get_database_size() 502 | database_last_updated = get_database_last_updated() 503 | sensors_countries = get_sensors_countries() 504 | sensors_cities = get_sensors_cities() 505 | 506 | return Response({ 507 | "sensor_networks": get_sensors_networks(), 508 | "nodes": { 509 | "active": get_active_nodes(), 510 | "count": nodes_count 511 | }, 512 | "sensors_count": sensors_count, 513 | "sensor_data_count": sensor_data_count, 514 | "sensors_countries": sensors_countries, 515 | "sensors_cities": sensors_cities, 516 | "database_size": database_size[0], 517 | "database_last_updated": database_last_updated, 518 | }) 519 | 520 | def get_active_nodes(): 521 | nodes_count = Node.objects.filter(last_notify__gte=timezone.now() - datetime.timedelta(days=14)).count() 522 | return nodes_count 523 | 524 | def get_sensors_networks(): 525 | user = User.objects.filter(username=settings.NETWORKS_OWNER).first() 526 | if user: 527 | networks = list(user.groups.values_list('name', flat=True)) 528 | networks.append("sensors.AFRICA") 529 | return {"networks": networks, "count": len(networks)} 530 | 531 | def get_sensors_countries(): 532 | sensors_countries = SensorLocation.objects.filter(country__isnull=False).values_list('country', flat=True) 533 | return sorted(set(sensors_countries)) 534 | 535 | def get_sensors_cities(): 536 | sensor_cities = Node.objects.filter(location__city__isnull=False).values_list('location__city', flat=True) 537 | return sorted(set(sensor_cities)) 538 | 539 | def get_database_size(): 540 | with connection.cursor() as c: 541 | c.execute(f"SELECT pg_size_pretty(pg_database_size('{connection.settings_dict['NAME']}'))") 542 | return c.fetchall() 543 | 544 | def get_database_last_updated(): 545 | sensor_data_value = SensorDataValue.objects.latest('created') 546 | if sensor_data_value: 547 | return sensor_data_value.modified 548 | 549 | 550 | class NowView(mixins.ListModelMixin, viewsets.GenericViewSet): 551 | """Show all public sensors active in the last 5 minutes with newest value""" 552 | 553 | authentication_classes = [SessionAuthentication, TokenAuthentication] 554 | permission_classes = [IsAuthenticated] 555 | serializer_class = NowSerializer 556 | 557 | def get_queryset(self): 558 | now = timezone.now() 559 | startdate = now - datetime.timedelta(minutes=5) 560 | return SensorData.objects.filter( 561 | sensor__public=True, modified__range=[startdate, now] 562 | ) 563 | 564 | 565 | class StatisticsView(viewsets.ViewSet): 566 | authentication_classes = [SessionAuthentication, TokenAuthentication] 567 | permission_classes = [IsAuthenticated] 568 | 569 | def list(self, request): 570 | user_count = User.objects.aggregate(count=Count('id'))['count'] 571 | sensor_count = Sensor.objects.aggregate(count=Count('id'))['count'] 572 | sensor_data_count = SensorData.objects.aggregate(count=Count('id'))['count'] 573 | sensor_data_value_count = SensorDataValue.objects.aggregate(count=Count('id'))['count'] 574 | sensor_type_count = SensorType.objects.aggregate(count=Count('id'))['count'] 575 | sensor_type_list = list(SensorType.objects.order_by('uid').values_list('name', flat=True)) 576 | location_count = SensorLocation.objects.aggregate(count=Count('id'))['count'] 577 | 578 | stats = { 579 | 'user': { 580 | 'count': user_count, 581 | }, 582 | 'sensor': { 583 | 'count': sensor_count, 584 | }, 585 | 'sensor_data': { 586 | 'count': sensor_data_count, 587 | }, 588 | 'sensor_data_value': { 589 | 'count': sensor_data_value_count, 590 | }, 591 | 'sensor_type': { 592 | 'count': sensor_type_count, 593 | 'list': sensor_type_list, 594 | }, 595 | 'location': { 596 | 'count': location_count, 597 | } 598 | } 599 | return Response(stats) 600 | -------------------------------------------------------------------------------- /sensorsafrica/celeryapp.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from celery import Celery 4 | from celery_slack import Slackify 5 | 6 | # Set sensorsafrica application settings module for sensorsafrica Celery instance 7 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensorsafrica.settings") 8 | 9 | # Create Celery instance and pass the project name 10 | # The instance is bound to the variable app 11 | app = Celery("sensorsafrica") 12 | 13 | # Pass config made of up values begging with the prefix of CELERY_ in settings.py 14 | app.config_from_object("django.conf:settings", namespace="CELERY") 15 | 16 | # Autodiscover tasks in tasks.py 17 | app.autodiscover_tasks() 18 | 19 | SLACK_WEBHOOK = os.environ.get("SENSORSAFRICA_CELERY_SLACK_WEBHOOK", "") 20 | SLACK_WEBHOOK_FAILURES_ONLY = os.environ.get( 21 | "SENSORSAFRICA_CELERY_SLACK_WEBHOOK_FAILURES_ONLY", "").strip().lower() in ('true', 't', '1') 22 | 23 | options = {'failures_only': SLACK_WEBHOOK_FAILURES_ONLY} 24 | 25 | slack_app = Slackify(app, SLACK_WEBHOOK, **options) 26 | -------------------------------------------------------------------------------- /sensorsafrica/fixtures/auth.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "model": "auth.user", 3 | "pk": 1, 4 | "fields": { 5 | "password": "pbkdf2_sha256$30000$ozmGGJTVhFuD$vaQiWgfUBidJgrWuAGOqUXygnvY3KnKcZHWSaAOquN4=", 6 | "last_login": "2019-01-11T11:04:09.317Z", 7 | "is_superuser": true, 8 | "username": "test", 9 | "first_name": "First Name", 10 | "last_name": "Last Name", 11 | "email": "hello@codeforafrica.org", 12 | "is_staff": true, 13 | "is_active": true, 14 | "date_joined": "2017-07-28T14:53:48Z", 15 | "groups": [], 16 | "user_permissions": [] 17 | } 18 | }] -------------------------------------------------------------------------------- /sensorsafrica/fixtures/sensordata.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "model": "sensors.sensordata", 3 | "pk": 1, 4 | "fields": { 5 | "created": "2019-01-21T08:39:07.720Z", 6 | "modified": "2019-01-21T08:39:07.720Z", 7 | "sampling_rate": null, 8 | "timestamp": "2019-01-21T08:39:07.720Z", 9 | "location": 1, 10 | "software_version": "0.1", 11 | "sensor": 1 12 | } 13 | },{ 14 | "model": "sensors.sensordatavalue", 15 | "pk": 1, 16 | "fields": { 17 | "created": "2019-01-21T08:39:07.720Z", 18 | "modified": "2019-01-21T08:39:07.720Z", 19 | "sensordata": 1, 20 | "value": 28.38, 21 | "value_type": "P2" 22 | } 23 | },{ 24 | "model": "sensors.sensordata", 25 | "pk": 2, 26 | "fields": { 27 | "created": "2019-01-21T08:39:07.720Z", 28 | "modified": "2019-01-21T08:39:07.720Z", 29 | "sampling_rate": null, 30 | "timestamp": "2019-01-21T08:39:07.720Z", 31 | "location": 1, 32 | "software_version": "0.1", 33 | "sensor": 1 34 | } 35 | },{ 36 | "model": "sensors.sensordatavalue", 37 | "pk": 2, 38 | "fields": { 39 | "created": "2019-01-21T08:39:07.720Z", 40 | "modified": "2019-01-21T08:39:07.720Z", 41 | "sensordata": 2, 42 | "value": 28.38, 43 | "value_type": "P2" 44 | } 45 | },{ 46 | "model": "sensors.sensordata", 47 | "pk": 3, 48 | "fields": { 49 | "created": "2019-01-21T08:39:07.720Z", 50 | "modified": "2019-01-21T08:39:07.720Z", 51 | "sampling_rate": null, 52 | "timestamp": "2019-01-21T08:39:07.720Z", 53 | "location": 1, 54 | "software_version": "0.1", 55 | "sensor": 1 56 | } 57 | },{ 58 | "model": "sensors.sensordatavalue", 59 | "pk": 3, 60 | "fields": { 61 | "created": "2019-01-21T08:39:07.720Z", 62 | "modified": "2019-01-21T08:39:07.720Z", 63 | "sensordata": 3, 64 | "value": 30, 65 | "value_type": "P2" 66 | } 67 | }] -------------------------------------------------------------------------------- /sensorsafrica/fixtures/sensors.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "model": "sensors.node", 3 | "pk": 1, 4 | "fields": { 5 | "created": "2018-12-18T08:39:07.720Z", 6 | "modified": "2018-12-18T08:39:07.720Z", 7 | "uid": "esp8266-mobile001", 8 | "owner": 1, 9 | "name": "Mobile", 10 | "description": "Mobile sensor 001", 11 | "height": 1, 12 | "sensor_position": 1, 13 | "location": 1, 14 | "email": "", 15 | "description_internal": "" 16 | } 17 | }, { 18 | "model": "sensors.sensor", 19 | "pk": 1, 20 | "fields": { 21 | "created": "2017-07-31T09:13:05.613Z", 22 | "modified": "2017-07-31T09:21:52.871Z", 23 | "node": 1, 24 | "pin": "1", 25 | "sensor_type": 1, 26 | "description": "", 27 | "public": true 28 | } 29 | }, { 30 | "model": "sensors.sensorlocation", 31 | "pk": 1, 32 | "fields": { 33 | "created": "2019-01-14T09:22:48.579Z", 34 | "modified": "2019-01-14T09:22:48.579Z", 35 | "location": "Syokimau- Bustani Villas 12", 36 | "latitude": "-1.36407000000", 37 | "longitude": "36.91406000000", 38 | "indoor": false, 39 | "street_name": "Mombasa Road", 40 | "street_number": "", 41 | "postalcode": "", 42 | "city": "Nairobi", 43 | "country": "Kenya", 44 | "traffic_in_area": 3, 45 | "oven_in_area": 1, 46 | "industry_in_area": 5, 47 | "owner": 1, 48 | "description": "", 49 | "timestamp": "2019-01-14T09:19:33Z" 50 | } 51 | }] -------------------------------------------------------------------------------- /sensorsafrica/fixtures/sensortypes.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "model": "sensors.sensortype", 3 | "pk": 1, 4 | "fields": { 5 | "created": "2017-07-31T09:07:51.904Z", 6 | "modified": "2018-01-15T08:05:22.696Z", 7 | "uid": "sds011", 8 | "name": "SDS011", 9 | "manufacturer": "Nova Fiteness", 10 | "description": "" 11 | } 12 | }, { 13 | "model": "sensors.sensortype", 14 | "pk": 2, 15 | "fields": { 16 | "created": "2017-07-31T09:08:37.703Z", 17 | "modified": "2017-07-31T09:08:37.703Z", 18 | "uid": "dht22", 19 | "name": "DHT22", 20 | "manufacturer": "Ada Fruit", 21 | "description": "" 22 | } 23 | }, { 24 | "model": "sensors.sensortype", 25 | "pk": 3, 26 | "fields": { 27 | "created": "2017-07-31T10:43:30.683Z", 28 | "modified": "2017-07-31T10:43:30.683Z", 29 | "uid": "mq7", 30 | "name": "MQ-7", 31 | "manufacturer": "HANWEI ELECTRONICS", 32 | "description": "" 33 | } 34 | }, { 35 | "model": "sensors.sensortype", 36 | "pk": 4, 37 | "fields": { 38 | "created": "2017-07-31T10:44:31.307Z", 39 | "modified": "2017-07-31T10:44:31.307Z", 40 | "uid": "ppd42ns", 41 | "name": "PPD42NS", 42 | "manufacturer": "SHINYEI", 43 | "description": "" 44 | } 45 | }, { 46 | "model": "sensors.sensortype", 47 | "pk": 5, 48 | "fields": { 49 | "created": "2017-08-01T08:09:17.323Z", 50 | "modified": "2017-08-01T08:09:17.323Z", 51 | "uid": "dht11", 52 | "name": "DHT11", 53 | "manufacturer": "Adafruit", 54 | "description": "Temperature and Humidity" 55 | } 56 | }, { 57 | "model": "sensors.sensortype", 58 | "pk": 6, 59 | "fields": { 60 | "created": "2017-08-01T08:10:52.305Z", 61 | "modified": "2017-08-01T08:10:52.305Z", 62 | "uid": "GPS", 63 | "name": "UltimateGPS", 64 | "manufacturer": "Adafruit", 65 | "description": "Longitude & Latitude" 66 | } 67 | }, { 68 | "model": "sensors.sensortype", 69 | "pk": 7, 70 | "fields": { 71 | "created": "2017-08-18T07:10:14.008Z", 72 | "modified": "2017-08-18T07:10:14.008Z", 73 | "uid": "gp2y1010au0f", 74 | "name": "GP2Y1010AU0F", 75 | "manufacturer": "WaveShare", 76 | "description": "Optical data sensor" 77 | } 78 | }, { 79 | "model": "sensors.sensortype", 80 | "pk": 8, 81 | "fields": { 82 | "created": "2018-10-22T13:59:33.260Z", 83 | "modified": "2018-10-22T13:59:33.260Z", 84 | "uid": "mq135", 85 | "name": "MQ135", 86 | "manufacturer": "Olimex", 87 | "description": "Ammonia, Nitrogen Oxides, Carbon Dioxide, benzene and alcohol" 88 | } 89 | }] -------------------------------------------------------------------------------- /sensorsafrica/management/commands/add_city_names.py: -------------------------------------------------------------------------------- 1 | from django.core.management import BaseCommand 2 | from django.db.models import Q 3 | 4 | from geopy.geocoders import Nominatim 5 | 6 | from feinstaub.sensors.models import Node 7 | 8 | 9 | class Command(BaseCommand): 10 | help = "Adds city names to SensorLocation by geo reversing the SensorLocation longitude and latitude." 11 | 12 | def handle(self, *args, **options): 13 | geolocator = Nominatim(user_agent="sensors-api") 14 | all_nodes = Node.objects.filter(Q(location__city=None) | Q(location__city='')) 15 | 16 | for node in all_nodes: 17 | try: 18 | location = geolocator.reverse(f"{node.location.latitude}, {node.location.longitude}") 19 | except Exception: 20 | # Nodes with location like Soul Buoy raises exceptions 21 | continue 22 | city = location.raw['address'].get('city') 23 | node.location.city = city 24 | node.location.save() 25 | -------------------------------------------------------------------------------- /sensorsafrica/management/commands/cache_lastactive_nodes.py: -------------------------------------------------------------------------------- 1 | from django.core.management import BaseCommand 2 | 3 | from django.db import connection 4 | from sensorsafrica.api.models import Node, SensorLocation, LastActiveNodes 5 | 6 | 7 | class Command(BaseCommand): 8 | help = "" 9 | 10 | def handle(self, *args, **options): 11 | with connection.cursor() as cursor: 12 | cursor.execute( 13 | """ 14 | SELECT 15 | sn.id, 16 | sn.location_id, 17 | MAX("timestamp") AS last_active_date 18 | FROM 19 | sensors_sensordata sd 20 | INNER JOIN sensors_sensor s ON s.id = sd.sensor_id 21 | INNER JOIN sensors_node sn ON sn.id = s.node_id 22 | INNER JOIN sensors_sensordatavalue sv ON sv.sensordata_id = sd.id 23 | AND sv.value_type in ('P1', 'P2') 24 | WHERE 25 | "timestamp" >= now() - INTERVAL '5 min' 26 | GROUP BY 27 | sn.id 28 | """) 29 | latest = cursor.fetchall() 30 | for data in latest: 31 | LastActiveNodes.objects.update_or_create( 32 | node=Node(pk=data[0]), 33 | location=SensorLocation(pk=data[1]), 34 | defaults={"last_data_received_at": data[2]}, 35 | ) 36 | -------------------------------------------------------------------------------- /sensorsafrica/management/commands/cache_static_json_data.py: -------------------------------------------------------------------------------- 1 | from django.core.management import BaseCommand 2 | from django.core.cache import cache 3 | 4 | from django.conf import settings 5 | 6 | from django.forms.models import model_to_dict 7 | 8 | from feinstaub.sensors.models import SensorLocation, Sensor, SensorType 9 | 10 | import os 11 | import json 12 | import datetime 13 | from django.utils import timezone 14 | 15 | from django.db import connection 16 | 17 | from rest_framework import serializers 18 | 19 | 20 | class SensorTypeSerializer(serializers.ModelSerializer): 21 | class Meta: 22 | model = SensorType 23 | fields = "__all__" 24 | 25 | 26 | class SensorSerializer(serializers.ModelSerializer): 27 | sensor_type = SensorTypeSerializer() 28 | 29 | class Meta: 30 | model = Sensor 31 | fields = "__all__" 32 | 33 | 34 | class SensorLocationSerializer(serializers.ModelSerializer): 35 | class Meta: 36 | model = SensorLocation 37 | fields = "__all__" 38 | 39 | 40 | class Command(BaseCommand): 41 | help = "" 42 | 43 | def add_arguments(self, parser): 44 | parser.add_argument('--interval', type=str) 45 | 46 | def handle(self, *args, **options): 47 | intervals = {'5m': '5 minutes', '1h': '1 hour', '24h': '24 hours'} 48 | paths = { 49 | '5m': [ 50 | '../../../static/v2/data.json', 51 | '../../../static/v2/data.dust.min.json', 52 | '../../../static/v2/data.temp.min.json' 53 | ], 54 | '1h': ['../../../static/v2/data.1h.json'], 55 | '24h': ['../../../static/v2/data.24h.json'] 56 | } 57 | cursor = connection.cursor() 58 | cursor.execute(''' 59 | SELECT sd.sensor_id, sdv.value_type, AVG(CAST(sdv."value" AS FLOAT)) as "value", COUNT("value"), sd.location_id 60 | FROM sensors_sensordata sd 61 | INNER JOIN sensors_sensordatavalue sdv 62 | ON sdv.sensordata_id = sd.id 63 | AND sdv.value_type <> 'timestamp' 64 | AND sdv.value ~ '^\\-?\\d+(\\.?\\d+)?$' 65 | WHERE "timestamp" >= (NOW() - interval %s) 66 | GROUP BY sd.sensor_id, sdv.value_type, sd.location_id 67 | ''', [intervals[options['interval']]]) 68 | 69 | data = {} 70 | while True: 71 | row = cursor.fetchone() 72 | if row is None: 73 | break 74 | 75 | if row[0] in data: 76 | data[row[0]]['sensordatavalues'].append(dict({ 77 | 'samples': row[3], 78 | 'value': row[2], 79 | 'value_type': row[1] 80 | })) 81 | else: 82 | data[row[0]] = dict({ 83 | 'location': SensorLocationSerializer(SensorLocation.objects.get(pk=row[4])).data, 84 | 'sensor': SensorSerializer(Sensor.objects.get(pk=row[0])).data, 85 | 'sensordatavalues': [{ 86 | 'samples': row[3], 87 | 'value': row[2], 88 | 'value_type': row[1] 89 | }] 90 | }) 91 | 92 | for path in paths[options['interval']]: 93 | with open( 94 | os.path.join(os.path.dirname( 95 | os.path.abspath(__file__)), path), 'w' 96 | ) as f: 97 | if 'dust' in path: 98 | json.dump(list(filter( 99 | lambda d: d['sensor']['sensor_type']['uid'] == 'sds011', data.values())), f) 100 | elif 'temp' in path: 101 | json.dump(list(filter( 102 | lambda d: d['sensor']['sensor_type']['uid'] == 'dht22', data.values())), f) 103 | else: 104 | json.dump(list(data.values()), f) 105 | -------------------------------------------------------------------------------- /sensorsafrica/management/commands/calculate_data_statistics.py: -------------------------------------------------------------------------------- 1 | from django.core.management import BaseCommand 2 | from django.core.paginator import Paginator 3 | from django.db.models import Avg, Count, FloatField, Max, Min, Q 4 | from django.db.models.functions import Cast, TruncHour 5 | from django.utils.text import slugify 6 | 7 | from feinstaub.sensors.models import Node, Sensor, SensorDataValue, SensorLocation 8 | 9 | from sensorsafrica.api.models import SensorDataStat 10 | 11 | 12 | def map_stat(stat, city): 13 | return SensorDataStat( 14 | city_slug=slugify(city), 15 | timestamp=stat["timestamp"], 16 | value_type=stat["value_type"], 17 | location=SensorLocation(pk=stat["sensordata__location"]), 18 | sensor=Sensor(pk=stat["sensordata__sensor"]), 19 | node=Node(pk=stat["sensordata__sensor__node"]), 20 | average=stat["average"], 21 | minimum=stat["minimum"], 22 | maximum=stat["maximum"], 23 | sample_size=stat["sample_size"], 24 | last_datetime=stat["last_datetime"], 25 | ) 26 | 27 | 28 | def chunked_iterator(queryset, chunk_size=100): 29 | paginator = Paginator(queryset, chunk_size) 30 | for page in range(1, paginator.num_pages + 1): 31 | yield paginator.page(page).object_list 32 | 33 | 34 | class Command(BaseCommand): 35 | help = "Calculate and store data statistics" 36 | 37 | def handle(self, *args, **options): 38 | 39 | cities = list( 40 | set( 41 | SensorLocation.objects.all() 42 | .values_list("city", flat=True) 43 | .order_by("city") 44 | ) 45 | ) 46 | 47 | for city in cities: 48 | if not city: 49 | continue 50 | 51 | last_date_time = ( 52 | SensorDataStat.objects.filter(city_slug=slugify(city)) 53 | .values_list("last_datetime", flat=True) 54 | .order_by("-last_datetime")[:1] 55 | ) 56 | 57 | if last_date_time: 58 | queryset = SensorDataValue.objects.filter( 59 | # Pick data from public sensors only 60 | Q(sensordata__sensor__public=True), 61 | Q(sensordata__location__city__iexact=city), 62 | # Get dates greater than last stat calculation 63 | Q(created__gt=last_date_time), 64 | # Ignore timestamp values 65 | ~Q(value_type="timestamp"), 66 | # Match only valid float text 67 | Q(value__regex=r"^\-?\d+(\.?\d+)?$"), 68 | ) 69 | else: 70 | queryset = SensorDataValue.objects.filter( 71 | # Pick data from public sensors only 72 | Q(sensordata__sensor__public=True), 73 | Q(sensordata__location__city__iexact=city), 74 | # Ignore timestamp values 75 | ~Q(value_type="timestamp"), 76 | # Match only valid float text 77 | Q(value__regex=r"^\-?\d+(\.?\d+)?$"), 78 | ) 79 | 80 | for stats in chunked_iterator( 81 | queryset.annotate(timestamp=TruncHour("created")) 82 | .values( 83 | "timestamp", 84 | "value_type", 85 | "sensordata__sensor", 86 | "sensordata__location", 87 | "sensordata__sensor__node", 88 | ) 89 | .order_by() 90 | .annotate( 91 | last_datetime=Max("created"), 92 | average=Avg(Cast("value", FloatField())), 93 | minimum=Min(Cast("value", FloatField())), 94 | maximum=Max(Cast("value", FloatField())), 95 | sample_size=Count("created", FloatField()), 96 | ) 97 | .filter( 98 | ~Q(average=float("NaN")), 99 | ~Q(minimum=float("NaN")), 100 | ~Q(maximum=float("NaN")), 101 | ) 102 | .order_by("timestamp") 103 | ): 104 | SensorDataStat.objects.bulk_create( 105 | list(map(lambda stat: map_stat(stat, city), stats)) 106 | ) 107 | -------------------------------------------------------------------------------- /sensorsafrica/management/commands/upload_to_ckan.py: -------------------------------------------------------------------------------- 1 | import calendar 2 | import datetime 3 | import os 4 | import time 5 | import datetime 6 | import tempfile 7 | 8 | import ckanapi 9 | import requests 10 | import pytz 11 | from django.core.management import BaseCommand 12 | from django.db.models import Max, Min 13 | from django.utils.text import slugify 14 | from feinstaub.sensors.models import SensorData, SensorLocation 15 | 16 | 17 | class Command(BaseCommand): 18 | help = "" 19 | 20 | def handle(self, *args, **options): 21 | CKAN_ARCHIVE_API_KEY = os.environ.get("CKAN_ARCHIVE_API_KEY") 22 | CKAN_ARCHIVE_OWNER_ID = os.environ.get("CKAN_ARCHIVE_OWNER_ID") 23 | CKAN_ARCHIVE_URL = os.environ.get("CKAN_ARCHIVE_URL") 24 | 25 | session = requests.Session() 26 | session.verify = False 27 | 28 | ckan = ckanapi.RemoteCKAN(CKAN_ARCHIVE_URL, apikey=CKAN_ARCHIVE_API_KEY, session=session) 29 | 30 | city_queryset = ( 31 | SensorLocation.objects.all() 32 | .values_list("city", flat=True) 33 | .order_by("city") 34 | .distinct("city") 35 | ) 36 | for city in city_queryset.iterator(): 37 | # Ensure we have a city 38 | if not city or city.isspace(): 39 | continue 40 | 41 | # Ensure city has actual data we can upload 42 | timestamp = SensorData.objects.filter(location__city=city).aggregate( 43 | Max("timestamp"), Min("timestamp") 44 | ) 45 | if not timestamp or not timestamp['timestamp__min'] or not timestamp['timestamp__max']: 46 | continue 47 | 48 | package_name = f"sensorsafrica-airquality-archive-{slugify(city)}" 49 | package_title = f"sensors.AFRICA Air Quality Archive {city}" 50 | 51 | try: 52 | package = ckan.action.package_show(id=package_name) 53 | #To Do:xavier Implement Logging 54 | self.stdout.write(f"Package '{package_name}' already exists. Skipping creation.") 55 | except ckanapi.NotFound: 56 | try: 57 | package = ckan.action.package_create( 58 | owner_org=CKAN_ARCHIVE_OWNER_ID, 59 | name=package_name, 60 | title=package_title, 61 | groups=[{"name": "sensorsafrica-airquality-archive"}] 62 | ) 63 | self.stdout.write("Created new package '%s' for city." % city) 64 | except ckanapi.ValidationError as e: 65 | self.stdout.write(f"Validation error creating package for city %s: %s" %city %e) 66 | continue 67 | except Exception as e: 68 | self.stdout.write(f"Unexpected error fetching package for city '{city}': {e}") 69 | continue 70 | 71 | resources = package["resources"] 72 | 73 | start_date = None 74 | for resource in resources: 75 | date = resource["name"].replace("Sensor Data Archive", "") 76 | if date: 77 | date = datetime.datetime.strptime(date, "%B %Y ") 78 | if not start_date or date > start_date: 79 | start_date = date 80 | 81 | if not start_date: 82 | start_date = timestamp["timestamp__min"].replace( 83 | day=1, hour=0, minute=0, second=0, microsecond=0 84 | ) 85 | 86 | end_date = timestamp["timestamp__max"].replace( 87 | day=1, hour=0, minute=0, second=0, microsecond=0 88 | ) 89 | 90 | start_date = start_date.replace(tzinfo=pytz.UTC) 91 | end_date = end_date.replace(tzinfo=pytz.UTC) 92 | 93 | date = start_date 94 | while date <= end_date: 95 | qs = ( 96 | SensorData.objects.filter( 97 | sensor__public=True, 98 | location__city=city, 99 | timestamp__month=date.month, 100 | timestamp__year=date.year, 101 | sensordatavalues__value__isnull=False, 102 | ) 103 | .select_related("sensor","location") 104 | .values( 105 | "sensor__id", 106 | "sensor__sensor_type__name", 107 | "location__id", 108 | "location__latitude", 109 | "location__longitude", 110 | "timestamp", 111 | "sensordatavalues__value_type", 112 | "sensordatavalues__value", 113 | ) 114 | .order_by("timestamp") 115 | ) 116 | 117 | if qs.exists(): 118 | resource_name = "{month} {year} Sensor Data Archive".format( 119 | month=calendar.month_name[date.month], year=date.year 120 | ) 121 | fp = tempfile.NamedTemporaryFile(mode="w+b", suffix=".csv") 122 | try: 123 | self._write_file(fp, qs) 124 | filepath = fp.name 125 | self._create_or_update_resource( 126 | resource_name, filepath, resources, ckan, package 127 | ) 128 | finally: 129 | # Cleanup temp file 130 | fp.close() 131 | 132 | # Don't DDOS openAFRICA 133 | time.sleep(5) 134 | 135 | # Incriment month 136 | date = datetime.datetime( 137 | day=1, 138 | month=date.month % 12 + 1, 139 | year=date.year + date.month // 12, 140 | tzinfo=pytz.UTC, 141 | ) 142 | 143 | self.stdout.write("Data upload completed successfully.") 144 | 145 | @staticmethod 146 | def _write_file(fp, qs): 147 | fp.write( 148 | b"sensor_id;sensor_type;location;lat;lon;timestamp;value_type;value\n" 149 | ) 150 | for sd in qs.iterator(): 151 | s = ";".join( 152 | [ 153 | str(sd["sensor__id"]), 154 | sd["sensor__sensor_type__name"], 155 | str(sd["location__id"]), 156 | "{:.3f}".format(sd["location__latitude"]), 157 | "{:.3f}".format(sd["location__longitude"]), 158 | sd["timestamp"].isoformat(), 159 | sd["sensordatavalues__value_type"], 160 | sd["sensordatavalues__value"], 161 | ] 162 | ) 163 | fp.write(bytes(s + "\n","utf-8")) 164 | 165 | @staticmethod 166 | def _create_or_update_resource(resource_name, filepath, resources, ckan, package): 167 | extension = "CSV" 168 | 169 | resource = list( 170 | filter(lambda resource: resource["name"] == resource_name, resources) 171 | ) 172 | if resource: 173 | resource = ckan.action.resource_update( 174 | id=resource[0]["id"], url="upload", upload=open(filepath) 175 | ) 176 | else: 177 | resource = ckan.action.resource_create( 178 | package_id=package["id"], 179 | name=resource_name, 180 | format=extension, 181 | url="upload", 182 | upload=open(filepath), 183 | ) 184 | -------------------------------------------------------------------------------- /sensorsafrica/migrations/0001_sensordatastat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.18 on 2019-02-08 04:18 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | import django_extensions.db.fields 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | initial = True 13 | 14 | dependencies = [ 15 | ('sensors', '0019_auto_20190125_0521'), 16 | ] 17 | 18 | operations = [ 19 | migrations.CreateModel( 20 | name='SensorDataStat', 21 | fields=[ 22 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 23 | ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), 24 | ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), 25 | ('city_slug', models.CharField(db_index=True, max_length=255)), 26 | ('value_type', models.CharField(db_index=True, max_length=255)), 27 | ('average', models.FloatField()), 28 | ('maximum', models.FloatField()), 29 | ('minimum', models.FloatField()), 30 | ('sample_size', models.IntegerField()), 31 | ('last_datetime', models.DateTimeField()), 32 | ('timestamp', models.DateTimeField()), 33 | ('location', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sensors.SensorLocation')), 34 | ('node', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sensors.Node')), 35 | ('sensor', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sensors.Sensor')), 36 | ], 37 | options={ 38 | 'ordering': ('-modified', '-created'), 39 | 'get_latest_by': 'modified', 40 | 'abstract': False, 41 | }, 42 | ), 43 | ] 44 | -------------------------------------------------------------------------------- /sensorsafrica/migrations/0002_city.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.18 on 2019-02-18 19:07 3 | from __future__ import unicode_literals 4 | 5 | import django_extensions.db.fields 6 | from django.db import migrations, models 7 | 8 | 9 | def forwards_func(apps, schema_editor): 10 | City = apps.get_model("sensorsafrica", "City") 11 | City.objects.bulk_create( 12 | [ 13 | City( 14 | latitude=6.5244, 15 | longitude=3.3792, 16 | slug="lagos", 17 | name="Lagos", 18 | country="Nigeria", 19 | ), 20 | City( 21 | latitude=-1.2921, 22 | longitude=36.8219, 23 | slug="nairobi", 24 | name="Nairobi", 25 | country="Kenya", 26 | ), 27 | City( 28 | latitude=-6.7924, 29 | longitude=39.2083, 30 | slug="dar-es-salaam", 31 | name="Dar es Salaam", 32 | country="Tanzania", 33 | ), 34 | ] 35 | ) 36 | 37 | 38 | def reverse_func(apps, schema_editor): 39 | pass 40 | 41 | 42 | class Migration(migrations.Migration): 43 | 44 | dependencies = [("sensorsafrica", "0001_sensordatastat")] 45 | 46 | operations = [ 47 | migrations.CreateModel( 48 | name="City", 49 | fields=[ 50 | ( 51 | "id", 52 | models.AutoField( 53 | auto_created=True, 54 | primary_key=True, 55 | serialize=False, 56 | verbose_name="ID", 57 | ), 58 | ), 59 | ( 60 | "created", 61 | django_extensions.db.fields.CreationDateTimeField( 62 | auto_now_add=True, verbose_name="created" 63 | ), 64 | ), 65 | ( 66 | "modified", 67 | django_extensions.db.fields.ModificationDateTimeField( 68 | auto_now=True, verbose_name="modified" 69 | ), 70 | ), 71 | ("slug", models.CharField(db_index=True, max_length=255)), 72 | ("name", models.CharField(db_index=True, max_length=255)), 73 | ("country", models.CharField(db_index=True, max_length=255)), 74 | ("location", models.CharField(db_index=True, max_length=255)), 75 | ( 76 | "latitude", 77 | models.DecimalField( 78 | blank=True, decimal_places=11, max_digits=14, null=True 79 | ), 80 | ), 81 | ( 82 | "longitude", 83 | models.DecimalField( 84 | blank=True, decimal_places=11, max_digits=14, null=True 85 | ), 86 | ) 87 | ], 88 | options={ 89 | "ordering": ("-modified", "-created"), 90 | "get_latest_by": "modified", 91 | "abstract": False, 92 | }, 93 | ), 94 | migrations.RunPython(forwards_func, reverse_func), 95 | ] 96 | -------------------------------------------------------------------------------- /sensorsafrica/migrations/0003_auto_20190222_1137.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.18 on 2019-02-22 11:37 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('sensorsafrica', '0002_city'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterModelOptions( 16 | name='city', 17 | options={'verbose_name_plural': 'Cities'}, 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /sensorsafrica/migrations/0004_auto_20190509_1145.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.18 on 2019-05-09 11:45 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.db.models.deletion 7 | import django_extensions.db.fields 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | dependencies = [ 13 | ('sensors', '0020_auto_20190314_1232'), 14 | ('sensorsafrica', '0003_auto_20190222_1137'), 15 | ] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name='LastActiveNodes', 20 | fields=[ 21 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 22 | ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')), 23 | ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')), 24 | ('last_data_received_at', models.DateTimeField()), 25 | ('location', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sensors.SensorLocation')), 26 | ('node', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sensors.Node')), 27 | ], 28 | ), 29 | migrations.AlterUniqueTogether( 30 | name='lastactivenodes', 31 | unique_together=set([('node', 'location')]), 32 | ), 33 | ] 34 | -------------------------------------------------------------------------------- /sensorsafrica/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/migrations/__init__.py -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/openstuttgart/__init__.py -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/openstuttgart/feinstaub/__init__.py -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/openstuttgart/feinstaub/sensors/__init__.py -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import django.utils.timezone 6 | import django_extensions.db.fields 7 | from django.conf import settings 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | dependencies = [ 13 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='Sensor', 19 | fields=[ 20 | ('id', models.AutoField(auto_created=True, serialize=False, verbose_name='ID', primary_key=True)), 21 | ('created', django_extensions.db.fields.CreationDateTimeField(blank=True, default=django.utils.timezone.now, editable=False, verbose_name='created')), 22 | ('modified', django_extensions.db.fields.ModificationDateTimeField(blank=True, default=django.utils.timezone.now, editable=False, verbose_name='modified')), 23 | ('uid', models.SlugField(unique=True)), 24 | ('description', models.CharField(max_length=10000)), 25 | ('owner', models.ForeignKey(to=settings.AUTH_USER_MODEL)), 26 | ], 27 | options={ 28 | 'get_latest_by': 'modified', 29 | 'ordering': ('-modified', '-created'), 30 | 'abstract': False, 31 | }, 32 | bases=(models.Model,), 33 | ), 34 | migrations.CreateModel( 35 | name='SensorData', 36 | fields=[ 37 | ('id', models.AutoField(auto_created=True, serialize=False, verbose_name='ID', primary_key=True)), 38 | ('created', django_extensions.db.fields.CreationDateTimeField(blank=True, default=django.utils.timezone.now, editable=False, verbose_name='created')), 39 | ('modified', django_extensions.db.fields.ModificationDateTimeField(blank=True, default=django.utils.timezone.now, editable=False, verbose_name='modified')), 40 | ('value1', models.IntegerField()), 41 | ('value2', models.IntegerField(blank=True, null=True)), 42 | ('sensor', models.ForeignKey(to='sensors.Sensor')), 43 | ], 44 | options={ 45 | 'get_latest_by': 'modified', 46 | 'ordering': ('-modified', '-created'), 47 | 'abstract': False, 48 | }, 49 | bases=(models.Model,), 50 | ), 51 | migrations.CreateModel( 52 | name='SensorLocation', 53 | fields=[ 54 | ('id', models.AutoField(auto_created=True, serialize=False, verbose_name='ID', primary_key=True)), 55 | ('created', django_extensions.db.fields.CreationDateTimeField(blank=True, default=django.utils.timezone.now, editable=False, verbose_name='created')), 56 | ('modified', django_extensions.db.fields.ModificationDateTimeField(blank=True, default=django.utils.timezone.now, editable=False, verbose_name='modified')), 57 | ('location', models.TextField(blank=True, null=True)), 58 | ('sensor', models.ForeignKey(to='sensors.Sensor')), 59 | ], 60 | options={ 61 | 'get_latest_by': 'modified', 62 | 'ordering': ('-modified', '-created'), 63 | 'abstract': False, 64 | }, 65 | bases=(models.Model,), 66 | ), 67 | migrations.CreateModel( 68 | name='SensorType', 69 | fields=[ 70 | ('id', models.AutoField(auto_created=True, serialize=False, verbose_name='ID', primary_key=True)), 71 | ('created', django_extensions.db.fields.CreationDateTimeField(blank=True, default=django.utils.timezone.now, editable=False, verbose_name='created')), 72 | ('modified', django_extensions.db.fields.ModificationDateTimeField(blank=True, default=django.utils.timezone.now, editable=False, verbose_name='modified')), 73 | ('uid', models.SlugField(unique=True)), 74 | ('name', models.CharField(max_length=1000)), 75 | ('manufacturer', models.CharField(max_length=1000)), 76 | ('description', models.CharField(max_length=10000)), 77 | ], 78 | options={ 79 | 'get_latest_by': 'modified', 80 | 'ordering': ('-modified', '-created'), 81 | 'abstract': False, 82 | }, 83 | bases=(models.Model,), 84 | ), 85 | migrations.AddField( 86 | model_name='sensor', 87 | name='sensor_type', 88 | field=models.ForeignKey(to='sensors.SensorType'), 89 | preserve_default=True, 90 | ), 91 | ] 92 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0002_auto_20150330_1800.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import datetime 6 | from django.utils.timezone import utc 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('sensors', '0001_initial'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddField( 17 | model_name='sensordata', 18 | name='timestamp', 19 | field=models.DateTimeField(default=datetime.datetime(2015, 3, 30, 18, 0, 43, 397896, tzinfo=utc)), 20 | preserve_default=True, 21 | ), 22 | migrations.AddField( 23 | model_name='sensorlocation', 24 | name='timestamp', 25 | field=models.DateTimeField(default=datetime.datetime(2015, 3, 30, 18, 0, 43, 398539, tzinfo=utc)), 26 | preserve_default=True, 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0003_auto_20150330_1805.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import django.utils.timezone 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('sensors', '0002_auto_20150330_1800'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='sensordata', 17 | name='sampling_rate', 18 | field=models.IntegerField(null=True, blank=True), 19 | preserve_default=True, 20 | ), 21 | migrations.AlterField( 22 | model_name='sensordata', 23 | name='timestamp', 24 | field=models.DateTimeField(default=django.utils.timezone.now), 25 | preserve_default=True, 26 | ), 27 | migrations.AlterField( 28 | model_name='sensorlocation', 29 | name='timestamp', 30 | field=models.DateTimeField(default=django.utils.timezone.now), 31 | preserve_default=True, 32 | ), 33 | ] 34 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0004_auto_20150331_1907.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | from django.conf import settings 6 | import django_extensions.db.fields 7 | import django.utils.timezone 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | dependencies = [ 13 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 14 | ('sensors', '0003_auto_20150330_1805'), 15 | ] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name='SensorDataValue', 20 | fields=[ 21 | ('id', models.AutoField(serialize=False, primary_key=True, auto_created=True, verbose_name='ID')), 22 | ('created', django_extensions.db.fields.CreationDateTimeField(editable=False, default=django.utils.timezone.now, blank=True, verbose_name='created')), 23 | ('modified', django_extensions.db.fields.ModificationDateTimeField(editable=False, default=django.utils.timezone.now, blank=True, verbose_name='modified')), 24 | ('value', models.TextField()), 25 | ('value_type', models.CharField(choices=[('P1', '1µm particles'), ('P2', '2.5µm particles'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('brightness', 'Brightness')], max_length=100)), 26 | ('sensordata', models.ForeignKey(to='sensors.SensorData')), 27 | ], 28 | options={ 29 | 'abstract': False, 30 | 'ordering': ('-modified', '-created'), 31 | 'get_latest_by': 'modified', 32 | }, 33 | bases=(models.Model,), 34 | ), 35 | migrations.RemoveField( 36 | model_name='sensordata', 37 | name='value1', 38 | ), 39 | migrations.RemoveField( 40 | model_name='sensordata', 41 | name='value2', 42 | ), 43 | migrations.RemoveField( 44 | model_name='sensorlocation', 45 | name='sensor', 46 | ), 47 | migrations.AddField( 48 | model_name='sensor', 49 | name='location', 50 | field=models.ForeignKey(default=False, to='sensors.SensorLocation'), 51 | preserve_default=False, 52 | ), 53 | migrations.AddField( 54 | model_name='sensordata', 55 | name='location', 56 | field=models.ForeignKey(default=1, blank=True, to='sensors.SensorLocation'), 57 | preserve_default=False, 58 | ), 59 | migrations.AddField( 60 | model_name='sensorlocation', 61 | name='description', 62 | field=models.TextField(null=True, blank=True), 63 | preserve_default=True, 64 | ), 65 | migrations.AddField( 66 | model_name='sensorlocation', 67 | name='indoor', 68 | field=models.BooleanField(default=False), 69 | preserve_default=False, 70 | ), 71 | migrations.AddField( 72 | model_name='sensorlocation', 73 | name='owner', 74 | field=models.ForeignKey(blank=True, help_text='If not set, location is public.', null=True, to=settings.AUTH_USER_MODEL), 75 | preserve_default=True, 76 | ), 77 | migrations.AlterField( 78 | model_name='sensor', 79 | name='description', 80 | field=models.TextField(null=True, blank=True), 81 | preserve_default=True, 82 | ), 83 | ] 84 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0005_auto_20150403_2041.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('sensors', '0004_auto_20150331_1907'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='sensorlocation', 16 | name='indoor', 17 | field=models.BooleanField(default=False), 18 | preserve_default=True, 19 | ), 20 | migrations.AlterField( 21 | model_name='sensortype', 22 | name='description', 23 | field=models.CharField(max_length=10000, blank=True, null=True), 24 | preserve_default=True, 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0006_auto_20150404_2050.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('sensors', '0005_auto_20150403_2041'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='sensordatavalue', 16 | name='value_type', 17 | field=models.CharField(max_length=100, choices=[('P1', '1µm particles'), ('P2', '2.5µm particles'), ('durP1', 'duration 1µm'), ('durP2', 'duration 2.5µm'), ('ratioP1', 'ratio 1µm in percent'), ('ratioP2', 'ratio 2.5µm in percent'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('brightness', 'Brightness')]), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0007_auto_20150405_2151.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('sensors', '0006_auto_20150404_2050'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='sensordatavalue', 16 | name='sensordata', 17 | field=models.ForeignKey(to='sensors.SensorData', related_name='sensordatavalues'), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0008_auto_20150503_1554.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('sensors', '0007_auto_20150405_2151'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='sensordata', 16 | name='sampling_rate', 17 | field=models.IntegerField(blank=True, null=True, help_text='in milliseconds'), 18 | ), 19 | migrations.AlterField( 20 | model_name='sensordatavalue', 21 | name='value_type', 22 | field=models.CharField(max_length=100, choices=[('P1', '1µm particles'), ('P2', '2.5µm particles'), ('durP1', 'duration 1µm'), ('durP2', 'duration 2.5µm'), ('ratioP1', 'ratio 1µm in percent'), ('ratioP2', 'ratio 2.5µm in percent'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('pressure', 'Pa'), ('altitude', 'meter'), ('pressure_sealevel', 'Pa'), ('brightness', 'Brightness'), ('dust_density', 'Dust density in mg/m3'), ('vo_raw', 'Dust voltage raw'), ('voltage', 'Dust voltage calculated'), ('P10', '1µm particles'), ('P25', '2.5µm particles'), ('durP10', 'duration 1µm'), ('durP25', 'duration 2.5µm'), ('ratioP10', 'ratio 1µm in percent'), ('ratioP20', 'ratio 2.5µm in percent')]), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0009_auto_20150503_1556.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('sensors', '0008_auto_20150503_1554'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='sensordatavalue', 16 | name='value_type', 17 | field=models.CharField(max_length=100, choices=[('P1', '1µm particles'), ('P2', '2.5µm particles'), ('durP1', 'duration 1µm'), ('durP2', 'duration 2.5µm'), ('ratioP1', 'ratio 1µm in percent'), ('ratioP2', 'ratio 2.5µm in percent'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('pressure', 'Pa'), ('altitude', 'meter'), ('pressure_sealevel', 'Pa'), ('brightness', 'Brightness'), ('dust_density', 'Dust density in mg/m3'), ('vo_raw', 'Dust voltage raw'), ('voltage', 'Dust voltage calculated'), ('P10', '1µm particles'), ('P25', '2.5µm particles'), ('durP10', 'duration 1µm'), ('durP25', 'duration 2.5µm'), ('ratioP10', 'ratio 1µm in percent'), ('ratioP25', 'ratio 2.5µm in percent')]), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0010_auto_20150620_1708.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('sensors', '0009_auto_20150503_1556'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='sensordatavalue', 16 | name='value', 17 | field=models.TextField(db_index=True), 18 | ), 19 | migrations.AlterField( 20 | model_name='sensordatavalue', 21 | name='value_type', 22 | field=models.CharField(choices=[('P1', '1µm particles'), ('P2', '2.5µm particles'), ('durP1', 'duration 1µm'), ('durP2', 'duration 2.5µm'), ('ratioP1', 'ratio 1µm in percent'), ('ratioP2', 'ratio 2.5µm in percent'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('pressure', 'Pa'), ('altitude', 'meter'), ('pressure_sealevel', 'Pa (sealevel)'), ('brightness', 'Brightness'), ('dust_density', 'Dust density in mg/m3'), ('vo_raw', 'Dust voltage raw'), ('voltage', 'Dust voltage calculated'), ('P10', '1µm particles'), ('P25', '2.5µm particles'), ('durP10', 'duration 1µm'), ('durP25', 'duration 2.5µm'), ('ratioP10', 'ratio 1µm in percent'), ('ratioP25', 'ratio 2.5µm in percent')], max_length=100, db_index=True), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0011_auto_20150807_1927.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | from django.conf import settings 6 | import django_extensions.db.fields 7 | import django.utils.timezone 8 | 9 | 10 | def migrate_sensor(apps, schema_editor): 11 | # We get the model from the versioned app registry; 12 | # if we directly import it, it'll be the wrong version 13 | Sensor = apps.get_model("sensors", "Sensor") 14 | Node = apps.get_model("sensors", "Node") 15 | db_alias = schema_editor.connection.alias 16 | if db_alias != "default": 17 | return 18 | for sensor in Sensor.objects.using(db_alias).all(): 19 | node = Node.objects.using(db_alias).create(uid=sensor.uid, 20 | description=sensor.description, 21 | owner=sensor.owner, 22 | location=sensor.location) 23 | sensor.node = node 24 | sensor.save() 25 | 26 | 27 | class Migration(migrations.Migration): 28 | 29 | dependencies = [ 30 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 31 | ('sensors', '0010_auto_20150620_1708'), 32 | ] 33 | 34 | operations = [ 35 | migrations.CreateModel( 36 | name='Node', 37 | fields=[ 38 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 39 | ('created', django_extensions.db.fields.CreationDateTimeField(blank=True, verbose_name='created', editable=False, default=django.utils.timezone.now)), 40 | ('modified', django_extensions.db.fields.ModificationDateTimeField(blank=True, verbose_name='modified', editable=False, default=django.utils.timezone.now)), 41 | ('uid', models.SlugField(unique=True)), 42 | ('description', models.TextField(blank=True, null=True)), 43 | ], 44 | options={ 45 | 'ordering': ['uid'], 46 | }, 47 | ), 48 | migrations.AddField( 49 | model_name='node', 50 | name='location', 51 | field=models.ForeignKey(to='sensors.SensorLocation'), 52 | ), 53 | migrations.AddField( 54 | model_name='node', 55 | name='owner', 56 | field=models.ForeignKey(to=settings.AUTH_USER_MODEL), 57 | ), 58 | migrations.AddField( 59 | model_name='sensor', 60 | name='node', 61 | field=models.ForeignKey(to='sensors.Node', blank=True, null=True), 62 | preserve_default=False, 63 | ), 64 | 65 | migrations.RunPython( 66 | migrate_sensor, 67 | ), 68 | 69 | migrations.AlterModelOptions( 70 | name='sensor', 71 | options={}, 72 | ), 73 | migrations.AlterModelOptions( 74 | name='sensorlocation', 75 | options={'ordering': ['location']}, 76 | ), 77 | migrations.AlterModelOptions( 78 | name='sensortype', 79 | options={'ordering': ['name']}, 80 | ), 81 | migrations.AddField( 82 | model_name='sensor', 83 | name='pin', 84 | field=models.CharField(max_length=10, help_text='differentiate the sensors on one node by giving pin used', default='-'), 85 | ), 86 | migrations.AlterField( 87 | model_name='sensordatavalue', 88 | name='value_type', 89 | field=models.CharField(db_index=True, max_length=100, choices=[('P1', '1µm particles'), ('P2', '2.5µm particles'), ('durP1', 'duration 1µm'), ('durP2', 'duration 2.5µm'), ('ratioP1', 'ratio 1µm in percent'), ('ratioP2', 'ratio 2.5µm in percent'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('pressure', 'Pa'), ('altitude', 'meter'), ('pressure_sealevel', 'Pa (sealevel)'), ('brightness', 'Brightness'), ('dust_density', 'Dust density in mg/m3'), ('vo_raw', 'Dust voltage raw'), ('voltage', 'Dust voltage calculated'), ('P10', '1µm particles'), ('P25', '2.5µm particles'), ('durP10', 'duration 1µm'), ('durP25', 'duration 2.5µm'), ('ratioP10', 'ratio 1µm in percent'), ('ratioP25', 'ratio 2.5µm in percent'), ('door_state', 'door state (open/closed)')]), 90 | ), 91 | migrations.AlterUniqueTogether( 92 | name='sensor', 93 | unique_together=set([('node', 'pin')]), 94 | ), 95 | migrations.RemoveField( 96 | model_name='sensor', 97 | name='location', 98 | ), 99 | migrations.RemoveField( 100 | model_name='sensor', 101 | name='owner', 102 | ), 103 | migrations.RemoveField( 104 | model_name='sensor', 105 | name='uid', 106 | ), 107 | ] 108 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0012_auto_20150807_1943.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('sensors', '0011_auto_20150807_1927'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='sensor', 16 | name='node', 17 | field=models.ForeignKey(default=1, to='sensors.Node'), 18 | preserve_default=False, 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0013_auto_20151025_1615.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | import django_extensions.db.fields 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('sensors', '0012_auto_20150807_1943'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='sensorlocation', 17 | name='latitude', 18 | field=models.DecimalField(decimal_places=11, max_digits=14, null=True, blank=True), 19 | ), 20 | migrations.AddField( 21 | model_name='sensorlocation', 22 | name='longitude', 23 | field=models.DecimalField(decimal_places=11, max_digits=14, null=True, blank=True), 24 | ), 25 | migrations.AlterField( 26 | model_name='node', 27 | name='created', 28 | field=django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created'), 29 | ), 30 | migrations.AlterField( 31 | model_name='node', 32 | name='modified', 33 | field=django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified'), 34 | ), 35 | migrations.AlterField( 36 | model_name='sensor', 37 | name='created', 38 | field=django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created'), 39 | ), 40 | migrations.AlterField( 41 | model_name='sensor', 42 | name='modified', 43 | field=django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified'), 44 | ), 45 | migrations.AlterField( 46 | model_name='sensor', 47 | name='node', 48 | field=models.ForeignKey(related_name='sensors', to='sensors.Node'), 49 | ), 50 | migrations.AlterField( 51 | model_name='sensordata', 52 | name='created', 53 | field=django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created'), 54 | ), 55 | migrations.AlterField( 56 | model_name='sensordata', 57 | name='modified', 58 | field=django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified'), 59 | ), 60 | migrations.AlterField( 61 | model_name='sensordata', 62 | name='sensor', 63 | field=models.ForeignKey(related_name='sensordatas', to='sensors.Sensor'), 64 | ), 65 | migrations.AlterField( 66 | model_name='sensordatavalue', 67 | name='created', 68 | field=django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created'), 69 | ), 70 | migrations.AlterField( 71 | model_name='sensordatavalue', 72 | name='modified', 73 | field=django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified'), 74 | ), 75 | migrations.AlterField( 76 | model_name='sensorlocation', 77 | name='created', 78 | field=django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created'), 79 | ), 80 | migrations.AlterField( 81 | model_name='sensorlocation', 82 | name='modified', 83 | field=django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified'), 84 | ), 85 | migrations.AlterField( 86 | model_name='sensortype', 87 | name='created', 88 | field=django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created'), 89 | ), 90 | migrations.AlterField( 91 | model_name='sensortype', 92 | name='modified', 93 | field=django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified'), 94 | ), 95 | ] 96 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0014_sensor_public.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('sensors', '0013_auto_20151025_1615'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='sensor', 16 | name='public', 17 | field=models.BooleanField(default=False), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0015_sensordata_software_version.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('sensors', '0014_sensor_public'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='sensordata', 16 | name='software_version', 17 | field=models.CharField(help_text='sensor software version', default='', max_length=100), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0016_auto_20160209_2030.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9.2 on 2016-02-09 20:30 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('sensors', '0015_sensordata_software_version'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterModelOptions( 16 | name='sensordatavalue', 17 | options={}, 18 | ), 19 | migrations.AlterUniqueTogether( 20 | name='sensordatavalue', 21 | unique_together=set([('sensordata', 'value_type')]), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0017_auto_20160416_1803.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.9 on 2016-04-16 18:03 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('sensors', '0016_auto_20160209_2030'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='sensordatavalue', 17 | name='value_type', 18 | field=models.CharField(choices=[('P1', '1µm particles'), ('P2', '2.5µm particles'), ('durP1', 'duration 1µm'), ('durP2', 'duration 2.5µm'), ('ratioP1', 'ratio 1µm in percent'), ('ratioP2', 'ratio 2.5µm in percent'), ('samples', 'samples'), ('min_micro', 'min_micro'), ('max_micro', 'max_micro'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('pressure', 'Pa'), ('altitude', 'meter'), ('pressure_sealevel', 'Pa (sealevel)'), ('brightness', 'Brightness'), ('dust_density', 'Dust density in mg/m3'), ('vo_raw', 'Dust voltage raw'), ('voltage', 'Dust voltage calculated'), ('P10', '1µm particles'), ('P25', '2.5µm particles'), ('durP10', 'duration 1µm'), ('durP25', 'duration 2.5µm'), ('ratioP10', 'ratio 1µm in percent'), ('ratioP25', 'ratio 2.5µm in percent'), ('door_state', 'door state (open/closed)')], db_index=True, max_length=100), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0018_auto_20170218_2329.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.5 on 2017-02-18 23:29 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import django.utils.timezone 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('sensors', '0017_auto_20160416_1803'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AlterField( 17 | model_name='sensor', 18 | name='pin', 19 | field=models.CharField(db_index=True, default='-', help_text='differentiate the sensors on one node by giving pin used', max_length=10), 20 | ), 21 | migrations.AlterField( 22 | model_name='sensor', 23 | name='public', 24 | field=models.BooleanField(db_index=True, default=False), 25 | ), 26 | migrations.AlterField( 27 | model_name='sensordata', 28 | name='timestamp', 29 | field=models.DateTimeField(db_index=True, default=django.utils.timezone.now), 30 | ), 31 | migrations.AlterField( 32 | model_name='sensordatavalue', 33 | name='value_type', 34 | field=models.CharField(choices=[('P1', '1µm particles'), ('P2', '2.5µm particles'), ('durP1', 'duration 1µm'), ('durP2', 'duration 2.5µm'), ('ratioP1', 'ratio 1µm in percent'), ('ratioP2', 'ratio 2.5µm in percent'), ('samples', 'samples'), ('min_micro', 'min_micro'), ('max_micro', 'max_micro'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('pressure', 'Pa'), ('altitude', 'meter'), ('pressure_sealevel', 'Pa (sealevel)'), ('brightness', 'Brightness'), ('dust_density', 'Dust density in mg/m3'), ('vo_raw', 'Dust voltage raw'), ('voltage', 'Dust voltage calculated'), ('P10', '1µm particles'), ('P25', '2.5µm particles'), ('durP10', 'duration 1µm'), ('durP25', 'duration 2.5µm'), ('ratioP10', 'ratio 1µm in percent'), ('ratioP25', 'ratio 2.5µm in percent'), ('door_state', 'door state (open/closed)'), ('lat', 'latitude'), ('lon', 'longitude'), ('height', 'height'), ('hdop', 'horizontal dilusion of precision'), ('timestamp', 'measured timestamp'), ('age', 'measured age'), ('satelites', 'number of satelites'), ('speed', 'current speed over ground'), ('azimuth', 'track angle')], db_index=True, max_length=100), 35 | ), 36 | migrations.AlterIndexTogether( 37 | name='sensordata', 38 | index_together=set([('modified',)]), 39 | ), 40 | ] 41 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0019_auto_20190125_0521.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.8 on 2019-01-25 05:21 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('sensors', '0018_auto_20170218_2329'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='node', 17 | name='description_internal', 18 | field=models.TextField(blank=True, null=True), 19 | ), 20 | migrations.AddField( 21 | model_name='node', 22 | name='email', 23 | field=models.EmailField(blank=True, max_length=254, null=True), 24 | ), 25 | migrations.AddField( 26 | model_name='node', 27 | name='height', 28 | field=models.IntegerField(null=True), 29 | ), 30 | migrations.AddField( 31 | model_name='node', 32 | name='last_notify', 33 | field=models.DateTimeField(blank=True, null=True), 34 | ), 35 | migrations.AddField( 36 | model_name='node', 37 | name='name', 38 | field=models.TextField(blank=True, null=True), 39 | ), 40 | migrations.AddField( 41 | model_name='node', 42 | name='sensor_position', 43 | field=models.IntegerField(null=True), 44 | ), 45 | migrations.AddField( 46 | model_name='sensorlocation', 47 | name='city', 48 | field=models.TextField(blank=True, null=True), 49 | ), 50 | migrations.AddField( 51 | model_name='sensorlocation', 52 | name='country', 53 | field=models.TextField(blank=True, null=True), 54 | ), 55 | migrations.AddField( 56 | model_name='sensorlocation', 57 | name='industry_in_area', 58 | field=models.IntegerField(null=True), 59 | ), 60 | migrations.AddField( 61 | model_name='sensorlocation', 62 | name='oven_in_area', 63 | field=models.IntegerField(null=True), 64 | ), 65 | migrations.AddField( 66 | model_name='sensorlocation', 67 | name='postalcode', 68 | field=models.TextField(blank=True, null=True), 69 | ), 70 | migrations.AddField( 71 | model_name='sensorlocation', 72 | name='street_name', 73 | field=models.TextField(blank=True, null=True), 74 | ), 75 | migrations.AddField( 76 | model_name='sensorlocation', 77 | name='street_number', 78 | field=models.TextField(blank=True, null=True), 79 | ), 80 | migrations.AddField( 81 | model_name='sensorlocation', 82 | name='traffic_in_area', 83 | field=models.IntegerField(null=True), 84 | ), 85 | migrations.AlterField( 86 | model_name='sensordatavalue', 87 | name='value_type', 88 | field=models.CharField(choices=[('P0', '1µm particles'), ('P1', '1µm particles'), ('P2', '2.5µm particles'), ('durP1', 'duration 1µm'), ('durP2', 'duration 2.5µm'), ('ratioP1', 'ratio 1µm in percent'), ('ratioP2', 'ratio 2.5µm in percent'), ('samples', 'samples'), ('min_micro', 'min_micro'), ('max_micro', 'max_micro'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('pressure', 'Pa'), ('altitude', 'meter'), ('pressure_sealevel', 'Pa (sealevel)'), ('brightness', 'Brightness'), ('dust_density', 'Dust density in mg/m3'), ('vo_raw', 'Dust voltage raw'), ('voltage', 'Dust voltage calculated'), ('P10', '1µm particles'), ('P25', '2.5µm particles'), ('durP10', 'duration 1µm'), ('durP25', 'duration 2.5µm'), ('ratioP10', 'ratio 1µm in percent'), ('ratioP25', 'ratio 2.5µm in percent'), ('door_state', 'door state (open/closed)'), ('lat', 'latitude'), ('lon', 'longitude'), ('height', 'height'), ('hdop', 'horizontal dilusion of precision'), ('timestamp', 'measured timestamp'), ('age', 'measured age'), ('satelites', 'number of satelites'), ('speed', 'current speed over ground'), ('azimuth', 'track angle')], db_index=True, max_length=100), 89 | ), 90 | ] 91 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0020_auto_20190314_1232.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.18 on 2019-03-14 12:32 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('sensors', '0019_auto_20190125_0521'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='node', 17 | name='exact_location', 18 | field=models.BooleanField(default=False), 19 | ), 20 | migrations.AddField( 21 | model_name='node', 22 | name='inactive', 23 | field=models.BooleanField(default=False), 24 | ), 25 | migrations.AddField( 26 | model_name='node', 27 | name='indoor', 28 | field=models.BooleanField(default=False), 29 | ), 30 | migrations.AddField( 31 | model_name='sensorlocation', 32 | name='altitude', 33 | field=models.DecimalField(blank=True, decimal_places=8, max_digits=14, null=True), 34 | ), 35 | migrations.AlterField( 36 | model_name='sensordatavalue', 37 | name='value', 38 | field=models.TextField(), 39 | ), 40 | migrations.AlterField( 41 | model_name='sensordatavalue', 42 | name='value_type', 43 | field=models.CharField(choices=[('P0', '1µm particles'), ('P1', '10µm particles'), ('P2', '2.5µm particles'), ('durP1', 'duration 1µm'), ('durP2', 'duration 2.5µm'), ('ratioP1', 'ratio 1µm in percent'), ('ratioP2', 'ratio 2.5µm in percent'), ('samples', 'samples'), ('min_micro', 'min_micro'), ('max_micro', 'max_micro'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('pressure', 'Pa'), ('altitude', 'meter'), ('pressure_sealevel', 'Pa (sealevel)'), ('brightness', 'Brightness'), ('dust_density', 'Dust density in mg/m3'), ('vo_raw', 'Dust voltage raw'), ('voltage', 'Dust voltage calculated'), ('P10', '1µm particles'), ('P25', '2.5µm particles'), ('durP10', 'duration 1µm'), ('durP25', 'duration 2.5µm'), ('ratioP10', 'ratio 1µm in percent'), ('ratioP25', 'ratio 2.5µm in percent'), ('door_state', 'door state (open/closed)'), ('lat', 'latitude'), ('lon', 'longitude'), ('height', 'height'), ('hdop', 'horizontal dilusion of precision'), ('timestamp', 'measured timestamp'), ('age', 'measured age'), ('satelites', 'number of satelites'), ('speed', 'current speed over ground'), ('azimuth', 'track angle'), ('noise_L01', 'Sound level L01'), ('noise_L95', 'Sound level L95'), ('noise_Leq', 'Sound level Leq')], db_index=True, max_length=100), 44 | ), 45 | ] 46 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0021_auto_20210204_1106.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.29 on 2021-02-04 11:06 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('sensors', '0020_auto_20190314_1232'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name='sensordatavalue', 17 | name='value_type', 18 | field=models.CharField(choices=[('P0', '1µm particles'), ('P1', '10µm particles'), ('P2', '2.5µm particles'), ('durP1', 'duration 1µm'), ('durP2', 'duration 2.5µm'), ('ratioP1', 'ratio 1µm in percent'), ('ratioP2', 'ratio 2.5µm in percent'), ('samples', 'samples'), ('min_micro', 'min_micro'), ('max_micro', 'max_micro'), ('temperature', 'Temperature'), ('humidity', 'Humidity'), ('pressure', 'Pa'), ('altitude', 'meter'), ('pressure_sealevel', 'Pa (sealevel)'), ('brightness', 'Brightness'), ('dust_density', 'Dust density in mg/m3'), ('vo_raw', 'Dust voltage raw'), ('voltage', 'Dust voltage calculated'), ('P10', '1µm particles'), ('P25', '2.5µm particles'), ('durP10', 'duration 1µm'), ('durP25', 'duration 2.5µm'), ('ratioP10', 'ratio 1µm in percent'), ('ratioP25', 'ratio 2.5µm in percent'), ('door_state', 'door state (open/closed)'), ('lat', 'latitude'), ('lon', 'longitude'), ('height', 'height'), ('hdop', 'horizontal dilusion of precision'), ('timestamp', 'measured timestamp'), ('age', 'measured age'), ('satelites', 'number of satelites'), ('speed', 'current speed over ground'), ('azimuth', 'track angle'), ('noise_L01', 'Sound level L01'), ('noise_L95', 'Sound level L95'), ('noise_Leq', 'Sound level Leq'), ('co_kohm', 'CO in kOhm'), ('co_ppb', 'CO in ppb'), ('eco2', 'eCO2 in ppm'), ('no2_kohm', 'NO2 in kOhm'), ('no2_ppb', 'NO2 in ppb'), ('ozone_ppb', 'O3 in ppb'), ('so2_ppb', 'SO2 in ppb')], db_index=True, max_length=100), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/0022_auto_20210211_2023.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.29 on 2021-02-11 20:23 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('sensors', '0021_auto_20210204_1106'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddIndex( 16 | model_name='sensorlocation', 17 | index=models.Index(fields=['country'], name='country_idx'), 18 | ), 19 | migrations.AddIndex( 20 | model_name='sensorlocation', 21 | index=models.Index(fields=['city'], name='city_idx'), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /sensorsafrica/openstuttgart/feinstaub/sensors/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/openstuttgart/feinstaub/sensors/migrations/__init__.py -------------------------------------------------------------------------------- /sensorsafrica/router.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from django.conf import settings 4 | 5 | class ReplicaRouter: 6 | read_replicas = list(settings.DATABASES.keys()) 7 | 8 | def db_for_read(self, model, **hints): 9 | return random.choice(self.read_replicas) 10 | 11 | def db_for_write(self, model, **hints): 12 | return "default" 13 | 14 | def allow_relation(self, obj1, obj2, **hints): 15 | """ 16 | Relation not applicable for our use case 17 | """ 18 | return None 19 | 20 | def allow_migrate(self,db,app_label,model_name=None, **hints): 21 | """ 22 | Restrict migration operations to the master db i.e. default 23 | """ 24 | return db == "default" 25 | 26 | 27 | -------------------------------------------------------------------------------- /sensorsafrica/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for sensorsafrica project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.10.8. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.10/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | import dj_database_url 16 | import sentry_sdk 17 | from sentry_sdk.integrations.celery import CeleryIntegration 18 | from sentry_sdk.integrations.django import DjangoIntegration 19 | 20 | from celery.schedules import crontab 21 | 22 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 23 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 24 | 25 | 26 | # Quick-start development settings - unsuitable for production 27 | # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ 28 | 29 | # SECURITY WARNING: keep the secret key used in production secret! 30 | SECRET_KEY = os.getenv( 31 | "SENSORSAFRICA_SECRET_KEY", "-kc8keig#xrdhi1l$rrj&s*s@3pz*4he)8u8h^w$2-_4y6@z3g" 32 | ) 33 | 34 | # SECURITY WARNING: don't run with debug turned on in production! 35 | DEBUG = os.getenv("SENSORSAFRICA_DEBUG", "True") == "True" 36 | 37 | ALLOWED_HOSTS = os.getenv("SENSORSAFRICA_ALLOWED_HOSTS", "*").split(",") 38 | 39 | CORS_ORIGIN_ALLOW_ALL = True 40 | 41 | # Application definition 42 | 43 | INSTALLED_APPS = [ 44 | "django.contrib.admin", 45 | "django.contrib.auth", 46 | "django.contrib.contenttypes", 47 | "django.contrib.sessions", 48 | "django.contrib.messages", 49 | "django.contrib.staticfiles", 50 | "django_extensions", 51 | "django_filters", 52 | # Django Rest Framework 53 | "rest_framework", 54 | "rest_framework.authtoken", 55 | # Feinstaub 56 | "feinstaub", 57 | "feinstaub.main", 58 | "feinstaub.sensors", 59 | # API 60 | "sensorsafrica", 61 | "corsheaders", 62 | ] 63 | 64 | MIDDLEWARE = [ 65 | "django.middleware.security.SecurityMiddleware", 66 | "whitenoise.middleware.WhiteNoiseMiddleware", 67 | "django.contrib.sessions.middleware.SessionMiddleware", 68 | "corsheaders.middleware.CorsMiddleware", 69 | "django.middleware.common.CommonMiddleware", 70 | "django.middleware.csrf.CsrfViewMiddleware", 71 | "django.contrib.auth.middleware.AuthenticationMiddleware", 72 | "django.contrib.messages.middleware.MessageMiddleware", 73 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 74 | ] 75 | 76 | ROOT_URLCONF = "sensorsafrica.urls" 77 | 78 | REST_FRAMEWORK = { 79 | "DEFAULT_AUTHENTICATION_CLASSES": ( 80 | "rest_framework.authentication.TokenAuthentication", 81 | "rest_framework.authentication.SessionAuthentication", 82 | ), 83 | "DEFAULT_FILTER_BACKENDS": ("django_filters.rest_framework.DjangoFilterBackend",), 84 | } 85 | 86 | TEMPLATES = [ 87 | { 88 | "BACKEND": "django.template.backends.django.DjangoTemplates", 89 | "APP_DIRS": True, 90 | "OPTIONS": { 91 | "context_processors": [ 92 | "django.template.context_processors.debug", 93 | "django.template.context_processors.request", 94 | "django.contrib.auth.context_processors.auth", 95 | "django.contrib.messages.context_processors.messages", 96 | ], 97 | "debug": DEBUG, 98 | }, 99 | } 100 | ] 101 | 102 | WSGI_APPLICATION = "sensorsafrica.wsgi.application" 103 | 104 | SESSION_ENGINE = "django.contrib.sessions.backends.file" 105 | 106 | # Database 107 | # https://docs.djangoproject.com/en/1.10/ref/settings/#databases 108 | 109 | DATABASE_URL = os.getenv( 110 | "SENSORSAFRICA_DATABASE_URL", 111 | "postgres://sensorsafrica:sensorsafrica@localhost:5432/sensorsafrica", 112 | ) 113 | 114 | DATABASES = {"default": dj_database_url.parse(DATABASE_URL), } 115 | 116 | # Password validation 117 | # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators 118 | 119 | AUTH_PASSWORD_VALIDATORS = [ 120 | { 121 | "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" 122 | }, 123 | {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, 124 | {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, 125 | {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, 126 | ] 127 | 128 | 129 | # Internationalization 130 | # https://docs.djangoproject.com/en/1.10/topics/i18n/ 131 | 132 | LANGUAGE_CODE = "en-us" 133 | 134 | TIME_ZONE = "UTC" 135 | 136 | USE_I18N = True 137 | 138 | USE_L10N = True 139 | 140 | USE_TZ = True 141 | 142 | 143 | # Static files (CSS, JavaScript, Images) 144 | # https://docs.djangoproject.com/en/1.10/howto/static-files/ 145 | 146 | STATIC_URL = "/static/" 147 | 148 | STATIC_ROOT = os.path.join(BASE_DIR, "static") 149 | 150 | 151 | # Simplified static file serving. 152 | # https://warehouse.python.org/project/whitenoise/ 153 | STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" 154 | 155 | # Recheck the filesystem to see if any files have changed before responding. 156 | WHITENOISE_AUTOREFRESH = True 157 | 158 | # Celery Broker 159 | CELERY_BROKER_URL = os.environ.get( 160 | "SENSORSAFRICA_RABBITMQ_URL", "amqp://sensorsafrica:sensorsafrica@localhost//" 161 | ) 162 | CELERY_IGNORE_RESULT = True 163 | 164 | CELERY_BEAT_SCHEDULE = { 165 | "statistics-task": { 166 | "task": "sensorsafrica.tasks.calculate_data_statistics", 167 | "schedule": crontab(hour="*", minute=0), 168 | }, 169 | "archive-task": { 170 | "task": "sensorsafrica.tasks.archive_data", 171 | "schedule": crontab(hour="*", minute=0), 172 | }, 173 | "cache-lastactive-nodes-task": { 174 | "task": "sensorsafrica.tasks.cache_lastactive_nodes", 175 | "schedule": crontab(minute="*/5"), 176 | }, 177 | "cache-static-json-data": { 178 | "task": "sensorsafrica.tasks.cache_static_json_data", 179 | "schedule": crontab(minute="*/5"), 180 | }, 181 | "cache-static-json-data-1h-24h": { 182 | "task": "sensorsafrica.tasks.cache_static_json_data_1h_24h", 183 | "schedule": crontab(hour="*", minute=0), 184 | }, 185 | } 186 | 187 | 188 | # Sentry 189 | sentry_sdk.init( 190 | os.environ.get("SENSORSAFRICA_SENTRY_DSN", ""), 191 | integrations=[CeleryIntegration(), DjangoIntegration()], 192 | ) 193 | 194 | 195 | # Put fenstaub migrations into sensorsafrica 196 | MIGRATION_MODULES = { 197 | "sensors": "sensorsafrica.openstuttgart.feinstaub.sensors.migrations" 198 | } 199 | 200 | NETWORKS_OWNER = os.getenv("NETWORKS_OWNER") 201 | -------------------------------------------------------------------------------- /sensorsafrica/static/v2/data.1h.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/static/v2/data.1h.json -------------------------------------------------------------------------------- /sensorsafrica/static/v2/data.24h.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/static/v2/data.24h.json -------------------------------------------------------------------------------- /sensorsafrica/static/v2/data.dust.min.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/static/v2/data.dust.min.json -------------------------------------------------------------------------------- /sensorsafrica/static/v2/data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/static/v2/data.json -------------------------------------------------------------------------------- /sensorsafrica/static/v2/data.temp.min.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CodeForAfrica/sensors.AFRICA-api/f52c30d4de6bf9b52f35a1578b260b5edb750c1e/sensorsafrica/static/v2/data.temp.min.json -------------------------------------------------------------------------------- /sensorsafrica/tasks.py: -------------------------------------------------------------------------------- 1 | from celery import shared_task 2 | from django.core.management import call_command 3 | 4 | 5 | @shared_task 6 | def calculate_data_statistics(): 7 | call_command("calculate_data_statistics") 8 | 9 | 10 | @shared_task 11 | def archive_data(): 12 | call_command("upload_to_ckan") 13 | 14 | 15 | @shared_task 16 | def cache_lastactive_nodes(): 17 | call_command("cache_lastactive_nodes") 18 | 19 | 20 | @shared_task 21 | def cache_static_json_data(): 22 | call_command("cache_static_json_data", interval='5m') 23 | 24 | 25 | @shared_task 26 | def cache_static_json_data_1h_24h(): 27 | call_command("cache_static_json_data", interval='1h') 28 | call_command("cache_static_json_data", interval='24h') 29 | -------------------------------------------------------------------------------- /sensorsafrica/templates/addsensordevice.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base.html" %} 2 | 3 | {% block content %} 4 |
5 | {% csrf_token %} 6 | {{ form.as_p }} 7 | 8 |
9 | {% endblock %} 10 | -------------------------------------------------------------------------------- /sensorsafrica/templates/rest_framework/api.html: -------------------------------------------------------------------------------- 1 | {% extends "rest_framework/base.html" %} 2 | 3 | {% block title %}sensors.AFRICA API{% endblock %} 4 | 5 | {% block branding %}sensors.AFRICA API{% endblock %} 6 | -------------------------------------------------------------------------------- /sensorsafrica/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import math 3 | from dateutil.relativedelta import relativedelta 4 | 5 | import pytest 6 | from django.core.management import call_command 7 | from django.utils import timezone 8 | from feinstaub.sensors.models import (Node, Sensor, SensorData, 9 | SensorDataValue, SensorLocation, 10 | SensorType) 11 | 12 | 13 | @pytest.fixture 14 | def django_db_setup(django_db_setup, django_db_blocker): 15 | with django_db_blocker.unblock(): 16 | call_command("loaddata", "auth.json") 17 | 18 | 19 | @pytest.fixture 20 | def logged_in_user(): 21 | from django.contrib.auth import get_user_model 22 | 23 | user_model = get_user_model() 24 | 25 | user = user_model.objects.get(username="test") 26 | 27 | from rest_framework.authtoken.models import Token 28 | 29 | Token.objects.create(user=user) 30 | 31 | return user 32 | 33 | 34 | @pytest.fixture 35 | def location(): 36 | l, x = SensorLocation.objects.get_or_create(description="somewhere") 37 | return l 38 | 39 | 40 | @pytest.fixture 41 | def sensor_type(): 42 | st, x = SensorType.objects.get_or_create( 43 | uid="a", name="b", manufacturer="c") 44 | return st 45 | 46 | @pytest.fixture 47 | def node(logged_in_user, location): 48 | n, x = Node.objects.get_or_create( 49 | uid="test123", owner=logged_in_user, location=location 50 | ) 51 | return n 52 | 53 | 54 | @pytest.fixture 55 | def sensor(logged_in_user, sensor_type, node): 56 | s, x = Sensor.objects.get_or_create(node=node, sensor_type=sensor_type) 57 | return s 58 | 59 | 60 | @pytest.fixture 61 | def locations(): 62 | return [ 63 | SensorLocation.objects.get_or_create( 64 | city="Dar es Salaam", country="Tanzania", description="active")[0], 65 | SensorLocation.objects.get_or_create( 66 | city="Bagamoyo", country="Tanzania", description="inactive")[0], 67 | SensorLocation.objects.get_or_create( 68 | city="Mombasa", country="Kenya", description="inactive")[0], 69 | SensorLocation.objects.get_or_create( 70 | city="Nairobi", country="Kenya", description="inactive")[0], 71 | SensorLocation.objects.get_or_create( 72 | city="Dar es Salaam", country="Tanzania", description="active | some other node location")[0], 73 | ] 74 | 75 | 76 | @pytest.fixture 77 | def nodes(logged_in_user, locations): 78 | return [ 79 | Node.objects.get_or_create( 80 | uid="0", owner=logged_in_user, location=locations[0])[0], 81 | Node.objects.get_or_create( 82 | uid="1", owner=logged_in_user, location=locations[1])[0], 83 | Node.objects.get_or_create( 84 | uid="2", owner=logged_in_user, location=locations[2])[0], 85 | Node.objects.get_or_create( 86 | uid="3", owner=logged_in_user, location=locations[3])[0], 87 | Node.objects.get_or_create( 88 | uid="4", owner=logged_in_user, location=locations[4])[0], 89 | ] 90 | 91 | 92 | @pytest.fixture 93 | def sensors(sensor_type, nodes): 94 | return [ 95 | # Active Dar Sensor 96 | Sensor.objects.get_or_create( 97 | node=nodes[0], sensor_type=sensor_type, public=True)[0], 98 | # Inactive with last data push beyond active threshold 99 | Sensor.objects.get_or_create( 100 | node=nodes[1], sensor_type=sensor_type)[0], 101 | # Inactive without any data 102 | Sensor.objects.get_or_create( 103 | node=nodes[2], sensor_type=sensor_type)[0], 104 | # Active Nairobi Sensor 105 | Sensor.objects.get_or_create( 106 | node=nodes[3], sensor_type=sensor_type)[0], 107 | # Active Dar Sensor another location 108 | Sensor.objects.get_or_create( 109 | node=nodes[4], sensor_type=sensor_type)[0], 110 | ] 111 | 112 | 113 | @pytest.fixture(autouse=True) 114 | def sensordata(sensors, locations): 115 | 116 | sensor_datas = [ 117 | # Bagamoyo SensorData 118 | SensorData(sensor=sensors[1], location=locations[1]), 119 | # Dar es Salaam SensorData 120 | SensorData(sensor=sensors[0], location=locations[0]), 121 | SensorData(sensor=sensors[0], location=locations[0]), 122 | SensorData(sensor=sensors[0], location=locations[0]), 123 | SensorData(sensor=sensors[0], location=locations[0]), 124 | SensorData(sensor=sensors[0], location=locations[0]), 125 | SensorData(sensor=sensors[0], location=locations[0]), 126 | SensorData(sensor=sensors[0], location=locations[0]), 127 | SensorData(sensor=sensors[0], location=locations[0]), 128 | ] 129 | # Nairobi SensorData 130 | for i in range(100): 131 | sensor_datas.append(SensorData( 132 | sensor=sensors[3], location=locations[3])) 133 | 134 | # Dar es Salaam another node location SensorData 135 | for i in range(6): 136 | sensor_datas.append(SensorData( 137 | sensor=sensors[4], location=locations[4])) 138 | 139 | data = SensorData.objects.bulk_create(sensor_datas) 140 | 141 | data[1].update_modified = False 142 | data[1].timestamp = timezone.now() - datetime.timedelta(minutes=40) 143 | data[1].save() 144 | 145 | return data 146 | 147 | 148 | @pytest.fixture(autouse=True) 149 | def datavalues(sensors, sensordata): 150 | data_values = [ 151 | # Bagamoyo 152 | SensorDataValue( 153 | sensordata=sensordata[0], value="2", value_type="humidity"), 154 | # Dar es salaam a day ago's data 155 | SensorDataValue(sensordata=sensordata[1], value="1", value_type="P2"), 156 | SensorDataValue(sensordata=sensordata[2], value="2", value_type="P2"), 157 | # Dar es salaam today's data avg 5.5 158 | SensorDataValue(sensordata=sensordata[3], value="3", value_type="P2"), 159 | SensorDataValue(sensordata=sensordata[4], value="4", value_type="P2"), 160 | SensorDataValue(sensordata=sensordata[5], value="5", value_type="P2"), 161 | SensorDataValue(sensordata=sensordata[6], value="6", value_type="P2"), 162 | SensorDataValue(sensordata=sensordata[7], value="7", value_type="P2"), 163 | SensorDataValue(sensordata=sensordata[8], value="0", value_type="P1"), 164 | SensorDataValue(sensordata=sensordata[8], value="8", value_type="P2"), 165 | SensorDataValue( 166 | sensordata=sensordata[8], value="some time stamp", value_type="timestamp" 167 | ), 168 | ] 169 | 170 | # Nairobi SensorDataValues 171 | for i in range(100): 172 | data_values.append( 173 | SensorDataValue( 174 | sensordata=sensordata[9 + i], value="0", value_type="P2") 175 | ) 176 | 177 | # Dar es Salaam another node location SensorDataValues 178 | for i in range(6): 179 | data_values.append(SensorDataValue( 180 | sensordata=sensordata[109 + i], value="0.0", value_type="P2")) 181 | 182 | values = SensorDataValue.objects.bulk_create(data_values) 183 | 184 | now = timezone.now() 185 | 186 | # Set Dar es salaam a day ago's data 187 | values[1].update_modified = False 188 | values[1].created = now - datetime.timedelta(days=2) 189 | values[1].save() 190 | values[2].update_modified = False 191 | values[2].created = now - datetime.timedelta(days=2) 192 | values[2].save() 193 | 194 | # Set data received at different hours 195 | values[3].update_modified = False 196 | values[3].created = now - datetime.timedelta(hours=1) 197 | values[3].save() 198 | values[4].update_modified = False 199 | values[4].created = now - datetime.timedelta(hours=2) 200 | values[4].save() 201 | values[5].update_modified = False 202 | values[5].created = now - datetime.timedelta(hours=3) 203 | values[5].save() 204 | 205 | 206 | @pytest.fixture 207 | def sensorsdatastats(datavalues): 208 | from django.core.management import call_command 209 | 210 | call_command("calculate_data_statistics") 211 | 212 | 213 | @pytest.fixture 214 | def additional_sensorsdatastats(sensors, locations, sensorsdatastats): 215 | sensordata = SensorData.objects.bulk_create([ 216 | SensorData(sensor=sensors[0], location=locations[0]), 217 | SensorData(sensor=sensors[0], location=locations[0]), 218 | SensorData(sensor=sensors[0], location=locations[0]), 219 | ]) 220 | 221 | SensorDataValue.objects.bulk_create([ 222 | # Dar es salaam today's additional datavalues avg 4 for P2 223 | SensorDataValue(sensordata=sensordata[0], value="4", value_type="P2"), 224 | SensorDataValue(sensordata=sensordata[1], value="4", value_type="P2"), 225 | SensorDataValue(sensordata=sensordata[2], value="4", value_type="P2"), 226 | ]) 227 | 228 | from django.core.management import call_command 229 | 230 | call_command("calculate_data_statistics") 231 | 232 | 233 | @pytest.fixture 234 | def large_sensorsdatastats(sensors, locations): 235 | 236 | now = timezone.now() 237 | months = 6 238 | points = math.floor((now - (now - relativedelta(months=months-1))).days * 24 * 60 / 5) 239 | minutes = points * 5 * months 240 | for point in range(1, points): 241 | created_sd = SensorData.objects.create(sensor=sensors[0], location=locations[0]) 242 | created_sv = SensorDataValue.objects.create(sensordata=created_sd, value="4", value_type="P2") 243 | created_sv.update_modified = False 244 | created_sv.created = now - datetime.timedelta(minutes=point * 5) 245 | created_sv.save() 246 | 247 | last_date = created_sv.created 248 | 249 | from django.core.management import call_command 250 | 251 | call_command("calculate_data_statistics") 252 | 253 | return { 254 | 'months': months, 255 | 'minutes': minutes, 256 | 'last_date': last_date 257 | } 258 | 259 | 260 | @pytest.fixture 261 | def last_active(sensors, locations, sensorsdatastats): 262 | timestamps = [ 263 | timezone.now(), 264 | timezone.now() + datetime.timedelta(minutes=2), 265 | timezone.now() + datetime.timedelta(minutes=4) 266 | ] 267 | sensordata = SensorData.objects.bulk_create([ 268 | SensorData( 269 | sensor=sensors[0], location=locations[0], timestamp=timestamps[0]), 270 | SensorData( 271 | sensor=sensors[3], location=locations[3], timestamp=timestamps[1]), 272 | SensorData( 273 | sensor=sensors[4], location=locations[4], timestamp=timestamps[2]), 274 | ]) 275 | 276 | SensorDataValue.objects.bulk_create([ 277 | SensorDataValue( 278 | sensordata=sensordata[0], value="4", value_type="P2"), 279 | SensorDataValue( 280 | sensordata=sensordata[1], value="4", value_type="P1"), 281 | # Won't be tracked as last active 282 | SensorDataValue( 283 | sensordata=sensordata[2], value="4", value_type="Temp"), 284 | ]) 285 | 286 | from django.core.management import call_command 287 | 288 | call_command("cache_lastactive_nodes") 289 | 290 | return timestamps 291 | -------------------------------------------------------------------------------- /sensorsafrica/tests/settings.py: -------------------------------------------------------------------------------- 1 | from sensorsafrica.settings import * 2 | 3 | 4 | CACHES = { 5 | 'default': { 6 | 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_city_view.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.django_db 5 | class TestCityView: 6 | def test_getting_cities(self, client, sensorsdatastats): 7 | response = client.get("/v2/cities/", format="json") 8 | assert response.status_code == 200 9 | 10 | data = response.json() 11 | 12 | assert data["count"] == 3 13 | 14 | assert { 15 | "latitude": "-6.79240000000", 16 | "longitude": "39.20830000000", 17 | "slug": "dar-es-salaam", 18 | "name": "Dar es Salaam", 19 | "country": "Tanzania", 20 | "label": "Dar es Salaam, Tanzania", 21 | } in data["results"] 22 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_filter_view.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import datetime 3 | from django.utils import timezone 4 | 5 | 6 | @pytest.mark.django_db 7 | class TestGettingFilteringPast5MinutesData: 8 | def test_getting_past_5_minutes_data_for_sensor_type(self, client, sensor_type, sensordata): 9 | response = client.get("/v1/filter/?type=%s" % sensor_type.uid, format="json") 10 | assert response.status_code == 200 11 | 12 | results = response.json() 13 | 14 | # subtract 1 data since one the many data is set as posted 40 minutes ago, not 5 minutes 15 | assert len(results) == (len(list(filter( 16 | lambda sd: sd.sensor.sensor_type.uid == sensor_type.uid, sensordata))) - 1) 17 | 18 | def test_getting_past_5_minutes_data_for_sensor_city(self, client, sensordata): 19 | response = client.get("/v1/filter/?city=Bagamoyo", format="json") 20 | assert response.status_code == 200 21 | 22 | results = response.json() 23 | 24 | # no subtract 1 since the 1 data posted 40 minutes ago is for Dar es Salaam 25 | assert len(results) == (len(list(filter( 26 | lambda sd: sd.location.city == "Bagamoyo", sensordata)))) 27 | 28 | def test_getting_past_5_minutes_data_for_sensor_country(self, client, sensordata): 29 | response = client.get("/v1/filter/?country=Tanzania", format="json") 30 | assert response.status_code == 200 31 | 32 | results = response.json() 33 | 34 | # subtract 1 data since one the many data in Tanzania is set as posted 40 minutes ago, not 5 minutes 35 | assert len(results) == (len(list(filter( 36 | lambda sd: sd.location.country == "Tanzania", sensordata))) - 1) 37 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_full_push.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pytz 3 | 4 | from rest_framework.test import APIRequestFactory 5 | from feinstaub.sensors.views import PostSensorDataView 6 | from feinstaub.sensors.models import SensorData 7 | 8 | 9 | @pytest.mark.django_db 10 | class TestSensorDataPushFull: 11 | 12 | @pytest.fixture 13 | def data_fixture(self): 14 | return { 15 | "sampling_rate": "15000", 16 | "timestamp": "2015-04-05 22:10:10+02:00", 17 | "sensordatavalues": [{"value": 10, "value_type": "P1"}, 18 | {"value": 99, "value_type": "P2"}] 19 | } 20 | 21 | def test_full_data_push(self, sensor, data_fixture): 22 | factory = APIRequestFactory() 23 | view = PostSensorDataView 24 | url = '/v1/push-sensor-data/' 25 | request = factory.post(url, data_fixture, 26 | format='json') 27 | 28 | # authenticate sensor 29 | request.META['HTTP_SENSOR'] = sensor.node.uid 30 | # FIXME: test for HTTP_NODE 31 | 32 | view_function = view.as_view({'post': 'create'}) 33 | response = view_function(request) 34 | 35 | assert response.status_code == 201 36 | 37 | sd = SensorData.objects.get(sensor=response.data['sensor']) 38 | 39 | assert sd.sensordatavalues.count() == 2 40 | assert sd.sensordatavalues.get(value_type="P1").value ==\ 41 | str(data_fixture['sensordatavalues'][0]['value']) 42 | assert sd.sensordatavalues.get(value_type="P2").value ==\ 43 | str(data_fixture['sensordatavalues'][1]['value']) 44 | 45 | assert sd.location == sensor.node.location 46 | 47 | cest = pytz.timezone('Europe/Berlin') 48 | assert str(cest.normalize(sd.timestamp)) == data_fixture['timestamp'] 49 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_large_dataset.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import pytest 4 | from django.utils import timezone 5 | 6 | 7 | @pytest.mark.django_db 8 | class TestGettingDataFromLargeDataset: 9 | 10 | def test_getting_air_data_on_large_dataset(self, client, large_sensorsdatastats): 11 | response = client.get( 12 | "/v2/data/air/?city=dar-es-salaam&interval=month&from=%s" % 13 | large_sensorsdatastats["last_date"].date(), 14 | format="json", 15 | ) 16 | assert response.status_code == 200 17 | 18 | data = response.json() 19 | 20 | assert data["count"] == 1 21 | 22 | assert type(data["results"][0]["P2"]) == list 23 | assert len(data["results"][0]["P2"]) == large_sensorsdatastats["months"] 24 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_lastactive.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import pytz 3 | 4 | from rest_framework.test import APIRequestFactory 5 | from feinstaub.sensors.views import PostSensorDataView 6 | from sensorsafrica.api.models import LastActiveNodes 7 | 8 | 9 | @pytest.mark.django_db 10 | class TestLastActiveSensor: 11 | def test_lastactive_command(self, sensors, last_active): 12 | sensors0_node = LastActiveNodes.objects.filter( 13 | node=sensors[0].node_id 14 | ).get() 15 | sensors3_node = LastActiveNodes.objects.filter( 16 | node=sensors[3].node_id 17 | ).get() 18 | sensors4_node = LastActiveNodes.objects.filter( 19 | node=sensors[4].node_id 20 | ).get() 21 | 22 | assert sensors0_node.last_data_received_at == last_active[0] 23 | assert sensors3_node.last_data_received_at == last_active[1] 24 | # Sensor is last active when they send the P1 and P2 25 | # Sensor[4] sent a none P1/P2 at timestamp last_active[2] 26 | assert sensors4_node.last_data_received_at != last_active[2] 27 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_nested_write_sensordata_push.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from rest_framework.test import APIRequestFactory 3 | import pytest 4 | 5 | from feinstaub.sensors.views import PostSensorDataView 6 | 7 | 8 | @pytest.mark.django_db 9 | class TestSensorDataPush: 10 | 11 | @pytest.fixture(params=[ 12 | # value, status_code, count 13 | [[{"value": 10, "value_type": "P1"}], 201, 1], 14 | [ 15 | [ 16 | {"value": 10, "value_type": "P1"}, 17 | {"value": 99, "value_type": "P2"} 18 | ], 19 | 201, 20 | 2 21 | ], 22 | # failes: 23 | [[], 400, 0], # because list of sensordatavalues is empty 24 | ['INVALID', 400, 1], 25 | [['INVALID'], 400, 1], 26 | [[{'INVALID_KEY': 1}], 400, 1], 27 | ]) 28 | def sensordatavalue_fixture(self, request): 29 | keys = ["value", "status_code", "count"] 30 | return dict(zip(keys, request.param)) 31 | 32 | def test_sensordata_push(self, sensor, sensordatavalue_fixture): 33 | factory = APIRequestFactory() 34 | view = PostSensorDataView 35 | url = '/v1/push-sensor-data/' 36 | request = factory.post(url, {'sensordatavalues': 37 | sensordatavalue_fixture.get('value'), }, 38 | format='json') 39 | 40 | # FIXME: test for HTTP_NODE 41 | request.META['HTTP_SENSOR'] = sensor.node.uid 42 | 43 | view_function = view.as_view({'post': 'create'}) 44 | response = view_function(request) 45 | 46 | if isinstance(response.data, dict): 47 | assert len(response.data.get('sensordatavalues')) ==\ 48 | sensordatavalue_fixture.get('count') 49 | 50 | assert response.status_code ==\ 51 | sensordatavalue_fixture.get('status_code') 52 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_node_view.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import datetime 3 | from django.utils import timezone 4 | 5 | from rest_framework.test import APIRequestFactory 6 | 7 | from feinstaub.sensors.models import Node 8 | from sensorsafrica.api.v2.views import NodesView 9 | 10 | 11 | @pytest.mark.django_db 12 | class TestNodesView: 13 | @pytest.fixture 14 | def data_fixture(self): 15 | return { 16 | "uid": "testnode1", 17 | } 18 | 19 | def test_create_node(self, data_fixture, logged_in_user, location): 20 | data_fixture["location"] = location.id 21 | data_fixture["owner"] = logged_in_user.id 22 | factory = APIRequestFactory() 23 | url = "/v2/nodes/" 24 | request = factory.post(url, data_fixture, format="json") 25 | 26 | # authenticate request 27 | request.user = logged_in_user 28 | 29 | view = NodesView 30 | view_function = view.as_view({"post": "create"}) 31 | response = view_function(request) 32 | 33 | assert response.status_code == 201 34 | node = Node.objects.get(uid=response.data["uid"]) 35 | assert node.uid == data_fixture["uid"] 36 | assert node.location == location 37 | assert node.owner == logged_in_user 38 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_sensor_location_view.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import datetime 3 | from django.utils import timezone 4 | 5 | from rest_framework.test import APIRequestFactory 6 | 7 | from feinstaub.sensors.models import SensorLocation 8 | from sensorsafrica.api.v2.views import SensorLocationsView 9 | 10 | 11 | @pytest.mark.django_db 12 | class TestSensorLocationsView: 13 | @pytest.fixture 14 | def data_fixture(self): 15 | return { 16 | "location": "Code for Africa Offices", 17 | } 18 | 19 | def test_create_sensor_location(self, data_fixture, logged_in_user): 20 | data_fixture["owner"] = logged_in_user.id 21 | factory = APIRequestFactory() 22 | url = "/v2/locations/" 23 | request = factory.post(url, data_fixture, format="json") 24 | 25 | # authenticate request 26 | request.user = logged_in_user 27 | 28 | view = SensorLocationsView 29 | view_function = view.as_view({"post": "create"}) 30 | response = view_function(request) 31 | 32 | assert response.status_code == 201 33 | sensor_type = SensorLocation.objects.get(id=response.data["id"]) 34 | 35 | assert sensor_type.location == data_fixture["location"] 36 | assert sensor_type.owner == logged_in_user 37 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_sensor_type_view.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import datetime 3 | from django.utils import timezone 4 | 5 | from rest_framework.test import APIRequestFactory 6 | 7 | from feinstaub.sensors.models import SensorType 8 | from sensorsafrica.api.v2.views import SensorTypesView 9 | 10 | 11 | @pytest.mark.django_db 12 | class TestSensorTypesView: 13 | @pytest.fixture 14 | def data_fixture(self): 15 | return { 16 | "uid": "nm1", 17 | "name": "N1", 18 | "manufacturer": "M1", 19 | } 20 | 21 | def test_create_sensor_type(self, data_fixture, logged_in_user): 22 | factory = APIRequestFactory() 23 | url = "/v2/sensor-types/" 24 | request = factory.post(url, data_fixture, format="json") 25 | 26 | # authenticate request 27 | request.user = logged_in_user 28 | 29 | view = SensorTypesView 30 | view_function = view.as_view({"post": "create"}) 31 | response = view_function(request) 32 | 33 | assert response.status_code == 201 34 | sensor_type = SensorType.objects.get(id=response.data["id"]) 35 | 36 | assert sensor_type.name == data_fixture["name"] 37 | assert sensor_type.manufacturer == data_fixture["manufacturer"] 38 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_sensor_view.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import datetime 3 | from django.utils import timezone 4 | 5 | from rest_framework.test import APIRequestFactory 6 | 7 | from feinstaub.sensors.models import Sensor 8 | from sensorsafrica.api.v2.views import SensorsView 9 | 10 | 11 | @pytest.mark.django_db 12 | class TestSensorsView: 13 | @pytest.fixture 14 | def data_fixture(self): 15 | return { 16 | "pin": "1", 17 | "sensor_type": 1, 18 | "node": 1, 19 | "public": False, 20 | } 21 | 22 | def test_create_sensor(self, data_fixture, logged_in_user, node, sensor_type): 23 | data_fixture["node"] = node.id 24 | data_fixture["sensor_type"] = sensor_type.id 25 | factory = APIRequestFactory() 26 | url = "/v2/sensors/" 27 | request = factory.post(url, data_fixture, format="json") 28 | 29 | # authenticate request 30 | request.user = logged_in_user 31 | 32 | view = SensorsView 33 | view_function = view.as_view({"post": "create"}) 34 | response = view_function(request) 35 | 36 | assert response.status_code == 201 37 | sensor = Sensor.objects.get(id=response.data["id"]) 38 | assert sensor.pin == data_fixture["pin"] 39 | assert sensor.public == data_fixture["public"] 40 | 41 | def test_getting_past_5_minutes_data_for_sensor_with_id(self, client, sensors): 42 | response = client.get("/v1/sensors/%s/" % sensors[0].id, format="json") 43 | assert response.status_code == 200 44 | 45 | results = response.json() 46 | 47 | # 7 are recent since one the 8 data is 40 minutes ago 48 | assert len(results) == 7 49 | assert "sensordatavalues" in results[0] 50 | assert "timestamp" in results[0] 51 | assert "value_type" in results[0]["sensordatavalues"][0] 52 | assert "value" in results[0]["sensordatavalues"][0] 53 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_sensordata_view.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import datetime 3 | from django.utils import timezone 4 | 5 | 6 | @pytest.mark.django_db 7 | class TestGettingRawData: 8 | def test_getting_all_data(self, client, logged_in_user, sensors): 9 | response = client.get("/v1/data/", format="json") 10 | assert response.status_code == 200 11 | 12 | results = response.json() 13 | 14 | assert results["count"] == 8 15 | 16 | def test_getting_filtered_data_by_public_sensor(self, client, sensors): 17 | response = client.get("/v1/data/?sensor=%s" % sensors[0].id, format="json") 18 | assert response.status_code == 200 19 | 20 | results = response.json() 21 | 22 | # 8 Dar es Salaam SensorDatas 23 | # Only one senor is made public 24 | assert results["count"] == 8 25 | 26 | def test_getting_filtered_data_by_private_sensor(self, client, sensors): 27 | response = client.get("/v1/data/?sensor=%s" % sensors[1].id, format="json") 28 | assert response.status_code == 200 29 | 30 | results = response.json() 31 | 32 | assert results["count"] == 0 33 | 34 | def test_getting_filtered_data_by_timestamp(self, client): 35 | # Filter out one Dar es Salaam SensorData 36 | # It has timestamp 40 minutes ago 37 | timestamp = timezone.now() - datetime.timedelta(minutes=40) 38 | # It won't accept the tz information unless the '+' sign is encoded %2B 39 | timestamp = timestamp.replace(tzinfo=None) 40 | response = client.get("/v1/data/?timestamp__gte=%s" % timestamp, format="json") 41 | assert response.status_code == 200 42 | 43 | results = response.json() 44 | 45 | assert results["count"] == 7 46 | -------------------------------------------------------------------------------- /sensorsafrica/tests/test_sensordatastats_view.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import pytest 4 | from django.utils import timezone 5 | 6 | 7 | @pytest.mark.django_db 8 | class TestGettingData: 9 | def test_getting_air_data_now(self, client, sensorsdatastats): 10 | response = client.get("/v2/data/air/?city=dar-es-salaam", format="json") 11 | assert response.status_code == 200 12 | 13 | data = response.json() 14 | 15 | assert data["count"] == 1 16 | 17 | result = data["results"][0] 18 | 19 | assert "P1" in result 20 | assert result["P1"]["average"] == 0.0 21 | assert result["P1"]["maximum"] == 0.0 22 | assert result["P1"]["minimum"] == 0.0 23 | 24 | assert "P2" in result 25 | 26 | # One node has an average of 5.5 for sample size 6 27 | # Another node has an average of 0 for sample size 6 28 | # The average for the city will be (5.5 * 6 + 0 * 6) / (6 + 6) = 2.75 29 | assert result["P2"]["average"] == 2.75 30 | 31 | assert result["P2"]["maximum"] == 8.0 32 | assert result["P2"]["minimum"] == 0.0 33 | 34 | def test_getting_air_data_now_all_cities(self, client, sensorsdatastats): 35 | response = client.get("/v2/data/air/", format="json") 36 | assert response.status_code == 200 37 | 38 | data = response.json() 39 | 40 | assert data["count"] == 3 41 | 42 | results = data["results"] 43 | 44 | assert results[0]["city_slug"] == "bagamoyo" 45 | assert results[1]["city_slug"] == "dar-es-salaam" 46 | assert "P1" in results[1] 47 | assert results[1]["city_slug"] == "dar-es-salaam" 48 | assert "P2" in results[1] 49 | assert results[2]["city_slug"] == "nairobi" 50 | 51 | def test_getting_air_data_now_filter_cities(self, client, sensorsdatastats): 52 | response = client.get( 53 | "/v2/data/air/?city=dar-es-salaam,bagamoyo", format="json" 54 | ) 55 | assert response.status_code == 200 56 | 57 | data = response.json() 58 | 59 | assert data["count"] == 2 60 | 61 | results = data["results"] 62 | 63 | assert results[0]["city_slug"] == "bagamoyo" 64 | assert results[1]["city_slug"] == "dar-es-salaam" 65 | assert "P1" in results[1] 66 | assert results[1]["city_slug"] == "dar-es-salaam" 67 | assert "P2" in results[1] 68 | 69 | def test_getting_air_data_value_type(self, client, sensorsdatastats): 70 | response = client.get( 71 | "/v2/data/air/?city=dar-es-salaam&value_type=P2", format="json" 72 | ) 73 | assert response.status_code == 200 74 | 75 | data = response.json() 76 | 77 | assert data["count"] == 1 78 | assert "P2" in data["results"][0] 79 | assert "P1" not in data["results"][0] 80 | assert "temperature" not in data["results"][0] 81 | assert "humidity" not in data["results"][0] 82 | 83 | def test_getting_air_data_from_date(self, client, sensorsdatastats): 84 | response = client.get( 85 | "/v2/data/air/?city=dar-es-salaam&from=%s" 86 | % (timezone.now() - datetime.timedelta(days=2)).date(), 87 | format="json", 88 | ) 89 | assert response.status_code == 200 90 | 91 | data = response.json() 92 | 93 | assert type(data["results"][0]["P2"]) == list 94 | 95 | # Data is in descending order by date 96 | most_recent_value = data["results"][0]["P2"][0] 97 | most_recent_date = datetime.datetime.strptime( 98 | most_recent_value["end_datetime"], "%Y-%m-%dT%H:%M:%SZ" 99 | ) 100 | 101 | # Check today is not included 102 | assert most_recent_date.date() < datetime.datetime.today().date() 103 | 104 | def test_getting_air_data_from_date_to_date(self, client, sensorsdatastats): 105 | now = timezone.now() 106 | response = client.get( 107 | "/v2/data/air/?city=dar-es-salaam&from=%s&to=%s" % (now.date(), now.date()), 108 | format="json", 109 | ) 110 | assert response.status_code == 200 111 | 112 | data = response.json() 113 | 114 | assert data["count"] == 1 115 | assert type(data["results"][0]["P1"]) == list 116 | assert type(data["results"][0]["P2"]) == list 117 | 118 | def test_getting_air_data_with_invalid_request(self, client, sensorsdatastats): 119 | response = client.get( 120 | "/v2/data/air/?city=dar-es-salaam&to=2019-02-08", format="json" 121 | ) 122 | assert response.status_code == 400 123 | assert response.json() == {"from": "Must be provide along with to query"} 124 | 125 | def test_getting_air_data_with_invalid_from_request(self, client, sensorsdatastats): 126 | response = client.get( 127 | "/v2/data/air/?city=dar-es-salaam&from=2019-23-08", format="json" 128 | ) 129 | assert response.status_code == 400 130 | assert response.json() == {"from": "Must be a date in the format Y-m-d."} 131 | 132 | def test_getting_air_data_with_invalid_to_request(self, client, sensorsdatastats): 133 | response = client.get( 134 | "/v2/data/air/?city=dar-es-salaam&from=2019-02-08&to=08-02-2019", 135 | format="json", 136 | ) 137 | assert response.status_code == 400 138 | assert response.json() == {"to": "Must be a date in the format Y-m-d."} 139 | 140 | def test_getting_air_data_now_with_additional_values( 141 | self, client, additional_sensorsdatastats 142 | ): 143 | response = client.get("/v2/data/air/?city=dar-es-salaam", format="json") 144 | assert response.status_code == 200 145 | 146 | data = response.json() 147 | 148 | assert data["count"] == 1 149 | 150 | result = data["results"][0] 151 | 152 | assert "P1" in result 153 | assert result["P1"]["average"] == 0.0 154 | assert result["P1"]["maximum"] == 0.0 155 | assert result["P1"]["minimum"] == 0.0 156 | 157 | assert "P2" in result 158 | 159 | # The previous average was 2.75 for sample size 12 160 | # The addional data average is 4 for sample size 3 161 | # The new average is (2.75 * 12 + 4 * 3) / (12 + 3) = 3 162 | assert result["P2"]["average"] == 3 163 | 164 | assert result["P2"]["maximum"] == 8.0 165 | assert result["P2"]["minimum"] == 0.0 166 | 167 | def test_getting_air_data_by_hour(self, client, sensorsdatastats): 168 | response = client.get( 169 | "/v2/data/air/?city=dar-es-salaam&interval=hour", 170 | format="json", 171 | ) 172 | assert response.status_code == 200 173 | 174 | data = response.json() 175 | 176 | assert data["count"] == 1 177 | 178 | assert type(data["results"][0]["P1"]) == list 179 | assert len(data["results"][0]["P1"]) == 1 180 | assert type(data["results"][0]["P2"]) == list 181 | assert len(data["results"][0]["P2"]) == 4 182 | 183 | def test_getting_air_data_by_month(self, client, sensorsdatastats): 184 | response = client.get( 185 | "/v2/data/air/?city=dar-es-salaam&interval=month", 186 | format="json", 187 | ) 188 | assert response.status_code == 200 189 | 190 | data = response.json() 191 | 192 | assert data["count"] == 1 193 | 194 | assert type(data["results"][0]["P1"]) == list 195 | assert len(data["results"][0]["P1"]) == 1 196 | assert type(data["results"][0]["P2"]) == list 197 | assert len(data["results"][0]["P2"]) == 1 198 | -------------------------------------------------------------------------------- /sensorsafrica/urls.py: -------------------------------------------------------------------------------- 1 | """sensorsafrica URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.11/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.conf.urls import include, url 17 | from django.contrib import admin 18 | from django.contrib.staticfiles.urls import staticfiles_urlpatterns 19 | from django.views.generic.base import RedirectView 20 | from feinstaub.sensors.views import AddSensordeviceView 21 | from rest_framework.authtoken.views import obtain_auth_token 22 | from rest_framework.documentation import include_docs_urls 23 | 24 | from .api.v1.router import push_sensor_data_urls 25 | from .api.v2.router import api_urls as sensors_api_v2 26 | 27 | urlpatterns = [ 28 | url(r"^$", RedirectView.as_view(url="/docs/", permanent=False)), 29 | url(r"^admin/", admin.site.urls), 30 | url(r"^v1/", include(push_sensor_data_urls)), 31 | url(r"^v2/", include(sensors_api_v2)), 32 | url(r"^get-auth-token/", obtain_auth_token), 33 | url(r"^auth/", include("rest_framework.urls", namespace="rest_framework")), 34 | url(r"^docs/", include_docs_urls(title="sensors.Africa API")), 35 | url(r"^adddevice/", AddSensordeviceView.as_view(), name="adddevice"), 36 | ] + staticfiles_urlpatterns() 37 | -------------------------------------------------------------------------------- /sensorsafrica/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for sensorsafrica project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sensorsafrica.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | with open("VERSION", "r") as f: 7 | version = f.read().strip() 8 | 9 | with open("requirements.txt") as f: 10 | install_requires = f.read().splitlines() 11 | 12 | setuptools.setup( 13 | name="sensorsafrica", 14 | version=version, 15 | author="CodeForAfrica", 16 | author_email="hello@codeforafrica.org", 17 | description="API to save and access data from deployed sensors in Africa.", 18 | long_description=long_description, 19 | long_description_content_type="text/markdown", 20 | url="https://github.com/CodeForAfricaLabs/sensors.AFRICA-api", 21 | packages=setuptools.find_packages(), 22 | install_requires=install_requires, 23 | classifiers=[ 24 | "Programming Language :: Python :: 3", 25 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 26 | "Operating System :: OS Independent", 27 | ], 28 | ) 29 | --------------------------------------------------------------------------------