├── .gitignore ├── .idea ├── .gitignore ├── Raspberry_Pi_monitoring_system.iml ├── deployment.xml ├── inspectionProfiles │ ├── Project_Default.xml │ └── profiles_settings.xml ├── misc.xml ├── modules.xml └── vcs.xml ├── LICENSE ├── README.md ├── alerts ├── __init__.py └── telegram_bot │ ├── TelegramBot.py │ └── __init__.py ├── configs ├── boards │ └── rpi_board.json ├── cameras.json ├── imaging_states.json ├── main.json ├── phyto_states.json ├── pipelines │ ├── imaging.json │ ├── phyto.json │ ├── sensing.json │ └── stayalive.json └── pml_sensors.json ├── delete.sh ├── main.py ├── monitoring_system ├── __init__.py ├── drivers │ ├── __init__.py │ ├── cameras │ │ ├── CameraDriver.py │ │ ├── CanonCameraDriver.py │ │ ├── WebCameraDriver.py │ │ └── __init__.py │ └── sensors │ │ ├── AnalogReadSensor.py │ │ ├── Board.py │ │ ├── Dht11Sensor.py │ │ ├── DigitalReadSensor.py │ │ ├── Sensor.py │ │ ├── SwitchSensor.py │ │ ├── __init__.py │ │ └── sensor_factory.py ├── logger │ ├── Logger.py │ └── __init__.py ├── scheduler │ ├── PipelineExecutor.py │ ├── Scheduler.py │ └── __init__.py └── utils │ ├── __init__.py │ ├── average.py │ ├── csv.py │ ├── get_serial_number.py │ ├── get_time.py │ ├── gpioexp.py │ ├── json.py │ ├── list_dirs.py │ ├── preprocess.py │ ├── preprocess_cameras.py │ ├── preprocess_sensors.py │ └── txt.py ├── requirements.txt ├── start.sh └── streamlit_server ├── refresh_server.py ├── server.py ├── start_autorefresh.sh └── start_server.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | 132 | 133 | # Raspberry Pi monitoring system 134 | data/ 135 | logs/ 136 | serial_number.txt 137 | configs/api/telegram.json 138 | 139 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Datasource local storage ignored files 5 | /dataSources/ 6 | /dataSources.local.xml 7 | # Editor-based HTTP Client requests 8 | /httpRequests/ 9 | -------------------------------------------------------------------------------- /.idea/Raspberry_Pi_monitoring_system.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /.idea/deployment.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 81 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Sergey Nesteruk 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Monitoring system 2 | This project is a monitoring system implemented on Raspberry Pi. 3 | 4 | The system includes: 5 | * Configuring multiple independent pipelines simultaneously 6 | * Scheduling pipelines 7 | * Collecting data from generic sensors and cameras 8 | * Logging 9 | * Visualising collected data on local web server 10 | * Automated continuation of work after restart if interrupted 11 | 12 | The system can be easily extended to work with various shields and sensors. 13 | 14 | 15 | *All CLI commands assume that you are using Linux (Raspbian).* 16 | 17 | 18 | ## System structure 19 | 1. `main.py` is the entry point of the program. 20 | 2. `start.sh` is used to run the monitoring system. 21 | 3. `delete.sh` is used to delete all the collected data. Be careful when using it! 22 | 4. `configs/` folder contains all the configurations: 23 | 1. `main.json` includes all other configs. Here we have to specify: 24 | * `project_name` 25 | * `logs_dir` - the path to store the logs 26 | * `data_dir` - the path to store the collected data 27 | * `cameras_config` - the list of cameras 28 | * `sensors_config` - the list of sensors 29 | * `pipelines` - the list of active pipelines 30 | * `log_level` - the level of logs to save. Read more at `Configuring logging` 31 | * `board` - the hardware scheme 32 | 2. `cameras.json` includes the list of all cameras. Here for every camera we have to specify: 33 | * `id` - camera custom unique name. Should reflect the position of camera. 34 | * `type` - the type of the camera. Now only RBG digital cameras supported. Future possible options: 3D, multispectral, thermal. 35 | * `width` and `height` - the desired image size. 36 | * `focus_skip` - the number of images to skip. Required for some cameras to autofocus. 37 | 3. `sensors.json` includes the list of all sensors. Here for every sensor: 38 | * the key of the dictionary is the unique sensor name. 39 | * `type` - the type of the sensor. Choose from implemented in `monitoring_system/drivers/sensors/sensor_factory.py`. 40 | The rest of the parameters depend on the selected sensor type. Usually, you want to specify the `pin`. Make sure to select right pin according to the chosen board. 41 | 4. `boards/` folder contains the options of board schemes. Here we specify: 42 | * board name 43 | * optional description 44 | * pins naming (bcm, wpi, loc) 45 | * supported functions for every pin and their correspondence in different naming schemes. 46 | 5. `pipelines/` folder contains all the pipelines. Here you only describe them. To make them actually work, include them to the `configs/main.jsom`. 47 | Pipeline must include: 48 | * `name` 49 | * `run_interval` - the scheduling rule in **cron** notation. Read more in `Configuring pipelines`. 50 | * `pipeline` - the list of tasks. Select tasks from `monitoring_system/scheduler/PipelineExecutor.py`. 51 | Must include `task_type`. The rest of the parameters depend on the chosen task. 52 | 5. `monitoring_system` folder includes all the utilities to launch the monitoring. 53 | 1. `drivers` includes low-lever operations with hardware: 54 | 1. `camera.py` contains `RGBCameraDriver` that makes and saves images with regular RGB digital cameras. 55 | 2. `sensors/` folder includes: 56 | * `Board.py` - utilities for board and storage for sensors states. 57 | * `Sensor.py` - abstract sensor class. Inherit all the other sensors from it. 58 | * `sensor_factory.py` - used to create sensors instances. 59 | * sensors implementations 60 | 2. `logger/` folder contains the utilities to log all the system actions. The logger is included to the `self` of the most hardware-related objects. 61 | 3. `scheduler/` folder includes: 62 | * `Scheduler.py` that inits all of the pipelines. 63 | * `PipelineExecutor.py` that manages the tasks in hte pipelines. If you want to add new tasks, add them here. 64 | 4. `utils/` folder contains common for other modules utils. 65 | 6. `streamlit_server/` folder contains files that visualise the collected data on the simple web interface. You only need to use Python for it, no JS required. 66 | 1. `server.py` is the main interface page. See Streamlit documentation to modify it. 67 | 2. `start_server.sh` is used to launch the visualisation. 68 | 3. If you want the interface to autorefresh the plots, you can also launch `start_autorefresh.sh`. But note that it is an experimental feature! Streamlit doesn't support it! 69 | 70 | Read more about launching at `Launching`. 71 | 72 | 73 | ## Configuring 74 | You can setup general configs in `configs/main.json`. 75 | 76 | ### Configuring pipelines 77 | To create new pipeline, add `.json` file to `/configs/pipelines` folder. 78 | To activate pipeline, add path to pipeline file to the list *pipelines* in `configs/main.json`. 79 | 80 | To setup pipeline running interval, use *run_interval* attribute in `.json` 81 | according to [APScheduler cron documentation](https://apscheduler.readthedocs.io/en/latest/modules/triggers/cron.html). 82 | For example, to run pipeline every 30 minutes, set *run_interval.minute* to `*/30`. 83 | 84 | To add jobs to a pipeline, add objects with it's description to *pipeline* in `.json`. 85 | It must have *task_type* attribute that corresponds to an implemented function in *_get_tasks_executors* method in `monitoring_system/scheduler/PipelineExecutor.py`. 86 | 87 | If you want to add new sensor, implement it in `monitoring_system/drivers/sensors`. It must be inherited from Sensor. 88 | 89 | If you want to add new type of task, add it to `monitoring_system/scheduler/PipelineExecutor.py` *_get_tasks_executors* method. 90 | If your task has to collect new data, you can add `@sensor` decorator, and optionally `@average` decorator. 91 | The parameters from a task in a pipeline are passed to a function as **kwargs, except the `task_type` that is omitted. 92 | 93 | 94 | ### Configuring logging 95 | To make logs, we use python package *logging*. 96 | This package has different levels of logs. 97 | To setup the minimum level of logs that you want to log, set *log_level* attribute in `configs/main.json`. 98 | 99 | 10 for 'DEBUG' 100 | 20 for 'INFO' 101 | 30 for 'WARNING' 102 | 40 for 'ERROR' 103 | 50 for 'CRITICAL' 104 | 105 | The logs stored in `logs/` folder. There you can find: logs for the main scheduler, and separate logs for each pipeline. 106 | To avoid creating huge log files, all the log files handlers are changed every day. 107 | 108 | 109 | ### Configuring cameras 110 | For each camera: 111 | 1. Unplug USB a camera 112 | 2. `ls -ltrh /dev/video*` 113 | 3. Plug a USB camera 114 | 4. `ls -ltrh /dev/video*` 115 | 5. Note new device name. For instance, /dev/video0 116 | 6. `sudo udevadm info --query=all --name=/dev/video0` (don't forget to change device number) 117 | 7. check second part of DEVLINKS string. It should look like `/dev/v4l/by-path/platform-3f980000.usb-usb-0:1.4:1.0-video-index0` 118 | 8. Write you device name to `/configs/cameras.json` 119 | 120 | 121 | ### Configuring sensors 122 | 1. Add your sensors to `configs/sensors.json`. 123 | `"naming"` is the pin naming scheme. Can be: 124 | * `"loc"` - physical location 125 | * `"bcm"` - BCM 126 | * `"wpi"` - wPi (GPIO) 127 | To see pins state and correspondence: `gpio readall`. 128 | To check what pins are in use in your project see `board.txt`. 129 | 130 | 2. Add driver to your sensor as a `monitoring_system/drivers/sensors/YouySensor.py` inherits from `Sensor.py`. 131 | Don't forget to implement `Sensor` abstract methods. 132 | 133 | 3. Add your sensor to `monitoring_system/drivers/sensors/sensor_factory.py`. And don't forget all the imports. 134 | 135 | 4. Use your sensor in pipeline. 136 | 137 | The most stable Raspberry Pi pins (physical locations) are: 11, 12, 13, 15, 16, 18 and 22. Try using them in your project if you don't use external boards. 138 | 139 | 140 | ## Launching 141 | To start monitoring system on reboot, type in CLI: 142 | `crontab -e` 143 | and add to the end of file: 144 | `@reboot Projects/Monitoring/start.sh` 145 | `@reboot Projects/Monitoring/streamlit_server/start_server.sh` 146 | `@reboot Projects/Monitoring/streamlit_server/start_autorefresh.sh` 147 | And save the file: 148 | `ctlr+o ctrl+x` in nano editor. 149 | 150 | Make sure to change `Projects/Monitoring` to your project location. 151 | 152 | 153 | ## Data 154 | The collected data is stored in `data/` folder. 155 | In `images/` subfolder you can find: 156 | 1. `images.csv` file with information about the collected images from all the cameras. 157 | 2. folders for each cameras with images. 158 | 159 | In `sensors/` subfolder you can find `sensors_measurements.csv` file with all the collected sensors measurements. 160 | The columns in the file are sorted by name alphabetically including separate columns for datetime values. 161 | 162 | Note that we record not the exact time of every measurement, but the time when its pipeline started to execute. So, every measurement in a single pipeline run has the same time, which simplifies further data processing. 163 | It will also match am image timestamp if sensing and imaging tasks are in the same pipeline. 164 | 165 | 166 | 167 | ## Restart monitoring 168 | To delete all the collected data and logs, run `Projects/Monitoring/delete.sh` 169 | Caution! Use it only if you understand what you are doing! 170 | 171 | 172 | ## Useful Utilities 173 | ### Making Raspbian OS copy 174 | 1. `ls -ltrh /dev/sd*` 175 | 2. Plug the SD card 176 | 3. `ls -ltrh /dev/sd*` 177 | 4. Note new device name. For instance, /dev/sdc 178 | 5. `sudo dd if=/dev/sdc | gzip > image.gz` 179 | 180 | 181 | To restore OS: 182 | 1. `sudo gzip -dc image.gz | sudo dd of=/dev/sdc` 183 | 184 | 185 | ### To monitor pins use: 186 | `watch -n 0.1 gpio readall` 187 | 188 | 189 | ### Unique device serial number 190 | can be seen in file `serial_number.txt`. 191 | 192 | 193 | ### Remote access 194 | to Raspberry Pi recommended via VNC. 195 | 196 | 197 | ### Remote code edit and execution 198 | is possible via Pycharm remote interpreter. 199 | To configure: 200 | * `ctrl+alt+S` 201 | * Project 202 | * Python Interpreter 203 | * Add -> ssh interpreter 204 | 205 | To see Raspberry address in local network for ssh interpreter: 206 | `ifconfig` 207 | 208 | -------------------------------------------------------------------------------- /alerts/__init__.py: -------------------------------------------------------------------------------- 1 | from alerts.telegram_bot import * 2 | -------------------------------------------------------------------------------- /alerts/telegram_bot/TelegramBot.py: -------------------------------------------------------------------------------- 1 | import sys 2 | print(sys.executable) 3 | 4 | import telegram 5 | 6 | 7 | class TelegramBot: 8 | def __init__(self, token): 9 | self.bot = telegram.Bot(token=token) 10 | 11 | def send_message(self, chat_id, text): 12 | self.bot.send_message(chat_id=chat_id, text=str(text)) 13 | -------------------------------------------------------------------------------- /alerts/telegram_bot/__init__.py: -------------------------------------------------------------------------------- 1 | from alerts.telegram_bot.TelegramBot import TelegramBot 2 | -------------------------------------------------------------------------------- /configs/boards/rpi_board.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rpi", 3 | "description": "Generic Raspberry Pi 3B/B+", 4 | "naming": "bcm", 5 | "pins": { 6 | "2": { 7 | "loc": 3, 8 | "type": ["digital", "sda1", "i2c"] 9 | }, 10 | "3": { 11 | "loc": 5, 12 | "type": ["digital", "scl1", "i2c"] 13 | }, 14 | "4": { 15 | "loc": 7, 16 | "type": ["digital"] 17 | }, 18 | "14": { 19 | "loc": 8, 20 | "type": ["digital", "uart0_txd"] 21 | }, 22 | "15": { 23 | "loc": 10, 24 | "type": ["digital", "uart0_rxd"] 25 | }, 26 | "17": { 27 | "loc": 11, 28 | "type": ["digital"] 29 | }, 30 | "18": { 31 | "loc": 12, 32 | "type": ["digital", "pcm_clk"] 33 | }, 34 | "27": { 35 | "loc": 13, 36 | "type": ["digital"] 37 | }, 38 | "22": { 39 | "loc": 15, 40 | "type": ["digital"] 41 | }, 42 | "23": { 43 | "loc": 16, 44 | "type": ["digital"] 45 | }, 46 | "24": { 47 | "loc": 18, 48 | "type": ["digital"] 49 | }, 50 | "10": { 51 | "loc": 19, 52 | "type": ["digital", "spi0_mosi"] 53 | }, 54 | "9": { 55 | "loc": 21, 56 | "type": ["digital", "spi0_miso"] 57 | }, 58 | "25": { 59 | "loc": 22, 60 | "type": ["digital"] 61 | }, 62 | "11": { 63 | "loc": 23, 64 | "type": ["digital", "spi0_sclk"] 65 | }, 66 | "8": { 67 | "loc": 24, 68 | "type": ["digital", "spi0_ce0_n"] 69 | }, 70 | "7": { 71 | "loc": 26, 72 | "type": ["digital", "spi0_ce1_n"] 73 | }, 74 | "id_sd": { 75 | "loc": 27, 76 | "type": ["i2c", "id", "eeprom"] 77 | }, 78 | "id_sc": { 79 | "loc": 28, 80 | "type": ["i2c", "id", "eeprom"] 81 | }, 82 | "5": { 83 | "loc": 29, 84 | "type": ["digital"] 85 | }, 86 | "6": { 87 | "loc": 31, 88 | "type": ["digital"] 89 | }, 90 | "12": { 91 | "loc": 32, 92 | "type": ["digital"] 93 | }, 94 | "13": { 95 | "loc": 33, 96 | "type": ["digital"] 97 | }, 98 | "19": { 99 | "loc": 35, 100 | "type": ["digital"] 101 | }, 102 | "16": { 103 | "loc": 36, 104 | "type": ["digital"] 105 | }, 106 | "26": { 107 | "loc": 37, 108 | "type": ["digital"] 109 | }, 110 | "20": { 111 | "loc": 38, 112 | "type": ["digital"] 113 | }, 114 | "21": { 115 | "loc": 40, 116 | "type": ["digital"] 117 | } 118 | }, 119 | "pow_loc": [1, 2, 4, 17], 120 | "gnd_loc": [6, 9, 14, 20, 25, 30, 34, 39] 121 | } -------------------------------------------------------------------------------- /configs/cameras.json: -------------------------------------------------------------------------------- 1 | { 2 | "web_cams": [ 3 | { 4 | "id": "top", 5 | "type": "rgb", 6 | "device": "/dev/v4l/by-id/usb-046d_HD_Pro_Webcam_C920_D4EA612F-video-index0", 7 | "width": 1920, 8 | "height": 1080, 9 | "focus_skip": 5 10 | }, 11 | { 12 | "id": "green", 13 | "type": "rgb", 14 | "device": "/dev/v4l/by-id/usb-046d_HD_Pro_Webcam_C920_7C8B612F-video-index0", 15 | "width": 1920, 16 | "height": 1080, 17 | "focus_skip": 1 18 | }, 19 | { 20 | "id": "third", 21 | "type": "rgb", 22 | "device": "/dev/v4l/by-id/usb-046d_HD_Pro_Webcam_C920_A4C212BF-video-index0", 23 | "width": 1920, 24 | "height": 1080, 25 | "focus_skip": 1 26 | } 27 | ], 28 | "canon_cams": [ 29 | { 30 | "id": "top", 31 | "type": "Canon", 32 | "device": "5c65819053084eb8bdc20ec0f058e783", 33 | "focus_skip": 0 34 | } 35 | ] 36 | } -------------------------------------------------------------------------------- /configs/imaging_states.json: -------------------------------------------------------------------------------- 1 | { 2 | "all_on": { 3 | "alarm_w": { 4 | "mode": "permanent", 5 | "type": "bool", 6 | "value": true 7 | }, 8 | "phyto": { 9 | "mode": "interrupt", 10 | "type": "bool", 11 | "value": false 12 | } 13 | }, 14 | "all_off": { 15 | "alarm_w": { 16 | "mode": "permanent", 17 | "type": "bool", 18 | "value": false 19 | } 20 | }, 21 | "phyto_recovery": { 22 | "phyto": { 23 | "mode": "recovery", 24 | "type": "bool", 25 | "value": "" 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /configs/main.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_name": "Peppers monitoring", 3 | "logs_dir": "./logs", 4 | "data_dir": "./data", 5 | "cameras_config": "./configs/cameras.json", 6 | "sensors_config": "./configs/pml_sensors.json", 7 | "pipelines": [ 8 | "./configs/pipelines/imaging.json", 9 | "./configs/pipelines/phyto.json", 10 | "./configs/pipelines/stayalive.json" 11 | ], 12 | "log_level": 20, 13 | "board": "./configs/boards/rpi_board.json" 14 | } -------------------------------------------------------------------------------- /configs/phyto_states.json: -------------------------------------------------------------------------------- 1 | { 2 | "phyto_on": { 3 | "phyto": { 4 | "mode": "permanent", 5 | "type": "bool", 6 | "value": true 7 | } 8 | }, 9 | "phyto_off": { 10 | "phyto": { 11 | "mode": "permanent", 12 | "type": "bool", 13 | "value": false 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /configs/pipelines/imaging.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "imaging", 3 | "run_interval": { 4 | "trigger": "cron", 5 | "second": null, 6 | "minute": "*/1", 7 | "hour": null, 8 | "day_of_week": null 9 | }, 10 | "pipeline": [ 11 | { 12 | "task_type": "actuator", 13 | "sensor_name": "alarm_y", 14 | "cmd": "blink", 15 | "params": { 16 | "repeats": 4, 17 | "t": 0.5 18 | } 19 | }, 20 | { 21 | "task_type": "actuator", 22 | "sensor_name": "alarm_y", 23 | "cmd": "on", 24 | "params": {} 25 | }, 26 | { 27 | "task_type": "switch_state", 28 | "states_list_path": "./configs/imaging_states.json", 29 | "state_name": "all_on", 30 | "is_current_imaging_state": true 31 | }, 32 | { 33 | "task_type": "sleep", 34 | "interval_seconds": 4 35 | }, 36 | { 37 | "task_type": "get_web_images" 38 | }, 39 | { 40 | "task_type": "get_canon_images" 41 | }, 42 | { 43 | "task_type": "switch_state", 44 | "states_list_path": "./configs/imaging_states.json", 45 | "state_name": "all_off", 46 | "is_current_imaging_state": true 47 | }, 48 | { 49 | "task_type": "get_web_images" 50 | }, 51 | { 52 | "task_type": "get_canon_images" 53 | }, 54 | { 55 | "task_type": "actuator", 56 | "sensor_name": "alarm_y", 57 | "cmd": "off", 58 | "params": {} 59 | }, 60 | { 61 | "task_type": "switch_state", 62 | "states_list_path": "./configs/imaging_states.json", 63 | "state_name": "phyto_recovery", 64 | "is_current_imaging_state": true 65 | } 66 | ] 67 | } -------------------------------------------------------------------------------- /configs/pipelines/phyto.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "phyto", 3 | "run_interval": { 4 | "trigger": "cron", 5 | "second": null, 6 | "minute": "30", 7 | "hour": "11", 8 | "day_of_week": null 9 | }, 10 | "pipeline": [ 11 | { 12 | "task_type": "switch_state", 13 | "states_list_path": "./configs/phyto_states.json", 14 | "state_name": "phyto_on", 15 | "is_current_imaging_state": false 16 | }, 17 | { 18 | "task_type": "sleep", 19 | "interval_seconds": 14400 20 | }, 21 | { 22 | "task_type": "switch_state", 23 | "states_list_path": "./configs/phyto_states.json", 24 | "state_name": "phyto_off", 25 | "is_current_imaging_state": false 26 | } 27 | ] 28 | } -------------------------------------------------------------------------------- /configs/pipelines/sensing.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sensing", 3 | "run_interval": { 4 | "trigger": "cron", 5 | "second": null, 6 | "minute": "*/1", 7 | "hour": null, 8 | "day_of_week": null 9 | }, 10 | "pipeline": [ 11 | { 12 | "task_type": "actuator", 13 | "sensor_name": "alarm_y", 14 | "cmd": "on", 15 | "params": {} 16 | }, 17 | { 18 | "task_type": "sleep", 19 | "interval_seconds": 2 20 | }, 21 | { 22 | "task_type": "sensor", 23 | "sensor_name": "soil_humidity", 24 | "cmd": "get_measurements", 25 | "repeats": 25, 26 | "repeat_interval": 0.1, 27 | "params": {} 28 | }, 29 | { 30 | "task_type": "actuator", 31 | "sensor_name": "alarm_y", 32 | "cmd": "off", 33 | "params": {} 34 | } 35 | ] 36 | } -------------------------------------------------------------------------------- /configs/pipelines/stayalive.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stayalive", 3 | "run_interval": { 4 | "trigger": "cron", 5 | "second": "*/2", 6 | "minute": null, 7 | "hour": null, 8 | "day_of_week": null 9 | }, 10 | "pipeline": [ 11 | { 12 | "task_type": "actuator", 13 | "sensor_name": "alarm_g", 14 | "cmd": "on", 15 | "params": {} 16 | }, 17 | { 18 | "task_type": "sleep", 19 | "interval_seconds": 1 20 | }, 21 | { 22 | "task_type": "actuator", 23 | "sensor_name": "alarm_g", 24 | "cmd": "off", 25 | "params": {} 26 | } 27 | ] 28 | } -------------------------------------------------------------------------------- /configs/pml_sensors.json: -------------------------------------------------------------------------------- 1 | { 2 | "naming": "wpi", 3 | "sensors": { 4 | "alarm_b": { 5 | "type": "switch", 6 | "pin": 6, 7 | "init": "off" 8 | }, 9 | "phyto": { 10 | "type": "switch", 11 | "pin": 6, 12 | "init": "off" 13 | }, 14 | "light_l": { 15 | "type": "switch", 16 | "pin": 2, 17 | "init": "off" 18 | }, 19 | "light_r": { 20 | "type": "switch", 21 | "pin": 5, 22 | "init": "off" 23 | }, 24 | "alarm_y": { 25 | "type": "switch", 26 | "pin": 4, 27 | "init": "off" 28 | } 29 | }, 30 | "troyka_cap_ext_sensors": { 31 | "alarm_w": { 32 | "type": "switch", 33 | "pin": 4, 34 | "init": "off" 35 | }, 36 | "alarm_g": { 37 | "type": "switch", 38 | "pin": 5, 39 | "init": "off" 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /delete.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd $DIR 5 | 6 | echo "Deleting all the collected data" 7 | 8 | sudo rm -rf logs/ data/ 9 | sudo rm board.txt 10 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3.7 2 | 3 | from monitoring_system.utils import read_json, get_serial_number, write_txt 4 | from monitoring_system.scheduler.Scheduler import Scheduler 5 | 6 | 7 | if __name__ == '__main__': 8 | main_config = read_json('./configs/main.json') 9 | scheduler = Scheduler(main_config=main_config) 10 | write_txt('serial_number.txt', get_serial_number()) 11 | scheduler.start() 12 | -------------------------------------------------------------------------------- /monitoring_system/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NesterukSergey/Raspberry_Pi_monitoring_system/736c077576ac49775ffd59d59614d9ef97e33f1d/monitoring_system/__init__.py -------------------------------------------------------------------------------- /monitoring_system/drivers/__init__.py: -------------------------------------------------------------------------------- 1 | from monitoring_system.drivers.cameras import * 2 | from monitoring_system.drivers.sensors import * 3 | -------------------------------------------------------------------------------- /monitoring_system/drivers/cameras/CameraDriver.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | from abc import ABC, abstractmethod 3 | from pathlib import Path 4 | from alerts import TelegramBot 5 | from monitoring_system.utils import read_json 6 | 7 | from monitoring_system.utils.csv import write_csv 8 | 9 | 10 | class CameraDriver(ABC): 11 | def __init__(self, system_state='normal', **kwargs): 12 | super().__init__() 13 | self.alert_bot = TelegramBot(read_json('./configs/api/telegram.json')['token']) 14 | self.__dict__.update(kwargs) 15 | self.cam = None 16 | self.captured_image = None 17 | self.system_state = system_state 18 | self._setup() 19 | 20 | @abstractmethod 21 | def _setup(self): 22 | pass 23 | 24 | @abstractmethod 25 | def capture(self): 26 | pass 27 | 28 | def _get_save_path(self): 29 | camera_type = str(self.camera_info['type']) + '_' + str(self.camera_info['id']) 30 | img_path = str(Path(self.folder).joinpath('images', camera_type, self.system_state)) 31 | file_name = '{}_{}_{}.jpg'.format(self.datetime_prefix, camera_type, self.system_state) 32 | save_path = str(Path(img_path).joinpath(file_name)) 33 | return img_path, save_path 34 | 35 | def _save_image(self): 36 | if self.captured_image is None: 37 | error_message = 'Unable to take photo from camera ' + str( 38 | self.camera_info['id']) + '; ' + str(self.camera_info['device']) 39 | self.log.error(error_message) 40 | self._send_alert(error_message) 41 | # raise RuntimeError(error_message) 42 | else: 43 | img_path, save_path = self._get_save_path() 44 | Path(img_path).mkdir(parents=True, exist_ok=True) 45 | cv2.imwrite(save_path, self.captured_image) 46 | self.log.info('Image captured with camera: ' + str(self.camera_info['id'])) 47 | 48 | def _save_image_data(self): 49 | _, save_path = self._get_save_path() 50 | image_info = self.datetime_dict.copy() 51 | image_info['img_path'] = save_path 52 | image_info['device_type'] = self.camera_info['type'] 53 | image_info['device'] = self.camera_info['device'] 54 | image_info['device_id'] = self.camera_info['id'] 55 | image_info['system_state'] = self.system_state 56 | write_csv(image_info, str(Path(self.folder).joinpath('images', 'images.csv'))) 57 | 58 | def _save_all(self): 59 | self._save_image() 60 | self._save_image_data() 61 | 62 | def _send_alert(self, message): 63 | self.alert_bot.send_message(read_json('./configs/api/telegram.json')['alert_chat_id'], message) 64 | -------------------------------------------------------------------------------- /monitoring_system/drivers/cameras/CanonCameraDriver.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sh 3 | import os 4 | from pathlib import Path 5 | 6 | from monitoring_system.drivers.cameras.CameraDriver import CameraDriver 7 | 8 | 9 | class CanonCameraDriver(CameraDriver): 10 | def __init__(self, **kwargs): 11 | super().__init__(**kwargs) 12 | 13 | def _setup(self): 14 | 15 | def get_serial_id(data): 16 | for s in data.split('\n'): 17 | if 'serial number' in s.lower(): 18 | s_num = s.split()[-1] 19 | return s_num 20 | 21 | try: 22 | info = sh.gphoto2('--summary') 23 | serial_id = get_serial_id(info) 24 | 25 | if str(serial_id) != str(self.camera_info['device']): 26 | self.log.warning( 27 | 'Wrong SRL camera connected. Expected: ' + str(self.camera_info['device']) + '. Got: ' + str(serial_id)) 28 | except Exception as e: 29 | self._send_alert('Can not init Canon camera. ' + str(e)) 30 | 31 | def capture(self): 32 | for i in range(self.camera_info['focus_skip'] + 1): 33 | time.sleep(0.1) 34 | 35 | try: 36 | s = sh.gphoto2('--capture-image-and-download', '--force-overwrite') 37 | if s.exit_code != 0: 38 | self.captured_image = None 39 | self._send_alert('Can not capture image with SLR camera: ' + str(self.camera_info['device'])) 40 | except Exception as e: 41 | self._send_alert(str(e)) 42 | return 43 | 44 | img_path, save_path = self._get_save_path() 45 | 46 | Path(img_path).mkdir(parents=True, exist_ok=True) 47 | sh.mv( 48 | '/home/pi/Projects/Monitoring/capt0000.jpg', 49 | save_path 50 | ) 51 | self.log.info('Image captured with camera: ' + str(self.camera_info['id'])) 52 | 53 | self._save_image_data() 54 | -------------------------------------------------------------------------------- /monitoring_system/drivers/cameras/WebCameraDriver.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import time 3 | import numpy as np 4 | 5 | from monitoring_system.drivers.cameras.CameraDriver import CameraDriver 6 | 7 | 8 | class WebCameraDriver(CameraDriver): 9 | def __init__(self, **kwargs): 10 | super().__init__(**kwargs) 11 | 12 | def _setup(self): 13 | self.cam = cv2.VideoCapture(self.camera_info['device']) 14 | 15 | if not self.cam.isOpened(): 16 | error_message = 'Unable to open WebCamera: ' + str(self.camera_info['id']) + '; ' + str(self.camera_info['device']) 17 | self.log.error(error_message) 18 | self._send_alert(error_message) 19 | # raise RuntimeError(error_message) 20 | return 21 | 22 | self.cam.set(cv2.CAP_PROP_FRAME_WIDTH, self.camera_info['width']) 23 | self.cam.set(cv2.CAP_PROP_FRAME_HEIGHT, self.camera_info['height']) 24 | self.cam.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')) 25 | 26 | # Take some time for camera initialization 27 | _, _ = self.cam.read() 28 | time.sleep(0.1) 29 | 30 | def capture(self): 31 | for i in range(self.camera_info['focus_skip'] + 1): 32 | ret, image = self.cam.read() 33 | 34 | if self.camera_info['average'] > 0: 35 | images = [] 36 | for i in range(self.camera_info['average']): 37 | image = self.capture_single() 38 | 39 | if image is not None: 40 | images.append(image) 41 | 42 | if len(images) < 1: 43 | self._send_alert('Not enough images to average') 44 | self.captured_image = image 45 | else: 46 | images = np.array(images) 47 | num_images = images.shape[0] 48 | res = (images.sum(axis=0) / num_images).astype('uint8') 49 | self.captured_image = res 50 | 51 | 52 | else: 53 | self.captured_image = image 54 | 55 | self.cam.release() 56 | cv2.destroyAllWindows() 57 | del self.cam 58 | 59 | self._save_all() 60 | 61 | def capture_single(self): 62 | 63 | time.sleep(0.1) 64 | 65 | try: 66 | ret, captured_image = self.cam.read() 67 | if not ret: 68 | captured_image = None 69 | 70 | return captured_image 71 | 72 | except Exception as e: 73 | self._send_alert(e) 74 | return None 75 | -------------------------------------------------------------------------------- /monitoring_system/drivers/cameras/__init__.py: -------------------------------------------------------------------------------- 1 | from monitoring_system.drivers.cameras.WebCameraDriver import WebCameraDriver 2 | from monitoring_system.drivers.cameras.CanonCameraDriver import CanonCameraDriver 3 | -------------------------------------------------------------------------------- /monitoring_system/drivers/sensors/AnalogReadSensor.py: -------------------------------------------------------------------------------- 1 | from monitoring_system.drivers.sensors.Sensor import Sensor 2 | 3 | 4 | class AnalogReadSensor(Sensor): 5 | def __init__(self, **kwargs): 6 | super().__init__(**kwargs) 7 | self.pin = self._get_wpi_pin(self.pin) 8 | self._register_pins([self.pin], []) 9 | 10 | def get_measurements(self): 11 | return { 12 | 'analog': self._analog_read(self.pin) 13 | } 14 | -------------------------------------------------------------------------------- /monitoring_system/drivers/sensors/Board.py: -------------------------------------------------------------------------------- 1 | from monitoring_system.utils import write_txt 2 | from monitoring_system.drivers.sensors.sensor_factory import sensor_factory 3 | 4 | 5 | class Board: 6 | def __init__(self, board_scheme, sensors, log): 7 | self.log = log 8 | self.board_scheme = board_scheme 9 | self.sensor_naming = sensors['naming'] 10 | self.sensors = sensors['sensors'] 11 | self.pins = self._name2loc() 12 | self.active_sensors = sensor_factory(self.sensor_naming, sensors, self.log) 13 | write_txt('board.txt', self.print_board()) 14 | 15 | def _name2loc(self): 16 | pins = {} 17 | for sensor in self.sensors: 18 | if 'pin' in list(self.sensors[sensor].keys()): 19 | loc = str(self.sensors[sensor]['pin']) 20 | if loc in list(pins.keys()): 21 | self.log.error('pin usage duplicates at pin: ' + loc) 22 | 23 | pins[loc] = { 24 | 'name': sensor, 25 | 'aux': '' 26 | } 27 | elif 'pins' in list(self.sensors[sensor].keys()): 28 | for pin in self.sensors[sensor]['pins']: 29 | loc = str(self.sensors[sensor]['pins'][pin]) 30 | if loc in list(pins.keys()): 31 | self.log.error('pin usage duplicates at pin: ' + loc) 32 | 33 | pins[loc] = { 34 | 'name': sensor, 35 | 'aux': '(' + str(pin) + ')' 36 | } 37 | else: 38 | error_message = 'Sensor must specify pin or pins: ' + str(sensor) 39 | self.log.error(error_message) 40 | raise NotImplemented(error_message) 41 | 42 | return pins 43 | 44 | def print_board(self): 45 | if self.board_scheme['name'] == 'rpi': 46 | half_len = 25 47 | full_len = (half_len + 4) * 2 48 | pad = '-' * full_len + '\n' 49 | fig = '' 50 | 51 | board_pins = list(self.pins) 52 | for i in range(1, 41, 2): 53 | left_pin = str(i) 54 | if left_pin in board_pins: 55 | left_name = self.pins[left_pin]['name'] 56 | left_aux = self.pins[left_pin]['aux'] 57 | left = '|| {} {} '.format(left_name, left_aux).ljust(half_len, '-') + '|' + '{}|'.format( 58 | left_pin).rjust(3) 59 | elif int(left_pin) in self.board_scheme['pow_loc']: 60 | left = '|| ({}) '.format('pow').ljust(half_len, ' ') + '|' + '{}|'.format( 61 | left_pin).rjust(3) 62 | elif int(left_pin) in self.board_scheme['gnd_loc']: 63 | left = '|| ({}) '.format('gnd').ljust(half_len, ' ') + '|' + '{}|'.format( 64 | left_pin).rjust(3) 65 | else: 66 | left = '||'.ljust(half_len, ' ') + '|' + '{}|'.format(left_pin).rjust(3) 67 | 68 | right_pin = str(i + 1) 69 | if right_pin in board_pins: 70 | right_name = self.pins[right_pin]['name'] 71 | right_aux = self.pins[right_pin]['aux'] 72 | right = '|' + '{}'.format(right_pin).ljust(2) + '|' + ' {} {} ||'.format(right_name, 73 | right_aux).rjust( 74 | half_len, '-') 75 | elif int(right_pin) in self.board_scheme['pow_loc']: 76 | right = '|' + '{}'.format(right_pin).ljust(2) + '|' + ' ({}) ||'.format('pow').rjust( 77 | half_len, ' ') 78 | elif int(right_pin) in self.board_scheme['gnd_loc']: 79 | right = '|' + '{}'.format(right_pin).ljust(2) + '|' + ' ({}) ||'.format('gnd').rjust( 80 | half_len, ' ') 81 | else: 82 | right = '|' + '{}'.format(right_pin).ljust(2) + '|' + '||'.rjust(half_len, ' ') 83 | 84 | s = left + right + '\n' 85 | fig += s 86 | 87 | fig = pad + '||' + ('{:^' + str(full_len - 4) + '}').format( 88 | self.board_scheme['description']) + '||\n' + pad + pad + fig + pad 89 | return fig 90 | else: 91 | try: 92 | active_pins = sorted(map(int, list(self.pins.keys()))) 93 | active_pins = map(str, active_pins) 94 | except: 95 | active_pins = sorted(list(self.pins.keys())) 96 | 97 | fig = self.board_scheme['description'] 98 | max_len = 0 99 | for pin in active_pins: 100 | s = ' pin: {} -- {} {} \n'.format(pin, self.pins[pin]['name'], self.pins[pin]['aux']) 101 | fig += s 102 | max_len = max(max_len, len(s)) 103 | 104 | pad = ('-' * max_len) + '\n' 105 | fig = pad + fig + pad 106 | return fig 107 | 108 | def exit(self): 109 | for sensor in self.active_sensors: 110 | self.active_sensors[sensor].exit() 111 | -------------------------------------------------------------------------------- /monitoring_system/drivers/sensors/Dht11Sensor.py: -------------------------------------------------------------------------------- 1 | import time 2 | import RPi.GPIO 3 | from monitoring_system.drivers.sensors.Sensor import Sensor 4 | 5 | RPi.GPIO.setwarnings(False) 6 | RPi.GPIO.setmode(RPi.GPIO.BCM) 7 | RPi.GPIO.cleanup() 8 | 9 | 10 | class Dht11Sensor(Sensor): 11 | def __init__(self, **kwargs): 12 | super().__init__(**kwargs) 13 | self.detector = DHT11(self.pin) 14 | 15 | # def exit(self): 16 | # pass 17 | 18 | def get_measurements(self): 19 | result = self.detector.read() 20 | 21 | if result.is_valid(): 22 | measurement = { 23 | 'temp': result.temperature, 24 | 'hum': result.humidity 25 | } 26 | else: 27 | measurement = { 28 | 'temp': None, 29 | 'hum': None 30 | } 31 | 32 | return measurement 33 | 34 | 35 | class DHT11Result: 36 | 'DHT11 sensor result returned by DHT11.read() method' 37 | 38 | ERR_NO_ERROR = 0 39 | ERR_MISSING_DATA = 1 40 | ERR_CRC = 2 41 | 42 | error_code = ERR_NO_ERROR 43 | temperature = -1 44 | humidity = -1 45 | 46 | def __init__(self, error_code, temperature, humidity): 47 | self.error_code = error_code 48 | self.temperature = temperature 49 | self.humidity = humidity 50 | 51 | def is_valid(self): 52 | return self.error_code == DHT11Result.ERR_NO_ERROR 53 | 54 | 55 | class DHT11: 56 | 'DHT11 sensor reader class for Raspberry' 57 | 58 | __pin = 0 59 | 60 | def __init__(self, pin): 61 | self.__pin = pin 62 | 63 | def read(self): 64 | self.__init__(self.__pin) 65 | RPi.GPIO.setup(self.__pin, RPi.GPIO.OUT) 66 | 67 | # send initial high 68 | self.__send_and_sleep(RPi.GPIO.HIGH, 0.05) 69 | 70 | # pull down to low 71 | self.__send_and_sleep(RPi.GPIO.LOW, 0.02) 72 | 73 | # change to input using pull up 74 | RPi.GPIO.setup(self.__pin, RPi.GPIO.IN, RPi.GPIO.PUD_UP) 75 | 76 | # collect data into an array 77 | data = self.__collect_input() 78 | 79 | # parse lengths of all data pull up periods 80 | pull_up_lengths = self.__parse_data_pull_up_lengths(data) 81 | 82 | # if bit count mismatch, return error (4 byte data + 1 byte checksum) 83 | if len(pull_up_lengths) != 40: 84 | return DHT11Result(DHT11Result.ERR_MISSING_DATA, 0, 0) 85 | 86 | # calculate bits from lengths of the pull up periods 87 | bits = self.__calculate_bits(pull_up_lengths) 88 | 89 | # we have the bits, calculate bytes 90 | the_bytes = self.__bits_to_bytes(bits) 91 | 92 | # calculate checksum and check 93 | checksum = self.__calculate_checksum(the_bytes) 94 | if the_bytes[4] != checksum: 95 | return DHT11Result(DHT11Result.ERR_CRC, 0, 0) 96 | 97 | # ok, we have valid data 98 | 99 | # The meaning of the return sensor values 100 | # the_bytes[0]: humidity int 101 | # the_bytes[1]: humidity decimal 102 | # the_bytes[2]: temperature int 103 | # the_bytes[3]: temperature decimal 104 | 105 | temperature = the_bytes[2] + float(the_bytes[3]) / 10 106 | humidity = the_bytes[0] + float(the_bytes[1]) / 10 107 | 108 | return DHT11Result(DHT11Result.ERR_NO_ERROR, temperature, humidity) 109 | 110 | def __send_and_sleep(self, output, sleep): 111 | RPi.GPIO.output(self.__pin, output) 112 | 113 | time.sleep(sleep) 114 | 115 | def __collect_input(self): 116 | # collect the data while unchanged found 117 | unchanged_count = 0 118 | 119 | # this is used to determine where is the end of the data 120 | max_unchanged_count = 100 121 | 122 | last = -1 123 | data = [] 124 | while True: 125 | current = RPi.GPIO.input(self.__pin) 126 | data.append(current) 127 | if last != current: 128 | unchanged_count = 0 129 | last = current 130 | else: 131 | unchanged_count += 1 132 | if unchanged_count > max_unchanged_count: 133 | break 134 | 135 | return data 136 | 137 | def __parse_data_pull_up_lengths(self, data): 138 | STATE_INIT_PULL_DOWN = 1 139 | STATE_INIT_PULL_UP = 2 140 | STATE_DATA_FIRST_PULL_DOWN = 3 141 | STATE_DATA_PULL_UP = 4 142 | STATE_DATA_PULL_DOWN = 5 143 | 144 | state = STATE_INIT_PULL_DOWN 145 | 146 | lengths = [] # will contain the lengths of data pull up periods 147 | current_length = 0 # will contain the length of the previous period 148 | 149 | for i in range(len(data)): 150 | 151 | current = data[i] 152 | current_length += 1 153 | 154 | if state == STATE_INIT_PULL_DOWN: 155 | if current == RPi.GPIO.LOW: 156 | # ok, we got the initial pull down 157 | state = STATE_INIT_PULL_UP 158 | continue 159 | else: 160 | continue 161 | if state == STATE_INIT_PULL_UP: 162 | if current == RPi.GPIO.HIGH: 163 | # ok, we got the initial pull up 164 | state = STATE_DATA_FIRST_PULL_DOWN 165 | continue 166 | else: 167 | continue 168 | if state == STATE_DATA_FIRST_PULL_DOWN: 169 | if current == RPi.GPIO.LOW: 170 | # we have the initial pull down, the next will be the data pull up 171 | state = STATE_DATA_PULL_UP 172 | continue 173 | else: 174 | continue 175 | if state == STATE_DATA_PULL_UP: 176 | if current == RPi.GPIO.HIGH: 177 | # data pulled up, the length of this pull up will determine whether it is 0 or 1 178 | current_length = 0 179 | state = STATE_DATA_PULL_DOWN 180 | continue 181 | else: 182 | continue 183 | if state == STATE_DATA_PULL_DOWN: 184 | if current == RPi.GPIO.LOW: 185 | # pulled down, we store the length of the previous pull up period 186 | lengths.append(current_length) 187 | state = STATE_DATA_PULL_UP 188 | continue 189 | else: 190 | continue 191 | 192 | return lengths 193 | 194 | def __calculate_bits(self, pull_up_lengths): 195 | # find shortest and longest period 196 | shortest_pull_up = 1000 197 | longest_pull_up = 0 198 | 199 | for i in range(0, len(pull_up_lengths)): 200 | length = pull_up_lengths[i] 201 | if length < shortest_pull_up: 202 | shortest_pull_up = length 203 | if length > longest_pull_up: 204 | longest_pull_up = length 205 | 206 | # use the halfway to determine whether the period it is long or short 207 | halfway = shortest_pull_up + (longest_pull_up - shortest_pull_up) / 2 208 | bits = [] 209 | 210 | for i in range(0, len(pull_up_lengths)): 211 | bit = False 212 | if pull_up_lengths[i] > halfway: 213 | bit = True 214 | bits.append(bit) 215 | 216 | return bits 217 | 218 | def __bits_to_bytes(self, bits): 219 | the_bytes = [] 220 | byte = 0 221 | 222 | for i in range(0, len(bits)): 223 | byte = byte << 1 224 | if (bits[i]): 225 | byte = byte | 1 226 | else: 227 | byte = byte | 0 228 | if ((i + 1) % 8 == 0): 229 | the_bytes.append(byte) 230 | byte = 0 231 | 232 | return the_bytes 233 | 234 | def __calculate_checksum(self, the_bytes): 235 | return the_bytes[0] + the_bytes[1] + the_bytes[2] + the_bytes[3] & 255 236 | -------------------------------------------------------------------------------- /monitoring_system/drivers/sensors/DigitalReadSensor.py: -------------------------------------------------------------------------------- 1 | from monitoring_system.drivers.sensors.Sensor import Sensor 2 | 3 | 4 | class DigitalReadSensor(Sensor): 5 | def __init__(self, **kwargs): 6 | super().__init__(**kwargs) 7 | self.pin = self._get_wpi_pin(self.pin) 8 | self._register_pins([self.pin], []) 9 | 10 | def get_measurements(self): 11 | return { 12 | 'digital': self._digital_read(self.pin) 13 | } 14 | -------------------------------------------------------------------------------- /monitoring_system/drivers/sensors/Sensor.py: -------------------------------------------------------------------------------- 1 | import time 2 | from abc import ABC, abstractmethod 3 | import wiringpi as wp 4 | # import gpioexp 5 | from monitoring_system.utils import gpioexp 6 | 7 | 8 | class Sensor(ABC): 9 | def __init__(self, **kwargs): 10 | super().__init__() 11 | self.__dict__.update(kwargs) 12 | self.all_pins = [] 13 | self.exp = gpioexp.gpioexp() 14 | 15 | def exit(self): 16 | for pin in self.all_pins: 17 | self._exit_pin(pin) 18 | 19 | def _get_wpi_pin(self, pin): 20 | if self.pin_naming == 'loc': 21 | return self._loc2bcm_wpi(pin)['wpi'] 22 | elif self.pin_naming == 'wpi': 23 | return pin 24 | elif self.pin_naming == 'bcm': 25 | return self._loc2bcm_wpi(self.board_scheme['pins'][str(pin)]['loc'])['wpi'] 26 | else: 27 | error_message = 'Unrecognized pin naming scheme: ' + str(self.pin_naming) 28 | self.log.error(error_message) 29 | raise NotImplemented(error_message) 30 | 31 | def _register_pins(self, in_pins=[], out_pins=[]): 32 | wp.wiringPiSetup() 33 | 34 | for sensor in in_pins: 35 | wp.pinMode(int(sensor), 0) 36 | self.all_pins.append(int(sensor)) 37 | time.sleep(0.01) 38 | 39 | for sensor in out_pins: 40 | wp.pinMode(int(sensor), 1) 41 | self.all_pins.append(int(sensor)) 42 | time.sleep(0.01) 43 | 44 | def _digital_write(self, pin, val): 45 | self.log.debug('Write ' + str(val) + ' to pin ' + str(pin)) 46 | 47 | if self.board_type == '': 48 | wp.digitalWrite(pin, bool(int(val))) 49 | time.sleep(0.01) 50 | elif self.board_type == 'troyka_cap_ext': 51 | self.exp.analogWrite(pin, bool(int(val))) 52 | time.sleep(0.01) 53 | 54 | def _digital_read(self, pin): 55 | if self.board_type == '': 56 | val = wp.digitalRead(pin) 57 | elif self.board_type == 'troyka_cap_ext': 58 | val = self.exp.analogRead(pin) 59 | 60 | self.log.debug('Read ' + str(val) + ' from pin ' + str(pin)) 61 | return val 62 | 63 | def _supports_analog(self, pin): 64 | if self.board_type == 'troyka_cap_ext': 65 | return True 66 | 67 | if self.board_scheme['name'] == 'rpi': 68 | error_message = "Board rpi doesn't support analog pins. At pin: " + str(pin) 69 | self.log.error(error_message) 70 | return False 71 | 72 | return False 73 | 74 | # TODO: add check by type of pin from board_scheme 75 | 76 | def _analog_write(self, pin, val): 77 | if self.board_type == '': 78 | if self._supports_analog(pin): 79 | wp.analogWrite(pin, val) 80 | else: 81 | self._digital_write(pin, val * 1023) 82 | elif self.board_type == 'troyka_cap_ext': 83 | self.exp.analogWrite(pin, val) 84 | 85 | def _analog_read(self, pin): 86 | if self.board_type == '': 87 | if self._supports_analog(pin): 88 | return wp.analogRead(pin) 89 | else: 90 | return self._digital_read(pin) * 1023 91 | elif self.board_type == 'troyka_cap_ext': 92 | return self.exp.analogRead(pin) 93 | 94 | def _loc2bcm_wpi(self, loc): 95 | loc = str(loc) 96 | d = { 97 | '3': { 98 | 'bcm': 2, 99 | 'wpi': 8 100 | }, 101 | '5': { 102 | 'bcm': 3, 103 | 'wpi': 9 104 | }, 105 | '7': { 106 | 'bcm': 4, 107 | 'wpi': 7 108 | }, 109 | '8': { 110 | 'bcm': 14, 111 | 'wpi': 15 112 | }, 113 | '10': { 114 | 'bcm': 15, 115 | 'wpi': 16 116 | }, 117 | '11': { 118 | 'bcm': 17, 119 | 'wpi': 0 120 | }, 121 | '12': { 122 | 'bcm': 17, 123 | 'wpi': 1 124 | }, 125 | '13': { 126 | 'bcm': 27, 127 | 'wpi': 2 128 | }, 129 | '15': { 130 | 'bcm': 22, 131 | 'wpi': 3 132 | }, 133 | '16': { 134 | 'bcm': 23, 135 | 'wpi': 4 136 | }, 137 | '18': { 138 | 'bcm': 24, 139 | 'wpi': 5 140 | }, 141 | '19': { 142 | 'bcm': 10, 143 | 'wpi': 12 144 | }, 145 | '21': { 146 | 'bcm': 9, 147 | 'wpi': 12 148 | }, 149 | '22': { 150 | 'bcm': 25, 151 | 'wpi': 6 152 | }, 153 | '23': { 154 | 'bcm': 11, 155 | 'wpi': 14 156 | }, 157 | '24': { 158 | 'bcm': 8, 159 | 'wpi': 10 160 | }, 161 | '26': { 162 | 'bcm': 7, 163 | 'wpi': 11 164 | }, 165 | '27': { 166 | 'bcm': 0, 167 | 'wpi': 30 168 | }, 169 | '28': { 170 | 'bcm': 1, 171 | 'wpi': 31 172 | }, 173 | '29': { 174 | 'bcm': 5, 175 | 'wpi': 21 176 | }, 177 | '31': { 178 | 'bcm': 6, 179 | 'wpi': 22 180 | }, 181 | '32': { 182 | 'bcm': 12, 183 | 'wpi': 26 184 | }, 185 | '33': { 186 | 'bcm': 13, 187 | 'wpi': 23 188 | }, 189 | '35': { 190 | 'bcm': 19, 191 | 'wpi': 24 192 | }, 193 | '36': { 194 | 'bcm': 16, 195 | 'wpi': 27 196 | }, 197 | '37': { 198 | 'bcm': 26, 199 | 'wpi': 25 200 | }, 201 | '38': { 202 | 'bcm': 20, 203 | 'wpi': 28 204 | }, 205 | '40': { 206 | 'bcm': 21, 207 | 'wpi': 29 208 | } 209 | } 210 | 211 | if loc not in list(d.keys()): 212 | error_message = 'Wrong physical pin address: ' + loc 213 | self.log.error(error_message) 214 | raise UserWarning(error_message) 215 | 216 | return d[loc] 217 | 218 | def _exit_pin(self, pin): 219 | wp.pinMode(int(pin), 1) 220 | time.sleep(0.05) 221 | self._digital_write(pin, 0) 222 | time.sleep(0.05) 223 | wp.pinMode(int(pin), 0) 224 | self.log.debug('Pin ' + str(self.pin) + ' exited') 225 | -------------------------------------------------------------------------------- /monitoring_system/drivers/sensors/SwitchSensor.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from monitoring_system.drivers.sensors.Sensor import Sensor 4 | 5 | 6 | class SwitchSensor(Sensor): 7 | def __init__(self, **kwargs): 8 | super().__init__(**kwargs) 9 | self.pin = self._get_wpi_pin(self.pin) 10 | self._register_pins([], [self.pin]) 11 | 12 | if kwargs['init'] == 'on': 13 | self.on() 14 | self.state = True 15 | if kwargs['init'] == 'off': 16 | self.off() 17 | self.state = False 18 | 19 | # def exit(self): 20 | # self._exit_pin(self.pin) 21 | 22 | def set_state(self, val): 23 | self.state = bool(val) 24 | self._digital_write(self.pin, int(self.state)) 25 | 26 | def on(self): 27 | self.set_state(1) 28 | 29 | def off(self): 30 | self.set_state(0) 31 | 32 | def blink(self, repeats, t): 33 | assert repeats > 0 34 | old_state = self.state 35 | 36 | for i in range(repeats): 37 | self.on() 38 | time.sleep(t) 39 | self.off() 40 | time.sleep(t) 41 | 42 | self.set_state(old_state) 43 | -------------------------------------------------------------------------------- /monitoring_system/drivers/sensors/__init__.py: -------------------------------------------------------------------------------- 1 | from monitoring_system.drivers.sensors.Board import Board 2 | from monitoring_system.drivers.sensors.Sensor import Sensor 3 | from monitoring_system.drivers.sensors.DigitalReadSensor import DigitalReadSensor 4 | from monitoring_system.drivers.sensors.SwitchSensor import SwitchSensor 5 | from monitoring_system.drivers.sensors.Dht11Sensor import Dht11Sensor 6 | -------------------------------------------------------------------------------- /monitoring_system/drivers/sensors/sensor_factory.py: -------------------------------------------------------------------------------- 1 | from monitoring_system.drivers.sensors.DigitalReadSensor import DigitalReadSensor 2 | from monitoring_system.drivers.sensors.SwitchSensor import SwitchSensor 3 | from monitoring_system.drivers.sensors.Dht11Sensor import Dht11Sensor 4 | from monitoring_system.drivers.sensors.AnalogReadSensor import AnalogReadSensor 5 | 6 | 7 | def sensor_factory(sensor_naming, sensors, log): 8 | d = {} 9 | sensor_types = { 10 | 'switch': SwitchSensor, 11 | 'dht11': Dht11Sensor, 12 | 'digital_read': DigitalReadSensor, 13 | 'analog_read': AnalogReadSensor, 14 | } 15 | 16 | for sensor in sensors['sensors']: 17 | sensor_type = sensors['sensors'][sensor]['type'] 18 | if sensor_type not in list(sensor_types.keys()): 19 | log.error('Unrecognized sensor: ' + sensor_type) 20 | else: 21 | d[sensor] = sensor_types[sensor_type](**sensors['sensors'][sensor], 22 | pin_naming=sensor_naming, log=log, board_type='') 23 | 24 | for sensor in sensors['troyka_cap_ext_sensors']: 25 | sensor_type = sensors['troyka_cap_ext_sensors'][sensor]['type'] 26 | if sensor_type not in list(sensor_types.keys()): 27 | log.error('Unrecognized sensor: ' + sensor_type) 28 | else: 29 | d[sensor] = sensor_types[sensor_type](**sensors['troyka_cap_ext_sensors'][sensor], 30 | pin_naming=sensor_naming, log=log, board_type='troyka_cap_ext') 31 | 32 | return d 33 | -------------------------------------------------------------------------------- /monitoring_system/logger/Logger.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | from logging.handlers import TimedRotatingFileHandler 4 | 5 | 6 | def get_logger(thread_name='main', file='logs/', level=20): 7 | logger = logging.getLogger(thread_name) 8 | logger.setLevel(level) 9 | 10 | file_handler = TimedRotatingFileHandler(os.path.join(file, thread_name + '.log'), when='midnight') 11 | log_format = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 12 | 13 | file_handler.setFormatter(log_format) 14 | logger.addHandler(file_handler) 15 | 16 | logger.propagate = False 17 | return logger 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /monitoring_system/logger/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NesterukSergey/Raspberry_Pi_monitoring_system/736c077576ac49775ffd59d59614d9ef97e33f1d/monitoring_system/logger/__init__.py -------------------------------------------------------------------------------- /monitoring_system/scheduler/PipelineExecutor.py: -------------------------------------------------------------------------------- 1 | import time 2 | from datetime import datetime 3 | from pathlib import Path 4 | 5 | from monitoring_system.utils import * 6 | from monitoring_system.drivers import * 7 | 8 | 9 | def _sensor(func): 10 | def _wrapper(self, *args, **kwargs): 11 | try: 12 | measurements = func(self, *args, **kwargs) 13 | for measurement in measurements: 14 | self.sensors_accumulator[measurement] = [measurements[measurement]] 15 | 16 | except Exception as e: 17 | self.log.warning('sensor ' + kwargs['sensor_name'] + ' disabled') 18 | self.log.error(e) 19 | raise e 20 | 21 | return _wrapper 22 | 23 | 24 | def _average(func): 25 | def _wrapper(self, *args, **kwargs): 26 | DEFAULT_REPEATS = 10 27 | DEFAULT_REPEATINTERVAL = 0.1 28 | repeats = kwargs['repeats'] if kwargs.get('repeats') else DEFAULT_REPEATS 29 | repeat_interval = kwargs['repeat_interval'] if kwargs.get('repeat_interval') else DEFAULT_REPEATINTERVAL 30 | assert repeats > 0 31 | assert repeat_interval > 0 32 | 33 | tmp = {} 34 | for i in range(repeats): 35 | res = func(self, *args, **kwargs) 36 | 37 | if i == 0: 38 | for k in res: 39 | tmp[k] = [res[k]] 40 | else: 41 | for k in res: 42 | tmp[k].append(res[k]) 43 | 44 | time.sleep(repeat_interval) 45 | 46 | result = {} 47 | for k in tmp: 48 | result[k] = average(tmp[k]) 49 | 50 | return result 51 | 52 | return _wrapper 53 | 54 | 55 | class PipelineExecutor: 56 | def __init__(self, logger, pipeline, main_config, pipeline_name, board): 57 | self.log = logger 58 | self.pipeline = pipeline 59 | self.main_config = main_config 60 | self.pipeline_name = pipeline_name 61 | self.board = board 62 | self.tasks_executors = self._get_tasks_executors() 63 | self.sensors_accumulator = None 64 | self.measurements_file = Path(self.main_config['data_dir']).joinpath( 65 | 'sensors/' + pipeline_name + '_measurements.csv') 66 | self.cam_config = read_json(self.main_config['cameras_config']) 67 | self.datetime_prefix, self.datetime_dict = get_time() 68 | self.current_imaging_state = 'unset' 69 | 70 | def execute(self): 71 | pipeline_start = datetime.now() 72 | self.datetime_prefix, self.datetime_dict = get_time() 73 | self.sensors_accumulator = self.datetime_dict.copy() 74 | 75 | for task in self.pipeline: 76 | try: 77 | params = task.copy() 78 | del params['task_type'] 79 | self.tasks_executors[task['task_type']](**params) 80 | except Exception as e: 81 | self.log.error('Error at task ' + task['task_type']) 82 | self.log.error(str(e)) 83 | raise e 84 | 85 | self._save_measurements() 86 | pipeline_execution_time = (datetime.now() - pipeline_start).seconds 87 | self.log.info('Pipeline executed in ' + str(pipeline_execution_time) + ' seconds') 88 | 89 | def _get_tasks_executors(self): 90 | return { 91 | 'hello_world': lambda: self.log.debug('Hello world!'), # Dummy task 92 | 'sleep': self._sleep, 93 | 'switch_state': self._switch_state, 94 | 'get_web_images': self._get_web_images, 95 | 'get_canon_images': self._get_canon_images, 96 | 'get_dummy': self._get_dummy, 97 | 'actuator': self._actuating, 98 | 'sensor': self._sensing, 99 | } 100 | 101 | def _save_measurements(self): 102 | if not list(self.sensors_accumulator.keys()) == list(self.datetime_dict): 103 | write_csv(self.sensors_accumulator, self.measurements_file) 104 | 105 | def _sleep(self, interval_seconds): 106 | self.log.debug('Start sleeping for ' + str(interval_seconds) + 's') 107 | time.sleep(interval_seconds) 108 | self.log.debug('Finish sleeping for ' + str(interval_seconds) + 's') 109 | 110 | def _switch_state(self, states_list_path, state_name, is_current_imaging_state): 111 | 112 | if is_current_imaging_state: 113 | self.current_imaging_state = state_name 114 | 115 | states = read_json(states_list_path) 116 | 117 | for actuator in states[state_name]: 118 | 119 | a = states[state_name][actuator] 120 | 121 | if a['mode'] == 'recovery': 122 | 123 | if actuator in self.main_config['system_state']: 124 | cmd = self.main_config['system_state'][actuator] 125 | else: 126 | if a['type'] == 'bool': 127 | cmd = 'off' 128 | 129 | else: 130 | 131 | if a['type'] == 'bool': 132 | cmd = 'on' if bool(a['value']) else 'off' 133 | 134 | if a['mode'] == 'permanent': 135 | self.main_config['system_state'][actuator] = cmd 136 | 137 | self._actuating( 138 | sensor_name=actuator, 139 | cmd=cmd, 140 | params={} 141 | ) 142 | 143 | self.log.info('Switching state to: {}'.format(state_name)) 144 | 145 | def _get_web_images(self): 146 | for c in self.cam_config['web_cams']: 147 | cam = WebCameraDriver( 148 | camera_info=c, 149 | folder=self.main_config['data_dir'], 150 | log=self.log, 151 | datetime_prefix=self.datetime_prefix, 152 | datetime_dict=self.datetime_dict, 153 | system_state=self.current_imaging_state 154 | ) 155 | 156 | cam.capture() 157 | 158 | def _get_canon_images(self): 159 | for c in self.cam_config['canon_cams']: 160 | cam = CanonCameraDriver( 161 | camera_info=c, 162 | folder=self.main_config['data_dir'], 163 | log=self.log, 164 | datetime_prefix=self.datetime_prefix, 165 | datetime_dict=self.datetime_dict, 166 | system_state=self.current_imaging_state 167 | ) 168 | 169 | cam.capture() 170 | 171 | def _get_slr_images(self): 172 | # ToDo: add SLR cameras support 173 | # Add parameters to SLR cameras constructor 174 | pass 175 | 176 | def _actuating(self, *args, **kwargs): 177 | cmd = 'self.board.active_sensors["' + kwargs['sensor_name'] + '"].' + kwargs['cmd'] + '(**' + str(kwargs['params']) + ')' 178 | exec(cmd) 179 | 180 | @_sensor 181 | @_average 182 | def _sensing(self, *args, **kwargs): 183 | ldict = { 184 | 'board': self.board 185 | } # Scope to use exec with local variables 186 | cmd = 'd = board.active_sensors["' + kwargs['sensor_name'] + '"].' + kwargs['cmd'] + '(**' + str(kwargs['params']) + ')' 187 | exec(cmd, globals(), ldict) 188 | d = ldict['d'] 189 | 190 | if d is None: 191 | # We don't raise an error here for smooth running 192 | error_message = 'No data collected with sensor ' + kwargs['sensor_name'] 193 | self.log.error(error_message) 194 | return None 195 | else: 196 | result = {} 197 | for k in list(d.keys()): 198 | result[kwargs['sensor_name'] + '_' + k] = d[k] 199 | return result 200 | 201 | @_sensor 202 | @_average 203 | def _get_dummy(self, *args, **kwargs): 204 | self.log.debug('Dummy value collected') 205 | import random 206 | return random.random() * kwargs['mean'] * 2 207 | -------------------------------------------------------------------------------- /monitoring_system/scheduler/Scheduler.py: -------------------------------------------------------------------------------- 1 | from apscheduler.schedulers.background import BlockingScheduler 2 | import atexit 3 | from pathlib import Path 4 | 5 | from monitoring_system.utils import * 6 | from monitoring_system.scheduler.PipelineExecutor import PipelineExecutor 7 | from monitoring_system.logger.Logger import get_logger 8 | from monitoring_system.drivers.sensors.Board import Board 9 | 10 | 11 | class Scheduler: 12 | def __init__(self, main_config): 13 | self.main_config = main_config 14 | self.main_config['system_state'] = {} 15 | 16 | self.create_dirs() 17 | self.logger = get_logger(main_config['project_name'], 18 | file=main_config['logs_dir'], 19 | level=main_config['log_level']) 20 | self.board = None 21 | self.scheduler = BlockingScheduler( 22 | logger=self.logger, 23 | job_defaults={'misfire_grace_time': 45}, 24 | ) 25 | self.setup() 26 | 27 | atexit.register(self._exit) 28 | 29 | def create_dirs(self): 30 | try: 31 | Path(self.main_config['logs_dir']).mkdir(parents=True, exist_ok=True) 32 | Path(self.main_config['data_dir']).mkdir(parents=True, exist_ok=True) 33 | 34 | Path(self.main_config['data_dir']).joinpath('sensors/').mkdir(parents=True, exist_ok=True) 35 | 36 | cameras_config = read_json(self.main_config['cameras_config']) 37 | for web_camera in cameras_config['web_cams']: 38 | Path(self.main_config['data_dir'])\ 39 | .joinpath('images/' + str(web_camera['type'] + '_' + str(web_camera['id'])))\ 40 | .mkdir(parents=True, exist_ok=True) 41 | except Exception as e: 42 | print('Error creating file structure!') 43 | 44 | def setup(self): 45 | try: 46 | board_scheme = read_json(self.main_config['board']) 47 | sensors = read_json(self.main_config['sensors_config']) 48 | board = Board(board_scheme, sensors, self.logger) 49 | self.board = board 50 | except Exception as e: 51 | self.logger.warning('No board specified in config or some error in Board init') 52 | self.logger.warning(str(e)) 53 | raise UserWarning(str(e)) 54 | 55 | for p in self.main_config['pipelines']: 56 | pipeline = read_json(p) 57 | pipeline_executor = PipelineExecutor( 58 | logger=get_logger(self.main_config['project_name'] + '.' + pipeline['name'], 59 | file=self.main_config['logs_dir'], 60 | level=self.main_config['log_level']), 61 | pipeline=pipeline['pipeline'], 62 | main_config=self.main_config, 63 | pipeline_name=pipeline['name'], 64 | board=self.board 65 | ) 66 | 67 | self.scheduler.add_job(func=(lambda executor=pipeline_executor: executor.execute()), 68 | **pipeline['run_interval']) 69 | 70 | def start(self): 71 | try: 72 | self.logger.info(self.main_config['project_name'] + ' started') 73 | self.scheduler.start() 74 | except Exception as e: 75 | self.logger.error('Error starting scheduler!') 76 | self.logger.error(str(e)) 77 | 78 | def _exit(self): 79 | self.board.exit() 80 | print('EXITING!!!') 81 | self.logger.info('System exited normally') 82 | self.scheduler.shutdown() 83 | -------------------------------------------------------------------------------- /monitoring_system/scheduler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NesterukSergey/Raspberry_Pi_monitoring_system/736c077576ac49775ffd59d59614d9ef97e33f1d/monitoring_system/scheduler/__init__.py -------------------------------------------------------------------------------- /monitoring_system/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from monitoring_system.utils.json import * 2 | from monitoring_system.utils.csv import write_csv, read_csv 3 | from monitoring_system.utils.get_time import get_time 4 | from monitoring_system.utils.txt import write_txt, read_txt 5 | from monitoring_system.utils.get_serial_number import get_serial_number 6 | from monitoring_system.utils.average import average 7 | from monitoring_system.utils.list_dirs import list_dirs 8 | from monitoring_system.utils.preprocess_cameras import * 9 | from monitoring_system.utils.preprocess_sensors import * 10 | from monitoring_system.utils.preprocess_sensors import * 11 | -------------------------------------------------------------------------------- /monitoring_system/utils/average.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def average(lst): 5 | n = np.array(lst, dtype=float) 6 | n = n[np.isfinite(n)] # Remove Nones 7 | filtered = _reject_outliers(n, m=1) 8 | mean = filtered.mean() 9 | 10 | if not np.isnan(mean): 11 | return mean 12 | else: 13 | return None 14 | 15 | 16 | def _reject_outliers(data, m=2): 17 | return data[abs(data - np.mean(data)) <= m * np.std(data)] 18 | -------------------------------------------------------------------------------- /monitoring_system/utils/csv.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from pathlib import Path 3 | 4 | 5 | def write_csv(df, file, sort=True): 6 | if 'DataFrame' not in str(type(df)): 7 | try: 8 | if sort: 9 | df = _sort_dict(df) 10 | df = _dict2df(df) 11 | except Exception as e: 12 | raise UserWarning('df must be DataFrame or dict', e) 13 | 14 | if Path(file).exists(): 15 | df.to_csv(file, index=0, mode='a', header=False) 16 | else: 17 | df.to_csv(file, index=0) 18 | 19 | 20 | def _dict2df(d): 21 | for k in d: 22 | if 'list' not in str(type(d[k])): 23 | d[k] = [d[k]] 24 | 25 | return pd.DataFrame(d) 26 | 27 | 28 | def _sort_dict(d): 29 | sorted_cols = sorted(d.keys(), key=lambda x: x.lower()) 30 | d_sorted = {} 31 | for k in sorted_cols: 32 | d_sorted[k] = d[k] 33 | 34 | return d_sorted 35 | 36 | 37 | def read_csv(file): 38 | if not Path(file).exists(): 39 | raise FileNotFoundError('No such .csv file: ' + str(file)) 40 | else: 41 | df = pd.read_csv(file) 42 | return df 43 | -------------------------------------------------------------------------------- /monitoring_system/utils/get_serial_number.py: -------------------------------------------------------------------------------- 1 | def get_serial_number(): 2 | cpu_serial = "0000000000000000" 3 | try: 4 | f = open('/proc/cpuinfo', 'r') 5 | for line in f: 6 | if line[0:6] == 'Serial': 7 | cpu_serial = line[10:26] 8 | f.close() 9 | except: 10 | cpu_serial = "ERROR000000000" 11 | 12 | return cpu_serial 13 | -------------------------------------------------------------------------------- /monitoring_system/utils/get_time.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | 4 | def get_time(): 5 | current_datetime = datetime.now() 6 | datetime_prefix = current_datetime.strftime('%y_%m_%d_%H_%M_%S') 7 | datetime_dict = { 8 | 'year': current_datetime.year, 9 | 'month': current_datetime.month, 10 | 'day': current_datetime.day, 11 | 'hour': current_datetime.hour, 12 | 'minute': current_datetime.minute, 13 | 'second': current_datetime.second 14 | } 15 | return datetime_prefix, datetime_dict 16 | -------------------------------------------------------------------------------- /monitoring_system/utils/gpioexp.py: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | 3 | import wiringpi as wp 4 | 5 | GPIO_EXPANDER_DEFAULT_I2C_ADDRESS = 0X2A 6 | GPIO_EXPANDER_WHO_AM_I = 0x00 7 | GPIO_EXPANDER_RESET = 0x01 8 | GPIO_EXPANDER_CHANGE_I2C_ADDR = 0x02 9 | GPIO_EXPANDER_SAVE_I2C_ADDR = 0x03 10 | GPIO_EXPANDER_PORT_MODE_INPUT = 0x04 11 | GPIO_EXPANDER_PORT_MODE_PULLUP = 0x05 12 | GPIO_EXPANDER_PORT_MODE_PULLDOWN = 0x06 13 | GPIO_EXPANDER_PORT_MODE_OUTPUT = 0x07 14 | GPIO_EXPANDER_DIGITAL_READ = 0x08 15 | GPIO_EXPANDER_DIGITAL_WRITE_HIGH = 0x09 16 | GPIO_EXPANDER_DIGITAL_WRITE_LOW = 0x0A 17 | GPIO_EXPANDER_ANALOG_WRITE = 0x0B 18 | GPIO_EXPANDER_ANALOG_READ = 0x0C 19 | GPIO_EXPANDER_PWM_FREQ = 0x0D 20 | GPIO_EXPANDER_ADC_SPEED = 0x0E 21 | 22 | INPUT = 0 23 | OUTPUT = 1 24 | INPUT_PULLUP = 2 25 | INPUT_PULLDOWN = 3 26 | 27 | def getPiI2CBusNumber(): 28 | """ 29 | Returns the I2C bus number (/dev/i2c-#) for the Raspberry Pi being used. 30 | Courtesy quick2wire-python-api 31 | https://github.com/quick2wire/quick2wire-python-api 32 | """ 33 | try: 34 | with open('/proc/cpuinfo','r') as f: 35 | for line in f: 36 | if line.startswith('Revision'): 37 | return 1 38 | except: 39 | return 0 40 | 41 | class gpioexp(object): 42 | """Troyka gpio expander.""" 43 | 44 | def __init__(self, gpioexp_address=GPIO_EXPANDER_DEFAULT_I2C_ADDRESS): 45 | 46 | # Setup I2C interface for accelerometer and magnetometer. 47 | wp.wiringPiSetup() 48 | self._i2c = wp.I2C() 49 | self._io = self._i2c.setupInterface('/dev/i2c-' + str(getPiI2CBusNumber()), gpioexp_address) 50 | # self._gpioexp.write_byte(self._addr, GPIO_EXPANDER_RESET) 51 | def reverse_uint16(self, data): 52 | result = ((data & 0xff) << 8) | ((data>>8) & 0xff) 53 | return result 54 | 55 | def digitalReadPort(self): 56 | port = self.reverse_uint16(self._i2c.readReg16(self._io, GPIO_EXPANDER_DIGITAL_READ)) 57 | return port 58 | 59 | def digitalRead(self, pin): 60 | mask = 0x0001 << pin 61 | result = 0 62 | if self.digitalReadPort() & mask: 63 | result = 1 64 | return result 65 | 66 | def digitalWritePort(self, value): 67 | value = self.reverse_uint16(value) 68 | self._i2c.writeReg16(self._io, GPIO_EXPANDER_DIGITAL_WRITE_HIGH, value) 69 | self._i2c.writeReg16(self._io, GPIO_EXPANDER_DIGITAL_WRITE_LOW, ~value) 70 | 71 | def digitalWrite(self, pin, value): 72 | sendData = self.reverse_uint16(0x0001<> log.txt 12 | 13 | while true 14 | do 15 | /usr/bin/python3.7 $DIR/$SCRIPT 16 | 17 | sleep 10 18 | echo "Restart monitoring server" >> log.txt 19 | done -------------------------------------------------------------------------------- /streamlit_server/refresh_server.py: -------------------------------------------------------------------------------- 1 | import time 2 | import argparse 3 | 4 | 5 | def rewrite_file(file): 6 | lines = None 7 | with open(file, 'r') as f: 8 | lines = f.readlines() 9 | f.close() 10 | 11 | reruns = int(lines[-1][2:]) 12 | reruns += 1 13 | lines[-1] = '# ' + str(reruns) 14 | 15 | with open(file, 'w') as f: 16 | f.writelines(lines) 17 | 18 | f.close() 19 | 20 | 21 | if __name__ == '__main__': 22 | parser = argparse.ArgumentParser(description='Server autorefresh') 23 | parser.add_argument( 24 | '--sec', 25 | type=int, 26 | default=60, 27 | help='interval in seconds (default: 60)' 28 | ) 29 | 30 | server_refresh = parser.parse_args() 31 | 32 | while True: 33 | rewrite_file('server.py') 34 | time.sleep(server_refresh.sec) 35 | -------------------------------------------------------------------------------- /streamlit_server/server.py: -------------------------------------------------------------------------------- 1 | import time 2 | import random 3 | import pandas as pd 4 | from pathlib import Path 5 | import plotly 6 | from PIL import Image 7 | import streamlit as st 8 | import sys 9 | sys.path.append('..') 10 | 11 | from monitoring_system.utils import * 12 | 13 | PATH_TO_DATA_FOLDER = '../data' 14 | 15 | 16 | def empty(): 17 | st.header('No data found!') 18 | st.text('Check path to data folder or start collecting new data.') 19 | 20 | 21 | @st.cache 22 | def get_img(path): 23 | return Image.open(Path(PATH_TO_DATA_FOLDER[:-4]).joinpath(path)) 24 | 25 | 26 | @st.cache 27 | def get_meas(df, col): 28 | return df[col] 29 | 30 | 31 | if __name__ == '__main__': 32 | try: 33 | check_files = list_dirs(PATH_TO_DATA_FOLDER) # Checking that data folder exists 34 | t = get_time() 35 | st.write('Last update time - {}:{}:{}'.format(t[1]['hour'], t[1]['minute'], t[1]['second'])) 36 | 37 | st.sidebar.subheader('Cameras') 38 | show_cameras = st.sidebar.checkbox('Show cameras', value=True) 39 | if show_cameras: 40 | images = get_collected_images(PATH_TO_DATA_FOLDER) 41 | images = augment_images(images) 42 | cameras = get_cameras(images) 43 | 44 | for cam_num, camera in enumerate(cameras): 45 | show_cam_n = st.sidebar.checkbox(camera, value=cam_num == 0) 46 | 47 | if show_cam_n: 48 | st.header('Camera: ' + str(camera)) 49 | camera_df = images[images['device'] == camera] 50 | 51 | cam_date = st.slider( 52 | 'Image number', 53 | min_value=1, 54 | max_value=len(camera_df['datetime'].values), 55 | value=len(camera_df['datetime'].values), 56 | key=str(camera) + '_slider' 57 | ) 58 | cam_date -= 1 59 | 60 | st.write('Timestamp:', camera_df['datetime'].values[cam_date]) 61 | img_path = camera_df[camera_df['datetime'] == camera_df['datetime'].values[cam_date]]['img_path'].item() 62 | img = get_img(img_path) 63 | st.image(img) 64 | st.write('_' * 20) 65 | 66 | 67 | st.sidebar.subheader('Measurements') 68 | show_measurements = st.sidebar.checkbox('Show measurements', value=True) 69 | if show_measurements: 70 | measurements = get_collected_sensors(PATH_TO_DATA_FOLDER) 71 | measurements = augment_sensors(measurements) 72 | sensors = get_sensors(measurements) 73 | 74 | for sens_num, sensor in enumerate(sensors): 75 | show_sens_n = st.sidebar.checkbox(sensor, value=True) 76 | if show_sens_n: 77 | meas = get_meas(measurements, sensor) 78 | st.line_chart(meas) 79 | 80 | 81 | st.sidebar.subheader('Miscellaneous') 82 | 83 | show_log = st.sidebar.checkbox('Show log') 84 | if show_log: 85 | log_files = list(map(str, list_dirs('../logs'))) 86 | selected_log = st.sidebar.radio('Choose log file', log_files) 87 | st.subheader('Logs:') 88 | st.text(selected_log) 89 | st.text(read_txt(selected_log)) 90 | 91 | show_board = st.sidebar.checkbox('Show board scheme') 92 | if show_board: 93 | board_scheme = read_txt('../board.txt') 94 | 95 | st.subheader('Board scheme:') 96 | st.text(board_scheme) 97 | 98 | except Exception as e: 99 | empty() 100 | raise e 101 | 102 | st.sidebar.button('Refresh') 103 | 104 | # The last line is used for external server refresh 105 | # Do not remove or modify it!!! 106 | # 1 -------------------------------------------------------------------------------- /streamlit_server/start_autorefresh.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd $DIR 5 | 6 | SCRIPT="refresh_server.py" 7 | 8 | sleep 5 9 | echo "Start streamlit autorefresh" >> log.txt 10 | 11 | while true 12 | do 13 | sudo python3 $SCRIPT 14 | 15 | sleep 60 16 | echo "Restart streamlit autorefresh" >> log.txt 17 | done -------------------------------------------------------------------------------- /streamlit_server/start_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 4 | cd $DIR 5 | 6 | SCRIPT="server.py" 7 | 8 | rm log.txt 9 | touch log.txt 10 | echo "Start streamlit server" >> log.txt 11 | 12 | while true 13 | do 14 | sudo streamlit run $SCRIPT 15 | 16 | sleep 60 17 | echo "Restart streamlit server" >> log.txt 18 | done 19 | 20 | --------------------------------------------------------------------------------