├── API.py ├── LICENSE.md ├── README.md ├── camera.py ├── classification.py ├── cloud.py ├── config.json ├── config.py ├── control.py ├── data.py ├── knob.py ├── launcher.py ├── lib_para_360_servo.py ├── main.py ├── models ├── README.md ├── boilover.tflite ├── boilover.txt ├── pan_on_off.tflite ├── pan_on_off.txt ├── pasta.tflite ├── pasta.txt ├── sauce.tflite ├── sauce.txt ├── stirring.tflite ├── stirring.txt ├── tflite-boiling_1_thermal_20200111031542-2020-01-11T18_45_13.068Z_dict.txt ├── tflite-boiling_1_thermal_20200111031542-2020-01-11T18_45_13.068Z_model.tflite ├── tflite-boiling_1_thermal_20200111031542-2020-01-11T18_45_13.068Z_tflite_metadata.json ├── tflite-boiling_water_1_20200111094256-2020-01-11T11_51_24.886Z_dict.txt ├── tflite-boiling_water_1_20200111094256-2020-01-11T11_51_24.886Z_model.tflite └── tflite-boiling_water_1_20200111094256-2020-01-11T11_51_24.886Z_tflite_metadata.json ├── pid.py ├── runlauncher ├── runonion ├── thermal_camera.py └── utils ├── README.md ├── calibration.py ├── create_automl_csv.py ├── create_temp_time_series.py ├── live_classify.py ├── print_angle.py ├── servo_calibration.py ├── servo_install.py ├── type_angle.py └── type_setpoint.py /API.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from flask import Flask 5 | from flask import request 6 | from flask_cors import CORS 7 | 8 | from main import OnionBot 9 | 10 | import logging 11 | 12 | # Silence Flask werkzeug logger 13 | logger = logging.getLogger("werkzeug") 14 | logger.setLevel(logging.ERROR) # Note! Set again below 15 | 16 | # Initialise OnionBot 17 | bot = OnionBot() 18 | bot.run() 19 | 20 | # Initialise flask server 21 | app = Flask(__name__) 22 | CORS(app) 23 | 24 | 25 | @app.route("/", methods=["GET", "POST"]) 26 | def index(): 27 | """Access the OnionBot portal over the local network""" 28 | 29 | if request.form["action"] == "start": 30 | logger.debug("start called") 31 | bot.start(request.form["value"]) 32 | return "1" 33 | 34 | if request.form["action"] == "stop": 35 | logger.debug("stop called") 36 | return bot.stop() 37 | 38 | if request.form["action"] == "get_latest_meta": 39 | logger.debug("get_latest_meta called") 40 | return bot.get_latest_meta() 41 | 42 | if request.form["action"] == "get_thermal_history": 43 | logger.debug("get_thermal_history called") 44 | return bot.get_thermal_history() 45 | 46 | if request.form["action"] == "set_label": 47 | logger.debug("set_label called") 48 | bot.set_label(request.form["value"]) 49 | return "1" 50 | 51 | if request.form["action"] == "set_no_label": 52 | logger.debug("set_no_label called") 53 | bot.set_no_label() 54 | return "1" 55 | 56 | if request.form["action"] == "set_classifiers": 57 | logger.debug("set_classifiers called") 58 | bot.set_classifiers(request.form["value"]) 59 | return "1" 60 | 61 | if request.form["action"] == "get_temperature_setpoint": 62 | logger.debug("get_temperature_setpoint called") 63 | return bot.get_temperature_setpoint() 64 | 65 | if request.form["action"] == "get_camera_frame_rate": 66 | logger.debug("get_camera_frame_rate called") 67 | return bot.get_camera_frame_rate() 68 | 69 | if request.form["action"] == "set_fixed_setpoint": 70 | logger.debug("set_fixed_setpoint called") 71 | bot.set_fixed_setpoint(request.form["value"]) 72 | return "1" 73 | 74 | if request.form["action"] == "set_temperature_target": 75 | logger.debug("set_temperature_target called") 76 | bot.set_temperature_target(request.form["value"]) 77 | return "1" 78 | 79 | if request.form["action"] == "set_temperature_hold": 80 | logger.debug("set_temperature_hold called") 81 | bot.set_temperature_hold() 82 | return "1" 83 | 84 | if request.form["action"] == "set_hob_off": 85 | logger.debug("set_hob_off called") 86 | bot.set_hob_off() 87 | return "1" 88 | 89 | if request.form["action"] == "set_pid_enabled": 90 | logger.debug("set_pid_enabled called") 91 | bot.set_pid_enabled(request.form["value"]) 92 | return "1" 93 | 94 | if request.form["action"] == "set_p_coefficient": 95 | logger.debug("set_p_coefficient called") 96 | bot.set_p_coefficient(request.form["value"]) 97 | return "1" 98 | 99 | if request.form["action"] == "set_i_coefficient": 100 | logger.debug("set_i_coefficient called") 101 | bot.set_i_coefficient(request.form["value"]) 102 | return "1" 103 | 104 | if request.form["action"] == "set_d_coefficient": 105 | logger.debug("set_d_coefficient called") 106 | bot.set_d_coefficient(request.form["value"]) 107 | return "1" 108 | 109 | if request.form["action"] == "set_pid_reset": 110 | logger.debug("set_pid_reset called") 111 | bot.set_pid_reset() 112 | return "1" 113 | 114 | if request.form["action"] == "set_frame_interval": 115 | logger.debug("set_frame_interval called") 116 | bot.set_frame_interval(request.form["value"]) 117 | return "1" 118 | 119 | if request.form["action"] == "get_all_labels": 120 | logger.debug("get_all_labels called") 121 | return bot.get_all_labels() 122 | 123 | if request.form["action"] == "get_all_classifiers": 124 | logger.debug("get_all_classifiers called") 125 | return bot.get_all_classifiers() 126 | 127 | if request.form["action"] == "pi-restart": 128 | os.system("sudo reboot") 129 | 130 | if request.form["action"] == "pi-shutdown": 131 | os.system("sudo shutdown now") 132 | 133 | if request.form["action"] == "restart": 134 | os.system(". ~/onionbot/runonion") 135 | 136 | if request.form["action"] == "quit": 137 | bot.quit() 138 | logger.info("Shutting down server") 139 | server_quit = request.environ.get("werkzeug.server.shutdown") 140 | if server_quit is None: 141 | raise RuntimeError("Not running with the Werkzeug Server") 142 | server_quit() 143 | sys.exit() 144 | os.system("sleep 1 ; pkill -f API.py") # If all else fails... 145 | 146 | 147 | if __name__ == "__main__": 148 | app.run(debug=False, host="0.0.0.0") 149 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Header](https://user-images.githubusercontent.com/32883278/97621285-a4208a80-1a1a-11eb-8b7f-90141d867982.png) 2 | 3 | # OnionBot | Python API 4 | 5 |

6 | 7 | 8 | 9 |

10 | 11 | *A collaborative computational cooking robot using computer vision built with Raspberry Pi* 12 | 13 | Python API integrating sensors, actuators and interfaces 14 | 15 | [See it in action on YouTube](https://youtu.be/W4utRCyo5C4) 16 | 17 | ### About 18 | *How can we apply robotics to home cooking?* 19 | 20 | Automation tech in the food industry is well known to reduce physical and cognitive demands on production-line operators. Perhaps the same technology could also reduce errors and help decision-making for home cooking? For example, how might robots augment the cooking skills of busy parents and professionals? 21 | 22 | The problem: kitchens pose very different design engineering challenges to production lines, because home cooking requires multi-purpose tools, not specialised machines. Robot arms can mimic human-kitchen interaction, but these are currently far too large and expensive to be feasible for the home. For multi-purpose sensing, cameras can detect a wide variety of cooking information, but there are currently no datasets for training cooking image classification algorithms. 23 | 24 | ![Hardware](https://user-images.githubusercontent.com/32883278/97621266-9ec34000-1a1a-11eb-82a4-4ef906dfa522.png) 25 | 26 | *Prototyping integration of industry automation techniques and machine vision into a simple robot that fits on a countertop.* 27 | 28 | ### System structure 29 | ![system2](https://user-images.githubusercontent.com/32883278/97644851-4d7b7680-1a43-11eb-94a6-876e7f35183a.png) 30 | 31 | `API.py` Access the OnionBot portal over the local network 32 | 33 | `camera.py` Control the camera using the Picamera module (threaded) 34 | 35 | `classification.py` Classify images with TensorFlow and the Coral Edge TPU (threaded) 36 | 37 | `cloud.py` Upload images to Google Cloud storage buckets (threaded) 38 | 39 | `config.json` Configure settings, labels and models 40 | 41 | `config.py` Interface with the `config.json` file 42 | 43 | `control.py` Wrapper for PID module, manage control data structures for `main.py` 44 | 45 | `data.py` Manage data structures and metadata for API 46 | 47 | `knob.py` Wrapper for servo module to control hob temperature setting (threaded) 48 | 49 | `launcher.py` Launch OnionBot software from the big red button 50 | 51 | `lib_para_360_servo.py` Parallax 360 [servo drivers](http://parallax.com/product/900-00008) 52 | 53 | `main.py` Main script (threaded) 54 | 55 | `pid.py` Proportional Integral Derivative hob temperature controller (threaded) 56 | 57 | `runlauncher` Launch big red button listener script 58 | 59 | `runonion` Launch OnionBot software 60 | 61 | `thermal_camera.py` Wrapper for the Adafruit MLX90640 thermal camera module (threaded) 62 | 63 | ### Dependencies 64 | 65 | 1. `pip3 install pillow` 66 | 2. `pip3 install adafruit-circuitpython-mlx90640` 67 | 3. `pip install Adafruit-Blinka` 68 | 4. [Tensorflow install guide](https://www.tensorflow.org/lite/models/image_classification/overview) 69 | 5. [Servo driver install](http://parallax.com/product/900-00008) 70 | 71 | 72 | ### The OnionBot idea was developed through a research project at Imperial College London 73 | 74 | Check out the paper for technical details in much more depth! 75 | 76 | [OnionBot: A System for Collaborative Computational Cooking - ArXiv](https://arxiv.org/pdf/2011.05039.pdf) 77 | 78 | ![arxiv](https://user-images.githubusercontent.com/32883278/98860117-18b3ea00-245b-11eb-976e-163721560a50.png) 79 | 80 | 81 | ### Interested in building a cooking automation robot? 82 | 83 | Get in touch! 84 | -------------------------------------------------------------------------------- /camera.py: -------------------------------------------------------------------------------- 1 | import multiprocessing as mp 2 | from multiprocessing import JoinableQueue, Event 3 | from queue import Empty 4 | 5 | from picamera import PiCamera 6 | 7 | import logging 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | class Camera(object): 13 | """Control the camera using the Picamera module (threaded)""" 14 | 15 | def __init__(self): 16 | self.file_queue = JoinableQueue(1) 17 | 18 | self.quit_event = Event() 19 | 20 | def _worker(self): 21 | 22 | logger.info("Initialising camera...") 23 | 24 | camera = PiCamera() 25 | camera.rotation = 180 26 | camera.zoom = (0.05, 0.0, 0.75, 0.95) 27 | camera.resolution = (1024, 768) 28 | 29 | while True: 30 | try: # Timeout raises queue.Empty 31 | file_path = self.file_queue.get(block=True, timeout=0.1) 32 | 33 | logger.debug("Capturing image") 34 | camera.capture(file_path, resize=(240, 240)) 35 | 36 | self.file_queue.task_done() 37 | 38 | except Empty: 39 | if self.quit_event.is_set(): 40 | logger.debug("Quitting camera thread...") 41 | break 42 | 43 | def start(self, file_path): 44 | logger.debug("Calling start") 45 | self.file_queue.put(file_path, block=True) 46 | 47 | def join(self): 48 | logger.debug("Calling join") 49 | self.file_queue.join() 50 | 51 | def launch(self): 52 | logger.debug("Initialising worker") 53 | self.p = mp.Process(target=self._worker, daemon=True) 54 | self.p.start() 55 | 56 | def quit(self): 57 | self.quit_event.set() 58 | self.p.join() 59 | -------------------------------------------------------------------------------- /classification.py: -------------------------------------------------------------------------------- 1 | from edgetpu.classification.engine import ClassificationEngine 2 | from edgetpu.utils import dataset_utils 3 | from PIL import Image 4 | from threading import Thread, Event 5 | from queue import Queue, Empty 6 | 7 | from config import Classifiers 8 | from json import dumps 9 | from collections import deque 10 | 11 | import logging 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | classifiers = Classifiers() 16 | 17 | 18 | class Classify(object): 19 | """Classify images with TensorFlow and the Coral Edge TPU (threaded)""" 20 | 21 | def __init__(self): 22 | 23 | logger.info("Initialising classifier...") 24 | 25 | self.library = classifiers.get_classifiers() 26 | self.loaded = {} 27 | self.active = [] 28 | 29 | self.quit_event = Event() 30 | self.file_queue = Queue() 31 | self.database = {} 32 | 33 | def _worker(self): 34 | 35 | logger.debug("Initialising classification worker") 36 | 37 | while True: 38 | try: # Timeout raises queue.Empty 39 | 40 | image = self.file_queue.get(block=True, timeout=0.1) 41 | 42 | except Empty: 43 | if self.quit_event.is_set(): 44 | logger.debug("Quitting thread...") 45 | break 46 | 47 | else: 48 | image = Image.open(image) 49 | 50 | library = self.library 51 | active = self.active 52 | database = self.database 53 | 54 | # Iterate over all classifiers 55 | for name in library: 56 | 57 | # Only classify active classifiers 58 | if name in active: 59 | 60 | # Ensure classifer is in database 61 | try: 62 | storage = database[name] 63 | except KeyError: 64 | storage = {} 65 | 66 | # Load classifier information 67 | engine = self.loaded[name]["model"] 68 | labels = self.loaded[name]["labels"] 69 | thresholds = self.loaded[name]["thresholds"] 70 | 71 | # Run inference 72 | logger.debug("Starting classifier %s " % (name)) 73 | 74 | try: 75 | results = engine.classify_with_image( 76 | image, top_k=3, threshold=0 77 | ) # Return top 3 probability items 78 | logger.debug("%s results: " % (results)) 79 | except OSError: 80 | logger.info("OSError detected, retrying") 81 | break 82 | 83 | # Create dictionary including those not in top_k 84 | big_dict = {} 85 | for result in results: 86 | label = labels[result[0]] 87 | confidence = round(result[1].item(), 2) 88 | big_dict[label] = confidence 89 | 90 | not_in_top_k = big_dict.keys() ^ labels.values() 91 | for label in not_in_top_k: 92 | # Zero confidence ensures moving average keeps moving 93 | big_dict[label] = 0 94 | 95 | # Iterate over the dictionary 96 | for label, confidence in big_dict.items(): 97 | 98 | # Ensure label is in classifier storage entry 99 | if label not in storage: 100 | storage[label] = {} 101 | storage[label]["queue"] = [0] * 5 102 | 103 | # Update nested storage dictionary 104 | this_label = storage[label] 105 | this_label["confidence"] = confidence 106 | 107 | # Use deque to update moving average 108 | queue = deque(this_label["queue"]) 109 | queue.append(confidence) 110 | queue.popleft() 111 | this_label["queue"] = list(queue) 112 | average = round(sum(queue) / 5, 2) 113 | this_label["average"] = average 114 | 115 | # Use threshold storage to check whether it exceeds 116 | this_label["threshold"] = thresholds[label] 117 | this_label["boolean"] = average >= thresholds[label] 118 | 119 | # Update database with all information from this classifier 120 | database[name] = storage 121 | 122 | # Remove classifiers in database that are not active 123 | elif name in database: 124 | del database[name] 125 | 126 | self.database = database 127 | 128 | self.file_queue.task_done() 129 | 130 | def load_classifiers(self, input_string): 131 | for name in input_string.split(","): 132 | 133 | # Check if classifier has already been loaded 134 | if name not in self.loaded: 135 | logger.debug("Loading classifier %s " % (name)) 136 | 137 | # Read attributes from library and initialise 138 | try: 139 | attr = self.library[name] 140 | output = {} 141 | output["labels"] = dataset_utils.read_label_file(attr["labels"]) 142 | output["model"] = ClassificationEngine(attr["model"]) 143 | output["thresholds"] = attr["thresholds"] 144 | self.loaded[name] = output 145 | except KeyError: 146 | raise KeyError("Classifier name not found in database") 147 | except FileNotFoundError: 148 | raise FileNotFoundError( 149 | "Model or labels not found in models folder" 150 | ) 151 | 152 | else: 153 | logger.debug("Classifier already loaded %s " % (name)) 154 | 155 | def set_classifiers(self, input_string): 156 | for name in input_string.split(","): 157 | 158 | # Check if classifier has already been loaded 159 | if name not in self.loaded: 160 | logger.debug("Classifier not loaded %s: loading " % (name)) 161 | self.load_classifiers(name) 162 | self.active = input_string.split(",") 163 | 164 | def get_classifiers(self): 165 | return dumps(self.library) 166 | 167 | def start(self, file_path): 168 | logger.debug("Calling start") 169 | self.file_queue.put(file_path) 170 | 171 | def join(self): 172 | logger.debug("Calling join") 173 | self.file_queue.join() 174 | 175 | def launch(self): 176 | logger.debug("Initialising classification worker") 177 | self.thread = Thread(target=self._worker, daemon=True) 178 | self.thread.start() 179 | 180 | def quit(self): 181 | self.quit_event.set() 182 | logger.debug("Waiting for classification thread to finish") 183 | self.thread.join() 184 | -------------------------------------------------------------------------------- /cloud.py: -------------------------------------------------------------------------------- 1 | from os import environ, path 2 | from google.cloud import storage 3 | from threading import Thread, Event 4 | from queue import Queue, Empty 5 | 6 | import logging 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/home/pi/onionbot-819a387e4e79.json" 11 | 12 | BUCKET = "onion_bucket" 13 | PATH = path.dirname(__file__) 14 | 15 | 16 | class Cloud(object): 17 | """Upload images to Google Cloud storage buckets (threaded)""" 18 | 19 | def __init__(self): 20 | 21 | logger.info("Initialising cloud upload...") 22 | 23 | self.quit_event = Event() 24 | self.thermal_file_queue = Queue() 25 | self.camera_file_queue = Queue() 26 | self.bucket = BUCKET 27 | 28 | def _camera_worker(self): 29 | 30 | logger.debug("Initialising upload worker for camera") 31 | 32 | client = storage.Client() 33 | bucket = client.get_bucket(BUCKET) 34 | 35 | while True: 36 | try: # Timeout raises queue.Empty 37 | local_path = self.camera_file_queue.get(block=True, timeout=0.1) 38 | cloud_path = local_path.replace(PATH + "/" + BUCKET + "/", "") 39 | 40 | blob = bucket.blob(cloud_path) 41 | blob.upload_from_filename(local_path) 42 | blob.make_public() 43 | logger.debug("Uploaded camera file to cloud: %s" % (local_path)) 44 | logger.debug("Blob is publicly accessible at %s" % (blob.public_url)) 45 | 46 | self.camera_file_queue.task_done() 47 | 48 | except Empty: 49 | if self.quit_event.is_set(): 50 | logger.debug("Quitting camera camera thread...") 51 | break 52 | 53 | def start_camera(self, file_path): 54 | logger.debug("Calling start for camera") 55 | self.camera_file_queue.put(file_path) 56 | 57 | def join_camera(self): 58 | logger.debug("Calling join for camera") 59 | self.camera_file_queue.join() 60 | 61 | def launch_camera(self): 62 | logger.debug("Initialising worker for camera") 63 | self.camera_thread = Thread(target=self._camera_worker, daemon=True) 64 | self.camera_thread.start() 65 | 66 | def _thermal_worker(self): 67 | 68 | logger.debug("Initialising upload worker for thermal") 69 | 70 | client = storage.Client() 71 | bucket = client.get_bucket(BUCKET) 72 | 73 | while True: 74 | try: # Timeout raises queue.Empty 75 | local_path = self.thermal_file_queue.get(block=True, timeout=0.1) 76 | cloud_path = local_path.replace(PATH + "/" + BUCKET + "/", "") 77 | 78 | blob = bucket.blob(cloud_path) 79 | blob.upload_from_filename(local_path) 80 | blob.make_public() 81 | logger.debug("Uploaded thermal file to cloud: %s" % (local_path)) 82 | logger.debug("Blob is publicly accessible at %s" % (blob.public_url)) 83 | 84 | self.thermal_file_queue.task_done() 85 | 86 | except Empty: 87 | if self.quit_event.is_set(): 88 | logger.debug("Quitting thermal camera thread...") 89 | break 90 | 91 | def start_thermal(self, file_path): 92 | logger.debug("Calling start for thermal") 93 | self.thermal_file_queue.put(file_path) 94 | 95 | def join_thermal(self): 96 | logger.debug("Calling join for thermal") 97 | self.thermal_file_queue.join() 98 | 99 | def launch_thermal(self): 100 | logger.debug("Initialising worker for thermal") 101 | self.thermal_thread = Thread(target=self._thermal_worker, daemon=True) 102 | self.thermal_thread.start() 103 | 104 | def get_public_path(self, local_path): 105 | if local_path: 106 | local_path = local_path.replace(PATH + "/", "") 107 | cloud_location = "https://storage.googleapis.com" 108 | 109 | return f"{cloud_location}/{local_path}" 110 | else: 111 | return None 112 | 113 | def quit(self): 114 | self.quit_event.set() 115 | logger.debug("Waiting for camera cloud thread to finish uploading") 116 | self.camera_thread.join() 117 | logger.debug("Waiting for thermal cloud thread to finish uploading") 118 | self.thermal_thread.join() 119 | -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "settings": { 3 | "frame_interval": "0", 4 | "Kp": 1.5, 5 | "Ki": 0.03, 6 | "Kd": 0.0, 7 | "sample_time": 0.01, 8 | "output_limit": 75 9 | }, 10 | "labels": { 11 | "type": "labels", 12 | "attributes": { 13 | "pan_on_off": { 14 | "0": "pan_on", 15 | "1": "pan_off" 16 | }, 17 | "pasta": { 18 | "0": "empty_pan", 19 | "1": "add_water", 20 | "2": "water_boiling", 21 | "3": "add_pasta" 22 | }, 23 | "sauce": { 24 | "0": "empty_pan", 25 | "1": "add_oil", 26 | "2": "add_onions", 27 | "3": "onions_cooked", 28 | "4": "add_puree", 29 | "5": "add_tomatoes" 30 | } 31 | } 32 | }, 33 | "classifiers": { 34 | "pasta": { 35 | "model": "models/pasta.tflite", 36 | "labels": "models/pasta.txt", 37 | "thresholds": { 38 | "add_pasta": 0.5, 39 | "empty_pan": 0.5, 40 | "water_boiling": 0.5, 41 | "add_water": 0.5 42 | } 43 | }, 44 | "sauce": { 45 | "model": "models/sauce.tflite", 46 | "labels": "models/sauce.txt", 47 | "thresholds": { 48 | "add_onions" : 0.5, 49 | "add_tomatoes" : 0.5, 50 | "add_oil" : 0.5, 51 | "empty_pan" : 0.5, 52 | "onions_cooked" : 0.7, 53 | "add_puree" : 0.5 54 | } 55 | }, 56 | "pan_on_off": { 57 | "model": "models/pan_on_off.tflite", 58 | "labels": "models/pan_on_off.txt", 59 | "thresholds": { 60 | "pan_off" : 0.5, 61 | "pan_on" : 0.5 62 | } 63 | }, 64 | "boilover": { 65 | "model": "models/boilover.tflite", 66 | "labels": "models/boilover.txt", 67 | "thresholds": { 68 | "not_boiling_over" : 0.5, 69 | "boiling_over" : 0.5 70 | } 71 | }, 72 | "stirring": { 73 | "model": "models/stirring.tflite", 74 | "labels": "models/stirring.txt", 75 | "thresholds": { 76 | "stirring" : 0.5, 77 | "not_stirring" : 0.5 78 | } 79 | } 80 | } 81 | } -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | from json import dumps, dump, load 2 | 3 | FILE = "/home/pi/onionbot/config.json" 4 | 5 | 6 | class Settings(object): 7 | """Interface with the `config.json` settings dictionary""" 8 | 9 | def get_setting(self, key): 10 | 11 | with open(FILE) as json_data_file: 12 | config = load(json_data_file) 13 | 14 | settings = config["settings"] 15 | 16 | try: 17 | return settings[key] 18 | except KeyError: 19 | raise KeyError("Settings key not found") 20 | 21 | def set_setting(self, key, value): 22 | 23 | with open(FILE) as json_data_file: 24 | config = load(json_data_file) 25 | 26 | settings = config["settings"] 27 | 28 | if key in settings: 29 | settings[key] = value 30 | config["settings"] = settings 31 | 32 | # dump new version of config 33 | with open(FILE, "w") as outfile: 34 | dump(config, outfile, indent=4) 35 | else: 36 | raise KeyError("Settings key not found") 37 | 38 | 39 | class Labels(object): 40 | """Interface with the `config.json` labels dictionary""" 41 | 42 | def get_labels(self): 43 | 44 | with open(FILE) as json_data_file: 45 | config = load(json_data_file) 46 | labels = config["labels"] 47 | return dumps(labels) 48 | 49 | 50 | class Classifiers(object): 51 | """Interface with the `config.json` classifiers dictionary""" 52 | 53 | def get_classifiers(self): 54 | 55 | with open(FILE) as json_data_file: 56 | config = load(json_data_file) 57 | classifiers = config["classifiers"] 58 | return classifiers 59 | -------------------------------------------------------------------------------- /control.py: -------------------------------------------------------------------------------- 1 | from threading import Thread, Event 2 | from collections import deque 3 | 4 | from config import Settings 5 | from pid import PID 6 | from knob import Knob 7 | 8 | import logging 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | config = Settings() 13 | pid = PID( 14 | Kp=config.get_setting("Kp"), 15 | Ki=config.get_setting("Ki"), 16 | Kd=config.get_setting("Kd"), 17 | sample_time=config.get_setting("sample_time"), 18 | output_limits=(0, config.get_setting("output_limit")), 19 | is_enabled=False, 20 | proportional_on_measurement=True, 21 | ) 22 | knob = Knob() 23 | 24 | 25 | DEADBAND_THRESHOLD = 2.5 26 | 27 | 28 | class Control(object): 29 | """Wrapper for PID module, manage control data structures for `main.py`""" 30 | 31 | def __init__(self): 32 | 33 | logger.info("Initialising control script...") 34 | 35 | self.quit_event = Event() 36 | 37 | self.temperature = 0 38 | 39 | self.fixed_setpoint = 0 40 | self.temperature_target = None 41 | self.servo_setpoint = 0 42 | self.servo_achieved = 0 43 | 44 | self.setpoint_history = deque([0] * 120) 45 | self.achieved_history = deque([0] * 120) 46 | 47 | self.data = { 48 | "servo_setpoint": None, 49 | "servo_setpoint_history": None, 50 | "servo_achieved": None, 51 | "servo_achieved_history": None, 52 | "temperature_target": None, 53 | "pid_enabled": None, 54 | "p_coefficient": None, 55 | "i_coefficient": None, 56 | "d_coefficient": None, 57 | "p_component": 0, 58 | "i_component": 0, 59 | "d_component": 0, 60 | } 61 | 62 | def _worker(self): 63 | 64 | while True: 65 | current_setpoint = knob.get_achieved() 66 | 67 | if pid.is_enabled: 68 | pid.setpoint = self.temperature_target 69 | target_setpoint = pid(self.temperature) 70 | else: 71 | target_setpoint = self.fixed_setpoint 72 | 73 | delta = abs(target_setpoint - current_setpoint) 74 | # logger.debug("Servo setpoint delta: {:.1f}".format(delta)) 75 | 76 | if delta >= DEADBAND_THRESHOLD: 77 | knob.update_setpoint(target_setpoint) 78 | 79 | if self.quit_event.is_set(): 80 | logger.debug("Quitting control thread...") 81 | break 82 | 83 | def launch(self): 84 | logger.debug("Initialising worker") 85 | self.thread = Thread(target=self._worker, daemon=True) 86 | self.thread.start() 87 | 88 | def update_fixed_setpoint(self, setpoint): 89 | logger.debug("Updating fixed setpoint to %s/100 " % (setpoint)) 90 | self.fixed_setpoint = float(setpoint) 91 | self.set_pid_enabled(False) 92 | self.temperature_target = "_" 93 | 94 | def hob_off(self): 95 | logger.debug("hob_off called") 96 | self.update_fixed_setpoint(0) 97 | 98 | def update_temperature_target(self, setpoint): 99 | logger.debug("Updating self.temperature_target to %s degrees " % (setpoint)) 100 | self.temperature_target = float(setpoint) 101 | self.set_pid_enabled(True) 102 | self.fixed_setpoint = None 103 | 104 | def hold_temperature(self): 105 | setpoint = str(self.temperature) 106 | logger.debug( 107 | "Updating self.temperature_target to hold at %s degrees " % (setpoint) 108 | ) 109 | self.temperature_target = float(setpoint) 110 | self.set_pid_enabled(True) 111 | self.fixed_setpoint = None 112 | 113 | def set_pid_enabled(self, enabled): 114 | pid.set_is_enabled(enabled, last_output=knob.get_setpoint()) 115 | 116 | def set_p_coefficient(self, Kp): 117 | Kp = float(Kp) 118 | config.set_setting("Kp", Kp) 119 | pid.set_coefficients(Kp, None, None) 120 | 121 | def set_i_coefficient(self, Ki): 122 | Ki = float(Ki) 123 | config.set_setting("Ki", Ki) 124 | pid.set_coefficients(None, Ki, None) 125 | 126 | def set_d_coefficient(self, Kd): 127 | Kd = float(Kd) 128 | config.set_setting("Kd", Kd) 129 | pid.set_coefficients(None, None, Kd) 130 | 131 | def set_pid_reset(self): 132 | pid.reset() 133 | 134 | # Create update_angle_setpoint separate to temp setpoint 135 | 136 | def refresh(self, temperature): 137 | """NOTE: Must be called only ONCE per frame for history to stay in sync with thermal""" 138 | logger.debug("Refresh called") 139 | 140 | self.temperature = float(temperature) 141 | 142 | setpoint = round(knob.get_setpoint()) 143 | logger.debug("Servo get_setpoint returned %s " % (setpoint)) 144 | 145 | setpoint_history = self.setpoint_history 146 | setpoint_history.append(setpoint) 147 | setpoint_history.popleft() 148 | self.setpoint_history = setpoint_history 149 | 150 | achieved = knob.get_achieved() 151 | logger.debug("Servo get_achieved returned %s " % (achieved)) 152 | 153 | achieved_history = self.achieved_history 154 | achieved_history.append(achieved) 155 | achieved_history.popleft() 156 | self.achieved_history = achieved_history 157 | 158 | pid_enabled = pid.is_enabled 159 | coefficients = pid.coefficients 160 | components = pid.components 161 | 162 | self.data = { 163 | "servo_setpoint": setpoint, 164 | "servo_setpoint_history": list(setpoint_history), 165 | "servo_achieved": achieved, 166 | "servo_achieved_history": list(achieved_history), 167 | "temperature_target": self.temperature_target, 168 | "pid_enabled": pid_enabled, 169 | "p_coefficient": coefficients[0], 170 | "i_coefficient": coefficients[1], 171 | "d_coefficient": coefficients[2], 172 | "p_component": components[0], 173 | "i_component": components[1], 174 | "d_component": components[2], 175 | } 176 | 177 | def quit(self): 178 | self.quit_event.set() 179 | knob.quit() 180 | self.thread.join(timeout=1) 181 | -------------------------------------------------------------------------------- /data.py: -------------------------------------------------------------------------------- 1 | from json import dump 2 | from cloud import Cloud 3 | from os import makedirs, path 4 | from datetime import datetime 5 | from collections import Counter 6 | 7 | import logging 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | cloud = Cloud() 12 | 13 | PATH = path.dirname(__file__) 14 | BUCKET = cloud.bucket 15 | 16 | 17 | class Data: 18 | def __init__(self): 19 | """Manage data structures and metadata for API""" 20 | 21 | logger.info("Initialising data management...") 22 | 23 | self.camera_filepath = None 24 | self.thermal_filepath = None 25 | self.thermal_history_filepath = None 26 | self.meta_filepath = None 27 | self.timer = datetime.now() 28 | 29 | def start_session(self, session_ID): 30 | 31 | # Labels file creation 32 | labels_file_dir = f"{PATH}/{BUCKET}/{session_ID}/camera" 33 | labels_file_path = labels_file_dir + "/labels.csv" 34 | 35 | if not path.isfile(labels_file_path): 36 | makedirs(labels_file_dir, exist_ok=True) 37 | with open(labels_file_path, "w") as file: 38 | file.write("image_path[,label]\n") 39 | file.close() 40 | 41 | label_list = [] 42 | with open(labels_file_path, "r") as file: 43 | line = file.readline() # Deliberately skip header line! 44 | while line: 45 | line = file.readline() 46 | label = ",".join(line.split(",")[1:]) # Isolate labels 47 | label = label.strip("\n") 48 | label_list.append(label) 49 | 50 | label_count = dict(Counter(label_list)) 51 | label_count.pop("", None) # Remove empty labels 52 | label_count.pop("None", None) # Remove unlabelled images 53 | 54 | self.labels_file_path = labels_file_path 55 | self.label_count = label_count 56 | 57 | def generate_file_data(self, session_ID, timer, measurement_ID, label): 58 | """Generate file_data for local and cloud storage for all file types""" 59 | 60 | file_data = {} 61 | time_stamp = timer.strftime("%Y-%m-%d_%H-%M-%S-%f") 62 | 63 | # Camera filepath 64 | new_path = f"{PATH}/{BUCKET}/{session_ID}/camera/{label}" 65 | makedirs(new_path, exist_ok=True) 66 | filename = f"{session_ID}_{str(measurement_ID).zfill(5)}_{time_stamp}_camera_{label}.jpg" 67 | file_data["camera_file"] = f"{new_path}/{filename}" 68 | 69 | if not session_ID: 70 | file_data["label_file"] = None 71 | file_data["label_count"] = None 72 | else: 73 | # Update labels file 74 | 75 | with open(self.labels_file_path, "a") as file: 76 | file.write( 77 | f"gs://{BUCKET}/{session_ID}/camera/{label}/{filename},{label}\n" 78 | ) 79 | file.close() 80 | 81 | file_data["label_file"] = self.labels_file_path 82 | 83 | if label: 84 | # Increment labels counter 85 | try: 86 | self.label_count[label] += 1 87 | except KeyError: 88 | self.label_count[label] = 1 89 | 90 | file_data["label_count"] = self.label_count 91 | 92 | # Thermal filepath 93 | new_path = f"{PATH}/{BUCKET}/{session_ID}/thermal/{label}" 94 | makedirs(new_path, exist_ok=True) 95 | filename = f"{session_ID}_{str(measurement_ID).zfill(5)}_{time_stamp}_thermal_{label}.jpg" 96 | file_data["thermal_file"] = f"{new_path}/{filename}" 97 | 98 | # Meta filepath 99 | new_path = f"{PATH}/{BUCKET}/{session_ID}/meta/{label}" 100 | makedirs(new_path, exist_ok=True) 101 | filename = f"{session_ID}_{str(measurement_ID).zfill(5)}_{time_stamp}_meta_{label}.json" 102 | file_data["meta"] = f"{new_path}/{filename}" 103 | 104 | return file_data 105 | 106 | def generate_meta( 107 | self, 108 | session_ID, 109 | timer, 110 | measurement_ID, 111 | label, 112 | file_data, 113 | thermal_data, 114 | control_data, 115 | classification_data, 116 | ): 117 | """Generate metadata to be parsed by portal""" 118 | 119 | interval = round((timer - self.timer).total_seconds(), 1) 120 | self.timer = timer 121 | time_stamp = timer.strftime("%Y-%m-%d_%H-%M-%S-%f") 122 | 123 | camera_filepath = cloud.get_public_path(file_data["camera_file"]) 124 | thermal_filepath = cloud.get_public_path(file_data["thermal_file"]) 125 | 126 | data = { 127 | "type": "meta", 128 | "id": f"{session_ID}_{measurement_ID}_{str(time_stamp)}", 129 | "attributes": { 130 | "session_ID": session_ID, 131 | "interval": interval, 132 | "label": label, 133 | "measurement_ID": measurement_ID, 134 | "time_stamp": time_stamp, 135 | "label_count": file_data["label_count"], 136 | "classification_data": classification_data, 137 | "camera_filepath": camera_filepath, 138 | "thermal_filepath": thermal_filepath, 139 | "temperature": thermal_data["temperature"], 140 | "thermal_history": thermal_data["thermal_history"], 141 | "servo_setpoint": control_data["servo_setpoint"], 142 | "servo_setpoint_history": control_data["servo_setpoint_history"], 143 | "servo_achieved": control_data["servo_achieved"], 144 | "servo_achieved_history": control_data["servo_achieved_history"], 145 | "temperature_target": control_data["temperature_target"], 146 | "pid_enabled": control_data["pid_enabled"], 147 | "p_coefficient": control_data["p_coefficient"], 148 | "i_coefficient": control_data["i_coefficient"], 149 | "d_coefficient": control_data["d_coefficient"], 150 | "p_component": control_data["p_component"], 151 | "i_component": control_data["i_component"], 152 | "d_component": control_data["d_component"], 153 | }, 154 | } 155 | 156 | with open(file_data["meta"], "w") as write_file: 157 | dump(data, write_file) 158 | 159 | return data 160 | -------------------------------------------------------------------------------- /knob.py: -------------------------------------------------------------------------------- 1 | from threading import Thread, Event 2 | import pigpio 3 | import time 4 | import lib_para_360_servo 5 | import statistics 6 | 7 | import logging 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | MIN_SAFE_ANGLE = 20 12 | MAX_SAFE_ANGLE = 310 13 | 14 | # OFF_ANGLE = 25 15 | MIN_SET_POINT_ANGLE = 50 16 | MAX_SET_POINT_ANGLE = 310 17 | 18 | TIMEOUT_PERIOD = 2.5 19 | 20 | 21 | class Knob(object): 22 | """ Wrapper for servo module to control hob temperature setting """ 23 | 24 | def __init__( 25 | self, 26 | unitsFC=360, 27 | dcMin=31.85, 28 | dcMax=956.41, 29 | feedback_gpio=5, 30 | servo_gpio=13, 31 | min_pw=1280, # 1210 32 | max_pw=1750, # 1750 33 | min_speed=-1, 34 | max_speed=1, 35 | sampling_time=0.01, 36 | Kp_p=0.1, # not too big values, otherwise output of position control would slow down too abrupt 37 | Ki_p=0.05, 38 | Kd_p=0.0, 39 | Kp_s=0.3, 40 | Ki_s=0, 41 | Kd_s=0, 42 | ): 43 | 44 | logger.info("Initialising servo motor") 45 | 46 | self.stop_event = Event() 47 | 48 | pi = pigpio.pi() 49 | 50 | self.pi = pi 51 | self.unitsFC = unitsFC 52 | self.dcMin = dcMin 53 | self.dcMax = dcMax 54 | self.sampling_time = sampling_time 55 | self.Kp_p = Kp_p 56 | self.Ki_p = Ki_p 57 | self.Kd_p = Kd_p 58 | self.Kp_s = Kp_s 59 | self.Ki_s = Ki_s 60 | self.Kd_s = Kd_s 61 | 62 | self.feedback = lib_para_360_servo.read_pwm(pi=self.pi, gpio=feedback_gpio) 63 | self.servo = lib_para_360_servo.write_pwm( 64 | pi=self.pi, 65 | gpio=servo_gpio, 66 | min_pw=min_pw, 67 | max_pw=max_pw, 68 | min_speed=min_speed, 69 | max_speed=max_speed, 70 | ) 71 | 72 | self.target_setpoint = 0 73 | 74 | def _set_speed(self, speed): 75 | 76 | self.servo.set_speed(speed) 77 | 78 | return None 79 | 80 | def _worker(self, target_angle): 81 | 82 | #logger.debug("Starting thread") 83 | 84 | target_angle = float(target_angle) 85 | 86 | safe_target = max(min(target_angle, MAX_SAFE_ANGLE), MIN_SAFE_ANGLE) 87 | 88 | #logger.debug("Worker called with target %s " % (safe_target)) 89 | 90 | target_angle = float(360 - target_angle) 91 | 92 | # initial values sum_error_* 93 | sum_error_p = 0 94 | sum_error_s = 0 95 | 96 | # initial values error_*_old 97 | error_p_old = 0 98 | error_s_old = 0 99 | 100 | reached_sp_counter = 0 101 | # position must be reached for one second to allow 102 | # overshoots/oscillations before stopping control loop 103 | wait_after_reach_sp = 1 / self.sampling_time 104 | 105 | # start time of the control loop 106 | start_time = time.time() 107 | 108 | list_ticks = [] 109 | 110 | #logger.debug("Starting while loop") 111 | 112 | while True: 113 | # DEBUGGING OPTION: 114 | # printing runtime of loop , see end of while true loop 115 | # start_time_each_loop = time.time() 116 | 117 | angle = self.get_angle() 118 | 119 | # # Position Control 120 | # Er = SP - PV 121 | error_p = target_angle - angle 122 | 123 | # Deadband-Filter to remove ocillating forwards and backwards after reaching set-point 124 | if error_p <= 5 and error_p >= -5: 125 | error_p = 0 126 | 127 | # I-Part 128 | sum_error_p += error_p 129 | # try needed, because Ki_p can be zero 130 | try: 131 | # limit I-Part to -1 and 1 132 | sum_error_p = max(min(1 / self.Ki_p, sum_error_p), -1 / self.Ki_p) 133 | except ZeroDivisionError: 134 | pass 135 | 136 | # POSITION PID-Controller 137 | output_p = ( 138 | self.Kp_p * error_p 139 | + self.Ki_p * self.sampling_time * sum_error_p 140 | + self.Kd_p / self.sampling_time * (error_p - error_p_old) 141 | ) 142 | # limit output of position control to speed range 143 | output_p = max(min(1, output_p), -1) 144 | 145 | error_p_old = error_p 146 | 147 | # # Speed Control 148 | # full speed forward and backward = +-650 ticks/s 149 | output_p_con = 650 * output_p 150 | 151 | try: 152 | # convert range output_p from -1 to 1 to ticks/s 153 | ticks = (angle - prev_angle) / self.sampling_time 154 | except NameError: 155 | prev_angle = angle 156 | continue 157 | 158 | # ticks per second (ticks/s), calculated from a moving median window with 5 values 159 | list_ticks.append(ticks) 160 | list_ticks = list_ticks[-5:] 161 | ticks = statistics.median(list_ticks) 162 | 163 | # Er = SP - PV 164 | error_s = output_p_con - ticks 165 | 166 | # I-Part 167 | sum_error_s += error_s 168 | # limit I-Part to -1 and 1 169 | 170 | # try needed, because Ki_s can be zero 171 | try: 172 | sum_error_s = max(min(650 / self.Ki_s, sum_error_s), -650 / self.Ki_s) 173 | except ZeroDivisionError: 174 | pass 175 | 176 | # SPEED PID-Controller 177 | output_s = ( 178 | self.Kp_s * error_s 179 | + self.Ki_s * self.sampling_time * sum_error_s 180 | + self.Kd_s / self.sampling_time * (error_s - error_s_old) 181 | ) 182 | 183 | error_s_old = error_s 184 | 185 | # convert range output_s fom ticks/s to -1 to 1 186 | output_s_con = output_s / 650 187 | self._set_speed(output_s_con) 188 | 189 | prev_angle = angle 190 | 191 | if error_p == 0: 192 | reached_sp_counter += 1 193 | 194 | if reached_sp_counter >= wait_after_reach_sp: 195 | self._set_speed(0.0) 196 | self.stop_event.set() 197 | #logger.debug("At position") 198 | elif time.time() - start_time >= TIMEOUT_PERIOD: 199 | self._set_speed(0.0) 200 | self.stop_event.set() 201 | #logger.debug("Timed out, position not reached") 202 | 203 | if self.stop_event.is_set(): 204 | #logger.debug("Thread closing") 205 | break 206 | 207 | # Pause control loop for chosen sample time 208 | time.sleep( 209 | self.sampling_time - ((time.time() - start_time) % self.sampling_time) 210 | ) 211 | 212 | # angular position in units full circle 213 | def get_angle(self): 214 | 215 | try: 216 | angle = ( 217 | (self.feedback.read() - self.dcMin) 218 | * self.unitsFC 219 | / (self.dcMax - self.dcMin + 1) 220 | ) 221 | except TypeError: 222 | raise TypeError("Could not calculate angle. Servo may not be connected.") 223 | 224 | angle = max(min((self.unitsFC - 1), angle), 0) 225 | 226 | return angle 227 | 228 | def update_setpoint(self, target_setpoint): 229 | self.stop_event.set() 230 | 231 | target_setpoint = float(target_setpoint) 232 | target_setpoint = max(min(target_setpoint, 100), 0) 233 | self.target_setpoint = target_setpoint 234 | angle_range = MAX_SET_POINT_ANGLE - MIN_SET_POINT_ANGLE 235 | target_angle = (target_setpoint * 0.01 * angle_range) + MIN_SET_POINT_ANGLE 236 | 237 | try: # Handle no thread running 238 | self.thread.join(timeout=1) 239 | if self.thread.is_alive(): 240 | raise RuntimeError("Knob thread failed to quit") 241 | except AttributeError: 242 | logger.debug("No thread to join") 243 | 244 | #logger.debug("Initialising worker with target_angle %s " % (target_angle)) 245 | self.stop_event.clear() 246 | self.thread = Thread(target=self._worker, args=(target_angle,), daemon=True) 247 | self.thread.start() 248 | 249 | def get_setpoint(self): 250 | 251 | return self.target_setpoint 252 | 253 | def get_achieved(self): 254 | 255 | window = [] 256 | 257 | for _ in range(5): 258 | achieved_angle = self.get_angle() 259 | time.sleep(0.001) 260 | window.append(achieved_angle) 261 | 262 | # Calculate centered average to eliminate noise spikes 263 | achieved_angle = (sum(window) - max(window) - min(window)) / (len(window) - 2) 264 | angle_range = MAX_SET_POINT_ANGLE - MIN_SET_POINT_ANGLE 265 | normalised = (100 * (achieved_angle - MIN_SET_POINT_ANGLE)) / angle_range 266 | 267 | return 100 - round(normalised) 268 | 269 | def quit(self): 270 | self.update_setpoint(0) 271 | time.sleep(1) 272 | self._set_speed(0) 273 | self.stop_event.set() 274 | self.thread.join(timeout=1) 275 | if self.thread.is_alive(): 276 | logger.error("Knob thread failed to quit") 277 | -------------------------------------------------------------------------------- /launcher.py: -------------------------------------------------------------------------------- 1 | import pigpio 2 | from time import sleep, time 3 | from os import system 4 | from requests import post 5 | import logging 6 | import socket 7 | 8 | """Launch OnionBot software from the big red button""" 9 | 10 | FORMAT = " %(levelname)-8s %(name)s %(process)d %(message)s" 11 | logging.basicConfig(format=FORMAT, level=logging.INFO) 12 | logger = logging.getLogger("launcher") 13 | 14 | pi = pigpio.pi() 15 | 16 | PIN = 21 17 | 18 | pi.set_mode(PIN, pigpio.INPUT) # GPIO 4 as input 19 | pi.set_pull_up_down(PIN, pigpio.PUD_UP) 20 | pi.set_glitch_filter(PIN, 100) 21 | 22 | 23 | timer = time() 24 | 25 | testIP = "8.8.8.8" 26 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 27 | s.connect((testIP, 0)) 28 | ip_address = s.getsockname()[0] 29 | 30 | 31 | def released_callback(gpio, level, tick): 32 | logger.debug("Reset button released") 33 | 34 | global timer 35 | time_elapsed = time() - timer 36 | logger.debug("Time elapsed: %0.2f" % (time_elapsed)) 37 | 38 | global ip_address 39 | if 0.01 < time_elapsed <= 1.5: 40 | logger.info("Calling shutdown over API") 41 | try: 42 | post("http://" + ip_address + ":5000/", data={"action": "quit"}) 43 | except: 44 | logger.info("API is not/no longer alive") 45 | 46 | elif 0.5 < time_elapsed <= 5: 47 | system("pkill -f API.py;") # If all else fails... 48 | sleep(1) 49 | logger.info("Starting Onionbot Software") 50 | system(". ~/onionbot/runonion &") 51 | 52 | elif 5 < time_elapsed <= 10: 53 | logger.info("Restarting Raspberry Pi") 54 | sleep(1) 55 | system("sudo reboot now") 56 | 57 | global released 58 | released.cancel() 59 | 60 | global pressed 61 | pressed = pi.callback(PIN, pigpio.FALLING_EDGE, pressed_callback) 62 | 63 | 64 | def pressed_callback(gpio, level, tick): 65 | logger.debug("Reset button pressed") 66 | 67 | global timer 68 | timer = time() 69 | 70 | global pressed 71 | pressed.cancel() 72 | 73 | global released 74 | released = pi.callback(PIN, pigpio.RISING_EDGE, released_callback) 75 | 76 | 77 | pressed = pi.callback(PIN, pigpio.FALLING_EDGE, pressed_callback) 78 | 79 | logger.info("Onionbot launcher is ready.") 80 | logger.info("Hold red button for 1s to start") 81 | logger.info("Point control panel to IP: %s" % (ip_address)) 82 | 83 | while True: 84 | try: 85 | sleep(0.1) 86 | except KeyboardInterrupt: 87 | logger.info("Launcher quit succesfully") 88 | break 89 | -------------------------------------------------------------------------------- /lib_para_360_servo.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import statistics 3 | import time 4 | 5 | import pigpio 6 | 7 | #https://www.parallax.com/sites/default/files/downloads/900-00360-Feedback-360-HS-Servo-v1.1.pdf 8 | #http://gpiozero.readthedocs.io/en/stable/remote_gpio.html 9 | #https://gpiozero.readthedocs.io/en/stable/recipes.html#pin-numbering 10 | 11 | class write_pwm: 12 | """ 13 | Stears a Parallax Feedback 360° High-Speed Servo `360_data_sheet`_ . 14 | 15 | This class stears a Parallax Feedback 360° High-Speed Servo. Out of the speed range, 16 | defined by ``min_speed`` and ``max_speed``, and the range of the pulsewidth, defined 17 | by ``min_pw`` and ``max_pw``, the class allows setting the servo speed and automatically 18 | calculates the appropriate pulsewidth for the chosen speed value. 19 | 20 | .. note:: 21 | ``min_pw`` and ``max_pw`` might needed to be interchanged, depending on if ``min_pw`` is 22 | moving the servo max_forward/clockwise or max_backwards/counter-clockwise, 23 | see methods :meth:`max_forward` and :meth:`max_backward`. 24 | :meth:`max_forward` -> ``min_pw`` should let the servo rotate clockwise. 25 | 26 | .. warning:: 27 | Be carefull with setting the min and max pulsewidth! Test carefully ``min_pw`` and ``max_pw`` 28 | before setting them. Wrong values can damage the servo, see set_servo_pulsewidth_ !!! 29 | 30 | :param pigpio.pi pi: 31 | Instance of a pigpio.pi() object. 32 | :param int gpio: 33 | GPIO identified by their Broadcom number, see elinux.org_ . 34 | To this GPIO the control wire of the servo has to be connected. 35 | :param int min_pw: 36 | Min pulsewidth, see **Warning**, carefully test the value before! 37 | **Default:** 1280, taken from the data sheet `360_data_sheet`_ . 38 | :param int max_pw: 39 | Max pulsewidth, see **Warning**, carefully test the value before! 40 | **Default:** 1720, taken from the data sheet `360_data_sheet`_ . 41 | :param int min_speed: 42 | Min speed which the servo is able to move. **Default:** -1. 43 | :param int max_speed: 44 | Max speed which the servo is able to move. **Default:** 1. 45 | 46 | .. _elinux.org: https://elinux.org/RPi_Low-level_peripherals#Model_A.2B.2C_B.2B_and_B2 47 | .. _set_servo_pulsewidth: http://abyz.me.uk/rpi/pigpio/python.html#set_servo_pulsewidth 48 | .. _`360_data_sheet`: https://www.parallax.com/sites/default/files/downloads/900-00360-Feedback-360-HS-Servo-v1.1.pdf 49 | """ 50 | 51 | def __init__(self, pi, gpio, min_pw = 1280, max_pw = 1720, min_speed = -1, max_speed = 1): 52 | 53 | self.pi = pi 54 | self.gpio = gpio 55 | self.min_pw = min_pw 56 | self.max_pw = max_pw 57 | self.min_speed = min_speed 58 | self.max_speed = max_speed 59 | #calculate slope for calculating the pulse width 60 | self.slope = (self.min_pw - ((self.min_pw + self.max_pw)/2)) / self.max_speed 61 | #calculate y-offset for calculating the pulse width 62 | self.offset = (self.min_pw + self.max_pw)/2 63 | 64 | def set_pw(self, pulse_width): 65 | """ 66 | Sets pulsewidth of the PWM. 67 | 68 | This method allows setting the pulsewidth of the PWM directly. This can be used to 69 | test which ``min_pw`` and ``max_pw`` are appropriate. For this the ``min_pw`` and ``max_pw`` 70 | are needed to be set very small and very big, so that they do not limit the set pulsewidth. 71 | Normally they are used to protect the servo by limiting the pulsewidth to a certain range. 72 | 73 | .. warning:: 74 | Be carefull with setting the min and max pulsewidth! Test carefully ``min_pw`` and ``max_pw`` 75 | before setting them. Wrong values can damage the servo, see set_servo_pulsewidth_ !!! 76 | 77 | :param int,float pulsewidth: 78 | Pulsewidth of the PWM signal. Will be limited to ``min_pw`` and ``max_pw``. 79 | 80 | .. _set_servo_pulsewidth: http://abyz.me.uk/rpi/pigpio/python.html#set_servo_pulsewidth 81 | """ 82 | 83 | pulse_width = max(min(self.max_pw, pulse_width), self.min_pw) 84 | 85 | self.pi.set_servo_pulsewidth(user_gpio = self.gpio, pulsewidth = pulse_width) 86 | 87 | def calc_pw(self, speed): 88 | 89 | pulse_width = self.slope * speed + self.offset 90 | 91 | return pulse_width 92 | 93 | def set_speed(self, speed): 94 | """ 95 | Sets speed of the servo. 96 | 97 | This method sets the servos rotation speed. The speed range is defined by 98 | by ``min_speed`` and ``max_speed`` . 99 | 100 | :param int,float speed: 101 | Should be between ``min_speed`` and ``max_speed`` , 102 | otherwise the value will be limited to those values. 103 | """ 104 | 105 | speed = max(min(self.max_speed, speed), self.min_speed) 106 | 107 | calculated_pw = self.calc_pw(speed = speed) 108 | self.set_pw(pulse_width = calculated_pw) 109 | 110 | def stop(self): 111 | """ 112 | Sets the speed of the servo to 0. 113 | """ 114 | 115 | pulse_width = (self.min_pw+self.max_pw)/2 116 | self.set_pw(pulse_width = pulse_width) 117 | 118 | def max_backward(self): 119 | """ 120 | Sets the speed of the servo to -1, so ``min_speed`` (max backwards, 121 | counter-clockwise) 122 | """ 123 | 124 | self.set_pw(self.max_pw) 125 | 126 | def max_forward(self): 127 | """ 128 | Sets the speed of the servo to 1, so ``max_speed`` (max forward, 129 | clockwise) 130 | """ 131 | 132 | self.set_pw(self.min_pw) 133 | 134 | class read_pwm: 135 | """ 136 | Reads position of a Parallax Feedback 360° High-Speed Servo `360_data_sheet`_ . 137 | 138 | This class reads the position of a Parallax Feedback 360° High-Speed Servo. At the 139 | moment, the period for a 910 Hz signal is hardcoded. 140 | 141 | :param pigpio.pi pi: 142 | Instance of a pigpio.pi() object. 143 | :param int gpio: 144 | GPIO identified by their Broadcom number, see elinux.org_ . 145 | To this GPIO the feedback wire of the servo has to be connected. 146 | 147 | .. todo:: 148 | Enable the class to be able to handle different signals, not just 910 Hz. 149 | 150 | .. _elinux.org: https://elinux.org/RPi_Low-level_peripherals#Model_A.2B.2C_B.2B_and_B2 151 | .. _`360_data_sheet`: https://www.parallax.com/sites/default/files/downloads/900-00360-Feedback-360-HS-Servo-v1.1.pdf 152 | """ 153 | 154 | def __init__(self, pi, gpio): 155 | 156 | self.pi = pi 157 | self.gpio = gpio 158 | self.period = 1/910*1000000 159 | self.tick_high = None 160 | self.duty_cycle = None 161 | self.duty_scale = 1000 162 | 163 | #http://abyz.me.uk/rpi/pigpio/python.html#set_mode 164 | self.pi.set_mode(gpio=self.gpio, mode=pigpio.INPUT) 165 | #http://abyz.me.uk/rpi/pigpio/python.html#callback 166 | self.cb = self.pi.callback(user_gpio=self.gpio, edge=pigpio.EITHER_EDGE, func=self.cbf) 167 | 168 | #calculate the duty cycle 169 | def cbf(self, gpio, level, tick): 170 | 171 | #change to low (a falling edge) 172 | if level == 0: 173 | #if first edge is a falling one the following code will fail 174 | #a try first time is faster than an if-statement every time 175 | try: 176 | #http://abyz.me.uk/rpi/pigpio/python.html#callback 177 | # tick 32 bit The number of microseconds since boot 178 | # WARNING: this wraps around from 179 | # 4294967295 to 0 roughly every 72 minutes 180 | #Tested: This is handled by the tickDiff function internally, if t1 (earlier tick) 181 | #is smaller than t2 (later tick), which could happen every 72 min. The result will 182 | #not be a negative value, the real difference will be properly calculated. 183 | self.duty_cycle = self.duty_scale*pigpio.tickDiff(t1=self.tick_high, t2=tick)/self.period 184 | 185 | except Exception: 186 | pass 187 | 188 | #change to high (a rising edge) 189 | elif level == 1: 190 | 191 | self.tick_high = tick 192 | 193 | def read(self): 194 | """ 195 | Returns the recent measured duty cycle. 196 | 197 | This method returns the recent measured duty cycle. 198 | 199 | :return: Recent measured duty cycle 200 | :rtype: float 201 | """ 202 | 203 | return self.duty_cycle 204 | 205 | def cancel(self): 206 | """ 207 | Cancel the started callback function. 208 | 209 | This method cancels the started callback function if initializing an object. 210 | As written in the pigpio callback_ documentation, the callback function may be cancelled 211 | by calling the cancel function after the created instance is not needed anymore. 212 | 213 | .. _callback: http://abyz.me.uk/rpi/pigpio/python.html#callback 214 | """ 215 | 216 | self.cb.cancel() 217 | 218 | class calibrate_pwm: 219 | """ 220 | Calibrates a Parallax Feedback 360° High-Speed Servo with the help of the :class:`read_pwm` class. 221 | 222 | This class helps to find out the min and max duty cycle of the feedback signal of a 223 | servo. This values ( ``dcMin`` / ``dcMax`` ) are then needed in :ref:`lib_motion` 224 | to have a more precise measurement of the position. The experience has shown that each 225 | servo has slightly different min/max duty cycle values, different than the once 226 | provided in the data sheet 360_data_sheet_ . Values smaller and bigger than the 227 | printed out once as "duty_cycle_min/duty_cycle_max" are outliers and should 228 | therefore not be considered. This can be seen in the printouts of smallest/biggest 229 | 250 values. There are sometimes a few outliers. Compare the printouts of different 230 | runs to get a feeling for it. 231 | 232 | .. note:: 233 | The robot wheels must be able to rotate free in the air for calibration. 234 | Rotating forward or backward might sometimes give slightly 235 | different results for min/max duty cycle. Choose the smallest value and the 236 | biggest value out of the forward and backward runs. Do both directions three 237 | times for each wheel, with speed = 0.2 and -0.2. Then 238 | chose the values. The speed has to be set manually, see :ref:`Examples`. 239 | 240 | :param pigpio.pi pi: 241 | Instance of a pigpio.pi() object. 242 | :param int gpio: 243 | GPIO identified by their Broadcom number, see elinux.org_ . 244 | To this GPIO the feedback wire of the servo has to be connected. 245 | :param int,float measurement_time: 246 | Time in seconds for how long duty cycle values will be collected, so for how long the 247 | measurement will be made. **Default:** 120. 248 | :returns: Printouts of different measurements 249 | 250 | At the moment, the period for a 910 Hz signal is hardcoded, as in :meth:`read_pwm` . 251 | 252 | .. todo:: 253 | Enable the class to be able to handle different signals, not just 910 Hz. 254 | 255 | .. _elinux.org: https://elinux.org/RPi_Low-level_peripherals#Model_A.2B.2C_B.2B_and_B2 256 | .. _`360_data_sheet`: https://www.parallax.com/sites/default/files/downloads/900-00360-Feedback-360-HS-Servo-v1.1.pdf 257 | """ 258 | 259 | def __init__(self, pi, gpio, measurement_time = 120): 260 | 261 | self.pi = pi 262 | self.gpio = gpio 263 | self.period = 1/910*1000000 264 | self.tick_high = None 265 | self.duty_cycle = None 266 | self.duty_scale = 1000 267 | self.list_duty_cycles = [] 268 | self.duty_cycle_min = None 269 | self.duty_cycle_max = None 270 | 271 | #http://abyz.me.uk/rpi/pigpio/python.html#set_mode 272 | self.pi.set_mode(gpio=self.gpio, mode=pigpio.INPUT) 273 | 274 | #http://abyz.me.uk/rpi/pigpio/python.html#callback 275 | self.cb = self.pi.callback(user_gpio=self.gpio, edge=pigpio.EITHER_EDGE, func=self.cbf) 276 | 277 | print('{}{}{}'.format('Starting measurements for: ', measurement_time, ' seconds.')) 278 | print('----------------------------------------------------------') 279 | time.sleep(measurement_time) 280 | 281 | #stop callback before sorting list to avoid getting added new elements unintended 282 | #http://abyz.me.uk/rpi/pigpio/python.html#callback 283 | self.cb.cancel() 284 | time.sleep(1) 285 | 286 | self.list_duty_cycles = sorted(self.list_duty_cycles) 287 | 288 | #some analyzis of the dc values 289 | sorted_set = list(sorted(set(self.list_duty_cycles))) 290 | print('{} {}'.format('Ascending sorted distinct duty cycle values:', sorted_set)) 291 | print('----------------------------------------------------------') 292 | differences_list = [sorted_set[i+1]-sorted_set[i] for i in range(len(sorted_set)-1)] 293 | rounded_differences_list = [round(differences_list[i],2) for i in range(len(differences_list)-1)] 294 | counted_sorted_list = collections.Counter(rounded_differences_list) 295 | print('{} {}'.format('Ascending counted, sorted and rounded distinct differences between duty cycle values:',counted_sorted_list)) 296 | print('----------------------------------------------------------') 297 | 298 | #Median and median_high/median_low are chosen, because the biggest 299 | #and smallest values are needed, and not an avarage of the smallest 300 | #and biggest values of the selection. 301 | #https://docs.python.org/3/library/statistics.html#statistics.median 302 | print('{} {}'.format('Smallest 250 values:', self.list_duty_cycles[:250])) 303 | self.duty_cycle_min = statistics.median_high(self.list_duty_cycles[:20]) 304 | print('----------------------------------------------------------') 305 | print('{} {}'.format('Biggest 250 values:',self.list_duty_cycles[-250:])) 306 | self.duty_cycle_max = statistics.median_low(self.list_duty_cycles[-20:]) 307 | print('----------------------------------------------------------') 308 | 309 | print('duty_cycle_min:', round(self.duty_cycle_min,2)) 310 | 311 | print('duty_cycle_max:', round(self.duty_cycle_max,2)) 312 | 313 | def cbf(self, gpio, level, tick): 314 | 315 | #change to low (a falling edge) 316 | if level == 0: 317 | #if first edge is a falling one the following code will not work 318 | #a try first time is faster than an if-statement every time 319 | try: 320 | self.duty_cycle = self.duty_scale*pigpio.tickDiff(t1=self.tick_high, t2=tick)/self.period 321 | self.list_duty_cycles.append(self.duty_cycle) 322 | 323 | except Exception: 324 | pass 325 | 326 | #change to high (a rising edge) 327 | elif level == 1: 328 | 329 | self.tick_high = tick 330 | 331 | def cancel(self): 332 | 333 | self.cb.cancel() 334 | 335 | if __name__ == "__main__": 336 | 337 | #just continue 338 | 339 | pass 340 | ''' 341 | pi = pigpio.pi() 342 | gpio = 14 343 | #obj = read_pwm(pi, gpio) 344 | 345 | x = calibrate_pwm(pi, gpio) 346 | 347 | 348 | while True: 349 | 350 | 351 | time.sleep(0.1) 352 | 353 | print(obj.read()) 354 | ''' 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from threading import Thread, Event 2 | from time import sleep 3 | 4 | from thermal_camera import ThermalCamera 5 | from camera import Camera 6 | from cloud import Cloud 7 | from classification import Classify 8 | from control import Control 9 | from data import Data 10 | from config import Settings, Labels 11 | 12 | from datetime import datetime 13 | from json import dumps 14 | import logging 15 | 16 | # Fix logging faliure issue 17 | for handler in logging.root.handlers[:]: 18 | logging.root.removeHandler(handler) 19 | 20 | # Initialise custom logging format 21 | FORMAT = "%(relativeCreated)6d %(levelname)-8s %(name)s %(process)d %(message)s" 22 | logging.basicConfig(format=FORMAT, level=logging.INFO) 23 | logger = logging.getLogger(__name__) 24 | 25 | settings = Settings() 26 | labels = Labels() 27 | camera = Camera() 28 | thermal = ThermalCamera() 29 | cloud = Cloud() 30 | classify = Classify() 31 | data = Data() 32 | control = Control() 33 | 34 | 35 | class OnionBot(object): 36 | """Main OnionBot script""" 37 | 38 | def __init__(self): 39 | 40 | self.quit_event = Event() 41 | 42 | # Launch multiprocessing threads 43 | logger.info("Launching worker threads...") 44 | camera.launch() 45 | thermal.launch() 46 | cloud.launch_camera() 47 | cloud.launch_thermal() 48 | classify.launch() 49 | control.launch() 50 | 51 | self.latest_meta = " " 52 | self.session_ID = None 53 | self.label = None 54 | 55 | logger.info("OnionBot is ready") 56 | 57 | def run(self): 58 | """Start logging""" 59 | 60 | def _worker(): 61 | 62 | measurement_ID = 0 63 | file_data = None 64 | meta = None 65 | 66 | while True: 67 | 68 | # Get time stamp 69 | timer = datetime.now() 70 | 71 | # Get update on key information 72 | measurement_ID += 1 73 | label = self.label 74 | session_ID = self.session_ID 75 | 76 | # Generate file_data for logs 77 | queued_file_data = data.generate_file_data( 78 | session_ID, timer, measurement_ID, label 79 | ) 80 | 81 | # Generate metadata for frontend 82 | queued_meta = data.generate_meta( 83 | session_ID=session_ID, 84 | timer=timer, 85 | measurement_ID=measurement_ID, 86 | label=label, 87 | file_data=queued_file_data, 88 | thermal_data=thermal.data, 89 | control_data=control.data, 90 | classification_data=classify.database, 91 | ) 92 | 93 | # Start sensor capture 94 | camera.start(queued_file_data["camera_file"]) 95 | thermal.start(queued_file_data["thermal_file"]) 96 | 97 | # While taking a picture, process previous data in meantime 98 | if file_data: 99 | 100 | cloud.start_camera(file_data["camera_file"]) 101 | cloud.start_thermal(file_data["thermal_file"]) 102 | classify.start(file_data["camera_file"]) 103 | 104 | # Wait for all meantime processes to finish 105 | cloud.join_camera() 106 | cloud.join_thermal() 107 | classify.join() 108 | 109 | # Push meta information to file level for API access 110 | self.labels_csv_filepath = file_data["label_file"] 111 | self.latest_meta = dumps(meta) 112 | 113 | # Wait for queued image captures to finish, refresh control data 114 | thermal.join() 115 | camera.join() 116 | control.refresh(thermal.data["temperature"]) 117 | 118 | # Log to console 119 | if meta is not None: 120 | attributes = meta["attributes"] 121 | logger.info( 122 | "Logged %s | session_ID %s | Label %s | Interval %0.2f | Temperature %s | PID enabled: %s | PID components: %0.1f, %0.1f, %0.1f " 123 | % ( 124 | attributes["measurement_ID"], 125 | attributes["session_ID"], 126 | attributes["label"], 127 | attributes["interval"], 128 | attributes["temperature"], 129 | attributes["pid_enabled"], 130 | attributes["p_component"], 131 | attributes["i_component"], 132 | attributes["d_component"], 133 | ) 134 | ) 135 | 136 | # Move queue forward one place 137 | file_data = queued_file_data 138 | meta = queued_meta 139 | 140 | # Add delay until ready for next loop 141 | frame_interval = float(settings.get_setting("frame_interval")) 142 | while True: 143 | if (datetime.now() - timer).total_seconds() > frame_interval: 144 | break 145 | elif self.quit_event.is_set(): 146 | break 147 | sleep(0.01) 148 | 149 | # Check quit flag 150 | if self.quit_event.is_set(): 151 | logger.debug("Quitting main thread...") 152 | break 153 | 154 | # Start thread 155 | logger.info("Starting main script") 156 | self.thread = Thread(target=_worker, daemon=True) 157 | self.thread.start() 158 | 159 | def start(self, session_ID): 160 | data.start_session(session_ID) 161 | self.session_ID = session_ID 162 | 163 | def stop(self): 164 | """Stop logging""" 165 | self.session_ID = None 166 | labels = self.labels_csv_filepath 167 | cloud.start_camera( 168 | labels 169 | ) # Use cloud uploader camera thread to upload label file 170 | cloud.join_camera() 171 | return cloud.get_public_path(labels) 172 | 173 | def get_latest_meta(self): 174 | """Returns cloud filepath of latest meta.json - includes path location of images""" 175 | return self.latest_meta 176 | 177 | def get_thermal_history(self): 178 | """Returns last 300 temperature readings""" 179 | return self.thermal_history 180 | 181 | def set_label(self, string): 182 | """Command to change current label - for building training datasets""" 183 | self.label = string 184 | 185 | def set_no_label(self): 186 | """Command to set label to None type""" 187 | self.label = None 188 | 189 | def set_classifiers(self, string): 190 | """Command to change current classifier for predictions""" 191 | classify.set_classifiers(string) 192 | 193 | def set_fixed_setpoint(self, value): 194 | """Command to change fixed setpoint""" 195 | control.update_fixed_setpoint(value) 196 | 197 | def set_temperature_target(self, value): 198 | """Command to change temperature target""" 199 | control.update_temperature_target(value) 200 | 201 | def set_temperature_hold(self): 202 | """Command to hold current temperature""" 203 | control.hold_temperature() 204 | 205 | def set_hob_off(self): 206 | """Command to turn hob off""" 207 | control.hob_off() 208 | 209 | def set_frame_interval(self, value): 210 | """Command to change camera target refresh rate""" 211 | settings.set_setting("frame_interval", value) 212 | 213 | def get_all_labels(self): 214 | """Returns available image labels for training""" 215 | return labels.get_labels() 216 | 217 | def get_all_classifiers(self): 218 | """Returns available models for prediction""" 219 | return classify.get_classifiers() 220 | 221 | def set_pid_enabled(self, enabled): 222 | """Command to start PID controller""" 223 | control.set_pid_enabled(enabled) 224 | 225 | def set_p_coefficient(self, coefficient): 226 | """Command to set PID P coeffient""" 227 | control.set_p_coefficient(coefficient) 228 | 229 | def set_i_coefficient(self, coefficient): 230 | """Command to set PID I coeffient""" 231 | control.set_i_coefficient(coefficient) 232 | 233 | def set_d_coefficient(self, coefficient): 234 | """Command to set PID D coeffient""" 235 | control.set_d_coefficient(coefficient) 236 | 237 | def set_pid_reset(self): 238 | """Command to reset PID components to 0 (not to be confused with coefficients)""" 239 | control.set_pid_reset() 240 | 241 | def quit(self): 242 | logger.info("Raising exit flag") 243 | self.quit_event.set() 244 | self.thread.join() 245 | logger.info("Main module quit") 246 | control.quit() 247 | logger.info("Control module quit") 248 | camera.quit() 249 | logger.info("Camera module quit") 250 | thermal.quit() 251 | logger.info("Thermal module quit") 252 | cloud.quit() 253 | logger.info("Cloud module quit") 254 | classify.quit() 255 | logger.info("Classifier module quit") 256 | logger.info("Quit process complete") 257 | -------------------------------------------------------------------------------- /models/README.md: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /models/boilover.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onionbot/OnionBot-API/5ecb608a7f674bc20597261a8380cd3470dd9650/models/boilover.tflite -------------------------------------------------------------------------------- /models/boilover.txt: -------------------------------------------------------------------------------- 1 | not_boiling_over 2 | boiling_over 3 | -------------------------------------------------------------------------------- /models/pan_on_off.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onionbot/OnionBot-API/5ecb608a7f674bc20597261a8380cd3470dd9650/models/pan_on_off.tflite -------------------------------------------------------------------------------- /models/pan_on_off.txt: -------------------------------------------------------------------------------- 1 | pan_off 2 | pan_on 3 | -------------------------------------------------------------------------------- /models/pasta.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onionbot/OnionBot-API/5ecb608a7f674bc20597261a8380cd3470dd9650/models/pasta.tflite -------------------------------------------------------------------------------- /models/pasta.txt: -------------------------------------------------------------------------------- 1 | add_pasta 2 | empty_pan 3 | water_boiling 4 | add_water 5 | -------------------------------------------------------------------------------- /models/sauce.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onionbot/OnionBot-API/5ecb608a7f674bc20597261a8380cd3470dd9650/models/sauce.tflite -------------------------------------------------------------------------------- /models/sauce.txt: -------------------------------------------------------------------------------- 1 | add_onions 2 | add_tomatoes 3 | add_oil 4 | empty_pan 5 | onions_cooked 6 | add_puree 7 | -------------------------------------------------------------------------------- /models/stirring.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onionbot/OnionBot-API/5ecb608a7f674bc20597261a8380cd3470dd9650/models/stirring.tflite -------------------------------------------------------------------------------- /models/stirring.txt: -------------------------------------------------------------------------------- 1 | stirring 2 | not_stirring 3 | -------------------------------------------------------------------------------- /models/tflite-boiling_1_thermal_20200111031542-2020-01-11T18_45_13.068Z_dict.txt: -------------------------------------------------------------------------------- 1 | not_boiling 2 | boiling 3 | -------------------------------------------------------------------------------- /models/tflite-boiling_1_thermal_20200111031542-2020-01-11T18_45_13.068Z_model.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onionbot/OnionBot-API/5ecb608a7f674bc20597261a8380cd3470dd9650/models/tflite-boiling_1_thermal_20200111031542-2020-01-11T18_45_13.068Z_model.tflite -------------------------------------------------------------------------------- /models/tflite-boiling_1_thermal_20200111031542-2020-01-11T18_45_13.068Z_tflite_metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "batchSize": 1, 3 | "imageChannels": 3, 4 | "imageHeight": 224, 5 | "imageWidth": 224, 6 | "inferenceType": "QUANTIZED_UINT8", 7 | "inputTensor": "image", 8 | "inputType": "QUANTIZED_UINT8", 9 | "outputTensor": "scores", 10 | "supportedTfVersions": [ 11 | "1.10", 12 | "1.11", 13 | "1.12", 14 | "1.13" 15 | ] 16 | } -------------------------------------------------------------------------------- /models/tflite-boiling_water_1_20200111094256-2020-01-11T11_51_24.886Z_dict.txt: -------------------------------------------------------------------------------- 1 | boiling 2 | not_boiling 3 | -------------------------------------------------------------------------------- /models/tflite-boiling_water_1_20200111094256-2020-01-11T11_51_24.886Z_model.tflite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onionbot/OnionBot-API/5ecb608a7f674bc20597261a8380cd3470dd9650/models/tflite-boiling_water_1_20200111094256-2020-01-11T11_51_24.886Z_model.tflite -------------------------------------------------------------------------------- /models/tflite-boiling_water_1_20200111094256-2020-01-11T11_51_24.886Z_tflite_metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "batchSize": 1, 3 | "imageChannels": 3, 4 | "imageHeight": 224, 5 | "imageWidth": 224, 6 | "inferenceType": "QUANTIZED_UINT8", 7 | "inputTensor": "image", 8 | "inputType": "QUANTIZED_UINT8", 9 | "outputTensor": "scores", 10 | "supportedTfVersions": [ 11 | "1.10", 12 | "1.11", 13 | "1.12", 14 | "1.13" 15 | ] 16 | } -------------------------------------------------------------------------------- /pid.py: -------------------------------------------------------------------------------- 1 | # Adaptation of https://pypi.org/project/simple-pid/ simple_pid controller 2 | 3 | import time 4 | import warnings 5 | 6 | 7 | def _clamp(value, limits): 8 | lower, upper = limits 9 | if value is None: 10 | return None 11 | elif upper is not None and value > upper: 12 | return upper 13 | elif lower is not None and value < lower: 14 | return lower 15 | return value 16 | 17 | 18 | try: 19 | # get monotonic time to ensure that time deltas are always positive 20 | _current_time = time.monotonic 21 | except AttributeError: 22 | # time.monotonic() not available (using python < 3.3), fallback to time.time() 23 | _current_time = time.time 24 | warnings.warn( 25 | "time.monotonic() not available in python < 3.3, using time.time() as fallback" 26 | ) 27 | 28 | 29 | class PID(object): 30 | """A simple PID controller.""" 31 | 32 | def __init__( 33 | self, 34 | Kp=1.0, 35 | Ki=0.0, 36 | Kd=0.0, 37 | setpoint=0, 38 | sample_time=0.01, 39 | output_limits=(None, None), 40 | is_enabled=True, 41 | proportional_on_measurement=True, 42 | ): 43 | """ 44 | Initialize a new PID controller. 45 | :param Kp: The value for the proportional gain Kp 46 | :param Ki: The value for the integral gain Ki 47 | :param Kd: The value for the derivative gain Kd 48 | :param setpoint: The initial setpoint that the PID will try to achieve 49 | :param sample_time: The time in seconds which the controller should wait before generating 50 | a new output value. The PID works best when it is constantly called (eg. during a 51 | loop), but with a sample time set so that the time difference between each update is 52 | (close to) constant. If set to None, the PID will compute a new output value every time 53 | it is called. 54 | :param output_limits: The initial output limits to use, given as an iterable with 2 55 | elements, for example: (lower, upper). The output will never go below the lower limit 56 | or above the upper limit. Either of the limits can also be set to None to have no limit 57 | in that direction. Setting output limits also avoids integral windup, since the 58 | integral term will never be allowed to grow outside of the limits. 59 | :param is_enabled: Whether the controller should be enabled (auto mode) or not (manual mode) 60 | :param proportional_on_measurement: Whether the proportional term should be calculated on 61 | the input directly rather than on the error (which is the traditional way). Using 62 | proportional-on-measurement avoids overshoot for some types of systems. 63 | """ 64 | self.Kp, self.Ki, self.Kd = Kp, Ki, Kd 65 | self.setpoint = setpoint 66 | self.sample_time = sample_time 67 | 68 | self._min_output, self._max_output = output_limits 69 | self._is_enabled = is_enabled 70 | self.proportional_on_measurement = proportional_on_measurement 71 | 72 | self.reset() 73 | 74 | def __call__(self, input_, dt=None): 75 | """ 76 | Update the PID controller. 77 | Call the PID controller with *input_* and calculate and return a control output if 78 | sample_time seconds has passed since the last update. If no new output is calculated, 79 | return the previous output instead (or None if no value has been calculated yet). 80 | :param dt: If set, uses this value for timestep instead of real time. This can be used in 81 | simulations when simulation time is different from real time. 82 | """ 83 | if not self.is_enabled: 84 | return self._last_output 85 | 86 | now = _current_time() 87 | if dt is None: 88 | dt = now - self._last_time if now - self._last_time else 1e-16 89 | elif dt <= 0: 90 | raise ValueError( 91 | "dt has nonpositive value {}. Must be positive.".format(dt) 92 | ) 93 | 94 | if ( 95 | self.sample_time is not None 96 | and dt < self.sample_time 97 | and self._last_output is not None 98 | ): 99 | # only update every sample_time seconds 100 | return self._last_output 101 | 102 | # compute error terms 103 | error = self.setpoint - input_ 104 | d_input = input_ - ( 105 | self._last_input if self._last_input is not None else input_ 106 | ) 107 | 108 | # compute the proportional term 109 | if not self.proportional_on_measurement: 110 | # regular proportional-on-error, simply set the proportional term 111 | self._proportional = self.Kp * error 112 | else: 113 | # add the proportional error on measurement to error_sum 114 | self._proportional -= self.Kp * d_input 115 | 116 | # compute integral and derivative terms 117 | self._integral += self.Ki * error * dt 118 | self._integral = _clamp( 119 | self._integral, self.output_limits 120 | ) # avoid integral windup 121 | 122 | self._derivative = -self.Kd * d_input / dt 123 | 124 | # compute final output 125 | output = self._proportional + self._integral + self._derivative 126 | output = _clamp(output, self.output_limits) 127 | 128 | # keep track of state 129 | self._last_output = output 130 | self._last_input = input_ 131 | self._last_time = now 132 | 133 | return output 134 | 135 | def __repr__(self): 136 | return ( 137 | "{self.__class__.__name__}(" 138 | "Kp={self.Kp!r}, Ki={self.Ki!r}, Kd={self.Kd!r}, " 139 | "setpoint={self.setpoint!r}, sample_time={self.sample_time!r}, " 140 | "output_limits={self.output_limits!r}, is_enabled={self.is_enabled!r}, " 141 | "proportional_on_measurement={self.proportional_on_measurement!r}" 142 | ")" 143 | ).format(self=self) 144 | 145 | @property 146 | def components(self): 147 | """ 148 | The P-, I- and D-terms from the last computation as separate components as a tuple. Useful 149 | for visualizing what the controller is doing or when tuning hard-to-tune systems. 150 | """ 151 | return self._proportional, self._integral, self._derivative 152 | 153 | @property 154 | def coefficients(self): 155 | """The coefficients used by the controller as a tuple: (Kp, Ki, Kd).""" 156 | return self.Kp, self.Ki, self.Kd 157 | 158 | # @coefficients.setter 159 | def set_coefficients(self, Kp, Ki, Kd): 160 | """Set the PID coefficients.""" 161 | if Kp is not None: 162 | self.Kp = float(Kp) 163 | if Ki is not None: 164 | self.Ki = float(Ki) 165 | if Kd is not None: 166 | self.Kd = float(Kd) 167 | 168 | @property 169 | def is_enabled(self): 170 | """Whether the controller is currently enabled (in auto mode) or not.""" 171 | return self._is_enabled 172 | 173 | @is_enabled.setter 174 | def is_enabled(self, enabled): 175 | """Enable or disable the PID controller.""" 176 | self.set_is_enabled(enabled) 177 | 178 | def set_is_enabled(self, enabled, last_output=None): 179 | """ 180 | Enable or disable the PID controller, optionally setting the last output value. 181 | This is useful if some system has been manually controlled and if the PID should take over. 182 | In that case, pass the last output variable (the control variable) and it will be set as 183 | the starting I-term when the PID is set to auto mode. 184 | :param enabled: Whether auto mode should be enabled, True or False 185 | :param last_output: The last output, or the control variable, that the PID should start 186 | from when going from manual mode to auto mode 187 | """ 188 | if enabled and not self._is_enabled: 189 | # switching from manual mode to auto, reset 190 | self.reset() 191 | 192 | self._integral = last_output if last_output is not None else 0 193 | self._integral = _clamp(self._integral, self.output_limits) 194 | 195 | self._is_enabled = enabled 196 | 197 | @property 198 | def output_limits(self): 199 | """ 200 | The current output limits as a 2-tuple: (lower, upper). 201 | See also the *output_limts* parameter in :meth:`PID.__init__`. 202 | """ 203 | return self._min_output, self._max_output 204 | 205 | @output_limits.setter 206 | def output_limits(self, limits): 207 | """Set the output limits.""" 208 | if limits is None: 209 | self._min_output, self._max_output = None, None 210 | return 211 | 212 | min_output, max_output = limits 213 | 214 | if None not in limits and max_output < min_output: 215 | raise ValueError("lower limit must be less than upper limit") 216 | 217 | self._min_output = min_output 218 | self._max_output = max_output 219 | 220 | self._integral = _clamp(self._integral, self.output_limits) 221 | self._last_output = _clamp(self._last_output, self.output_limits) 222 | 223 | def reset(self): 224 | """ 225 | Reset the PID controller internals. 226 | This sets each term to 0 as well as clearing the integral, the last output and the last 227 | input (derivative calculation). 228 | """ 229 | self._proportional = 0 230 | self._integral = 0 231 | self._derivative = 0 232 | 233 | self._last_time = _current_time() 234 | self._last_output = None 235 | self._last_input = None 236 | -------------------------------------------------------------------------------- /runlauncher: -------------------------------------------------------------------------------- 1 | cat << "EOF" 2 | 3 | 4 | // 5 | \ \ // 6 | \ //// 7 | // // 8 | // // 9 | ///8// 10 | / 8 /\ \ 11 | / 8 .,, \ 12 | /8 o i \ 13 | /8 o i ; `\ 14 | /8' 8..o 1.. `.\ 15 | /#' o i '' x . i i `.\ 16 | I# i8 i 1' x `. i ii I 17 | I 8o i i 1o.` '. . i i i I 18 | I 88 ioi .8. . . i i i I 19 | I o888ooi WW. .o. i i I 20 | \888##ooo`W1 8 ' i ,'/ 21 | \####WWWoo W 8..' #,'/ 22 | \W88##oo8888### o./ 23 | \###j[\##//##}/ 24 | ++///--\//_ 25 | \\ \ \ \ \_ 26 | / / \ 27 | 28 | Starting ONIONBOT Launcher 29 | 30 | EOF 31 | sudo pkill -f launcher.py 32 | python3 ~/onionbot/launcher.py -------------------------------------------------------------------------------- /runonion: -------------------------------------------------------------------------------- 1 | echo " " 2 | echo " _/_/ _/ _/ _/_/_/ _/_/ _/ _/ _/_/_/ _/_/ _/_/_/_/_/ " 3 | echo " _/ _/ _/_/ _/ _/ _/ _/ _/_/ _/ _/ _/ _/ _/ _/ " 4 | echo " _/ _/ _/ _/ _/ _/ _/ _/ _/ _/ _/ _/_/_/ _/ _/ _/ " 5 | echo "_/ _/ _/ _/_/ _/ _/ _/ _/ _/_/ _/ _/ _/ _/ _/ " 6 | echo " _/_/ _/ _/ _/_/_/ _/_/ _/ _/ _/_/_/ _/_/ _/ " 7 | echo " " 8 | echo "COPYRIGHT BEN COBLEY 2020" 9 | echo "Starting ONIONBOT Software Version 0.2" 10 | echo " " 11 | sudo pkill -f API.py 12 | python3 ~/onionbot/API.py 13 | -------------------------------------------------------------------------------- /thermal_camera.py: -------------------------------------------------------------------------------- 1 | from threading import Thread, Event 2 | from queue import Queue, Empty 3 | 4 | import math 5 | from statistics import mean 6 | import time 7 | import board 8 | import busio 9 | from PIL import Image 10 | from collections import deque 11 | 12 | import adafruit_mlx90640 13 | import logging 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | CHESSBOARD_MAX_THRESHOLD = 300 19 | CHESSBOARD_MIN_THRESHOLD = 5 20 | 21 | TEMPERATURE_MULTIPLIER = 1.35 22 | TEMPERATURE_OFFSET = -6 23 | 24 | 25 | INTERPOLATE = 10 26 | 27 | MINTEMP = 20.0 # -40 #low range of the sensor (this will be black on the screen) 28 | MAXTEMP = 200.0 # previous 50 #max 300 #high range of the sensor (this will be white on the screen) 29 | SCALE = 25 30 | 31 | 32 | # the list of colors we can choose from 33 | heatmap = ( 34 | (0.0, (0, 0, 0)), 35 | (0.20, (0, 0, 0.5)), 36 | (0.40, (0, 0.5, 0)), 37 | (0.60, (0.5, 0, 0)), 38 | (0.80, (0.75, 0.75, 0)), 39 | (0.90, (1.0, 0.75, 0)), 40 | (1.00, (1.0, 1.0, 1.0)), 41 | ) 42 | 43 | # how many color values we can have 44 | COLORDEPTH = 1000 45 | 46 | colormap = [0] * COLORDEPTH 47 | 48 | 49 | class ThermalCamera(object): 50 | """Save image to file""" 51 | 52 | def __init__(self, i2c=None, visualise_on=False): 53 | 54 | self.quit_event = Event() 55 | 56 | logger.info("Initialising thermal camera...") 57 | 58 | if i2c is None: 59 | i2c = busio.I2C(board.SCL, board.SDA) 60 | 61 | mlx = adafruit_mlx90640.MLX90640(i2c) 62 | mlx.refresh_rate = adafruit_mlx90640.RefreshRate.REFRESH_32_HZ 63 | 64 | self.mlx = mlx 65 | 66 | self.file_queue = Queue(1) 67 | 68 | self.temperature = 0 69 | self.thermal_history = deque([0] * 120) 70 | 71 | self.data = { 72 | "temperature": None, 73 | "thermal_history": None, 74 | } 75 | 76 | def _constrain(self, val, min_val, max_val): 77 | return min(max_val, max(min_val, val)) 78 | 79 | def _map_value(self, x, in_min, in_max, out_min, out_max): 80 | return (x - in_min) * (out_max - out_min) / (in_max - in_min) + out_min 81 | 82 | def _gaussian(self, x, a, b, c, d=0): 83 | return a * math.exp(-((x - b) ** 2) / (2 * c ** 2)) + d 84 | 85 | def _gradient(self, x, width, cmap, spread=1): 86 | width = float(width) 87 | r = sum( 88 | [ 89 | self._gaussian(x, p[1][0], p[0] * width, width / (spread * len(cmap))) 90 | for p in cmap 91 | ] 92 | ) 93 | g = sum( 94 | [ 95 | self._gaussian(x, p[1][1], p[0] * width, width / (spread * len(cmap))) 96 | for p in cmap 97 | ] 98 | ) 99 | b = sum( 100 | [ 101 | self._gaussian(x, p[1][2], p[0] * width, width / (spread * len(cmap))) 102 | for p in cmap 103 | ] 104 | ) 105 | r = int(self._constrain(r * 255, 0, 255)) 106 | g = int(self._constrain(g * 255, 0, 255)) 107 | b = int(self._constrain(b * 255, 0, 255)) 108 | return r, g, b 109 | 110 | def _worker(self): 111 | def _value(frame): 112 | 113 | logger.debug("Proccessing numerical data") 114 | 115 | f = frame 116 | center_square = [ 117 | f[72], 118 | f[73], 119 | f[74], 120 | f[75], 121 | f[88], 122 | f[89], 123 | f[90], 124 | f[91], 125 | f[104], 126 | f[105], 127 | f[106], 128 | f[107], 129 | f[120], 130 | f[121], 131 | f[122], 132 | f[123], 133 | ] 134 | 135 | temperature = mean(center_square) 136 | temperature = (temperature * TEMPERATURE_MULTIPLIER) + TEMPERATURE_OFFSET 137 | temperature = "{:.1f}".format(temperature) 138 | 139 | self.temperature = temperature 140 | 141 | thermal_history = self.thermal_history 142 | thermal_history.append(temperature) 143 | thermal_history.popleft() 144 | self.thermal_history = thermal_history 145 | 146 | return thermal_history 147 | 148 | def _image(frame, file_path): 149 | 150 | logger.debug("Proccessing image data") 151 | 152 | for i in range(COLORDEPTH): 153 | colormap[i] = self._gradient(i, COLORDEPTH, heatmap) 154 | 155 | pixels = [0] * 768 156 | for i, pixel in enumerate(frame): 157 | coloridx = self._map_value(pixel, MINTEMP, MAXTEMP, 0, COLORDEPTH - 1) 158 | coloridx = int(self._constrain(coloridx, 0, COLORDEPTH - 1)) 159 | pixels[i] = colormap[coloridx] 160 | 161 | img = Image.new("RGB", (32, 24)) 162 | img.putdata(pixels) 163 | img = img.resize((24 * INTERPOLATE, 24 * INTERPOLATE), Image.BICUBIC) 164 | 165 | img = img.transpose(method=Image.ROTATE_90) 166 | img = img.transpose(method=Image.FLIP_LEFT_RIGHT) 167 | img.save(file_path) 168 | 169 | while True: 170 | 171 | try: # Timeout raises queue.Empty 172 | file_path = self.file_queue.get(block=True, timeout=0.1) 173 | 174 | logger.debug("Capturing frame") 175 | frame = [0] * 768 176 | stamp = time.monotonic() 177 | while True: 178 | try: 179 | self.mlx.getFrame(frame) 180 | except ValueError: # Handle ValueError in module 181 | logger.debug("Frame capture error, retrying [ValueError]") 182 | time.sleep(0.1) 183 | continue 184 | except RuntimeError: # Handle RuntimeError in module 185 | logger.debug("Frame capture error, retrying [RuntimeError]") 186 | time.sleep(0.1) 187 | continue 188 | except OSError: # Handle RuntimeError in module 189 | logger.info("Frame capture error, retrying [OSError]") 190 | time.sleep(0.1) 191 | continue 192 | 193 | if 0 in frame: # Handle chessboard error zeros 194 | logger.debug("Frame capture error, retrying [Zero Error]") 195 | time.sleep(0.1) 196 | continue 197 | 198 | if max(frame) > CHESSBOARD_MAX_THRESHOLD: 199 | logger.debug( 200 | "Frame capture error, retrying [Max Temp Error: %0.2f ]" 201 | % (max(frame)) 202 | ) 203 | time.sleep(0.1) 204 | continue 205 | elif min(frame) < CHESSBOARD_MIN_THRESHOLD: 206 | logger.debug( 207 | "Frame capture error, retrying [MIN Temp Error: %0.2f ]" 208 | % (min(frame)) 209 | ) 210 | time.sleep(0.1) 211 | continue 212 | 213 | break 214 | 215 | logger.debug("Read 2 frames in %0.3f s" % (time.monotonic() - stamp)) 216 | 217 | # Call numerical and graphical functions 218 | _value(frame) 219 | _image(frame, file_path) 220 | 221 | self.file_queue.task_done() 222 | 223 | except Empty: 224 | if self.quit_event.is_set(): 225 | logger.debug("Quitting thermal camera thread...") 226 | break 227 | 228 | def get_temperature(self): 229 | temperature = self.temperature 230 | logger.debug("self.temperature is %s " % (temperature)) 231 | 232 | return temperature 233 | 234 | def get_thermal_history(self): 235 | thermal_history = list(self.thermal_history) 236 | logger.debug("self.thermal_history is %s " % (thermal_history)) 237 | 238 | return thermal_history 239 | 240 | def start(self, file_path): 241 | logger.debug("Calling start") 242 | self.file_queue.put(file_path, block=True) 243 | 244 | def join(self): 245 | logger.debug("Calling join") 246 | self.file_queue.join() 247 | 248 | self.data = { 249 | "temperature": self.get_temperature(), 250 | "thermal_history": self.get_thermal_history(), 251 | } 252 | 253 | def launch(self): 254 | logger.debug("Initialising worker") 255 | self.thread = Thread(target=self._worker, daemon=True) 256 | self.thread.start() 257 | 258 | def quit(self): 259 | self.quit_event.set() 260 | self.thread.join() 261 | -------------------------------------------------------------------------------- /utils/README.md: -------------------------------------------------------------------------------- 1 | 2 | ### Utilities for testing, calibration and file renaming that are not run in the main script 3 | -------------------------------------------------------------------------------- /utils/calibration.py: -------------------------------------------------------------------------------- 1 | from thermal_camera import ThermalCamera 2 | from camera import Camera 3 | import PIL.Image 4 | import PIL.ImageEnhance 5 | from cloud import Cloud as cloud 6 | 7 | import datetime 8 | 9 | cloud = cloud() 10 | 11 | tc = ThermalCamera(visualise_on=False) 12 | nc = Camera() 13 | 14 | 15 | time = datetime.datetime.now() 16 | 17 | path1 = cloud.get_path("testsession", "nc", "jpg", time, "0") 18 | n = nc.capture(path1) 19 | #cloud.upload_from_filename(path1) 20 | 21 | 22 | path2 = cloud.get_path("testsession", "tc", "jpg", time, "0") 23 | tc.capture_frame() 24 | t = tc.save_latest_jpg(path2) 25 | #cloud.upload_from_filename(path2) 26 | 27 | timg = PIL.Image.open(t).convert("RGBA") 28 | nimg = PIL.Image.open(n).convert("RGBA") 29 | 30 | # optional lightness of watermark from 0.0 to 1.0 31 | #brightness = 0.5 32 | #watermark = PIL.ImageEnhance.Brightness(watermark).enhance(brightness) 33 | 34 | # apply the watermark 35 | some_xy_offset = (0, 0) #10, 20 36 | # the mask uses the transparency of the watermark (if it exists) 37 | #nimg.paste(timg, some_xy_offset, mask=timg) 38 | 39 | nimg = PIL.Image.blend(nimg, timg, 0.5) 40 | 41 | nimg.show() 42 | nimg.save('calibration_image.png') 43 | -------------------------------------------------------------------------------- /utils/create_automl_csv.py: -------------------------------------------------------------------------------- 1 | from os import walk 2 | import csv 3 | 4 | 5 | session_ID = "monday1" 6 | sensor = "camera" 7 | label = "not_boiling" 8 | path = f"/Users/bencobley/Downloads/{session_ID}/{sensor}/{label}" 9 | csv_file = f"/Users/bencobley/Downloads/{session_ID}/{sensor}_{label}.csv" 10 | 11 | 12 | files = [] 13 | directories = [] 14 | for (dirpath, dirnames, filenames) in walk(path): 15 | files.extend(filenames) 16 | break 17 | 18 | 19 | total_count = 0 20 | skip_count = 0 21 | added_count = 0 22 | 23 | with open(csv_file, mode="w") as label_file: 24 | file_writer = csv.writer( 25 | label_file, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL 26 | ) 27 | 28 | file_writer.writerow(["image_path[,label]"]) 29 | 30 | for filename in filenames: 31 | total_count += 1 32 | skip_count += 1 33 | if skip_count == 4: 34 | file_writer.writerow( 35 | [ 36 | str( 37 | f"gs://onionbucket/logs/{session_ID}/{sensor}/{label}/{filename},{label}" 38 | ) 39 | ] 40 | ) 41 | skip_count = 0 42 | added_count += 1 43 | 44 | 45 | print(f"Added {added_count} files to csv") 46 | -------------------------------------------------------------------------------- /utils/create_temp_time_series.py: -------------------------------------------------------------------------------- 1 | from os import walk 2 | import csv 3 | import json 4 | 5 | 6 | session_ID = "monday1_backup" 7 | path = F"/Users/bencobley/Downloads/{session_ID}/meta_flat" 8 | csv_file = F"/Users/bencobley/Downloads/{session_ID}/temperature_time_series.csv" 9 | 10 | 11 | files = [] 12 | directories = [] 13 | for (dirpath, dirnames, filenames) in walk(path): 14 | files.extend(filenames) 15 | break 16 | 17 | filenames = [file for file in filenames if file.endswith(".json")] 18 | 19 | with open(csv_file, mode='w') as temperature_file: 20 | file_writer = csv.writer(temperature_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) 21 | 22 | for filename in filenames: 23 | print (filename) 24 | with open(F"{path}/{filename}") as jsonfile: 25 | data = json.load(jsonfile) 26 | attributes = data['attributes'] 27 | file_writer.writerow([str(attributes['time_stamp']),str(attributes['active_label']),str((attributes['temperature'])) ]) 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /utils/live_classify.py: -------------------------------------------------------------------------------- 1 | # python3 2 | # 3 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """Example using TF Lite to classify objects with the Raspberry Pi camera.""" 17 | 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | from __future__ import print_function 21 | 22 | import argparse 23 | import io 24 | import time 25 | import numpy as np 26 | import picamera 27 | 28 | from PIL import Image 29 | from tflite_runtime.interpreter import Interpreter 30 | 31 | 32 | def load_labels(path): 33 | with open(path, 'r') as f: 34 | return {i: line.strip() for i, line in enumerate(f.readlines())} 35 | 36 | 37 | def set_input_tensor(interpreter, image): 38 | tensor_index = interpreter.get_input_details()[0]['index'] 39 | input_tensor = interpreter.tensor(tensor_index)()[0] 40 | input_tensor[:, :] = image 41 | 42 | 43 | def classify_image(interpreter, image, top_k=1): 44 | """Returns a sorted array of classification results.""" 45 | set_input_tensor(interpreter, image) 46 | interpreter.invoke() 47 | output_details = interpreter.get_output_details()[0] 48 | output = np.squeeze(interpreter.get_tensor(output_details['index'])) 49 | 50 | # If the model is quantized (uint8 data), then dequantize the results 51 | if output_details['dtype'] == np.uint8: 52 | scale, zero_point = output_details['quantization'] 53 | output = scale * (output - zero_point) 54 | 55 | ordered = np.argpartition(-output, top_k) 56 | return [(i, output[i]) for i in ordered[:top_k]] 57 | 58 | 59 | def main(): 60 | parser = argparse.ArgumentParser( 61 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 62 | parser.add_argument( 63 | '--model', help='File path of .tflite file.', required=True) 64 | parser.add_argument( 65 | '--labels', help='File path of labels file.', required=True) 66 | args = parser.parse_args() 67 | 68 | labels = load_labels(args.labels) 69 | 70 | interpreter = Interpreter(args.model) 71 | interpreter.allocate_tensors() 72 | _, height, width, _ = interpreter.get_input_details()[0]['shape'] 73 | 74 | with picamera.PiCamera(resolution=(640, 480), framerate=30) as camera: 75 | camera.start_preview() 76 | try: 77 | stream = io.BytesIO() 78 | for _ in camera.capture_continuous( 79 | stream, format='jpeg', use_video_port=True): 80 | stream.seek(0) 81 | image = Image.open(stream).convert('RGB').resize((width, height), 82 | Image.ANTIALIAS) 83 | start_time = time.time() 84 | results = classify_image(interpreter, image) 85 | elapsed_ms = (time.time() - start_time) * 1000 86 | label_id, prob = results[0] 87 | stream.seek(0) 88 | stream.truncate() 89 | camera.annotate_text = '%s %.2f\n%.1fms' % (labels[label_id], prob, 90 | elapsed_ms) 91 | finally: 92 | camera.stop_preview() 93 | 94 | 95 | if __name__ == '__main__': 96 | main() 97 | -------------------------------------------------------------------------------- /utils/print_angle.py: -------------------------------------------------------------------------------- 1 | from servo import Servo 2 | import time 3 | 4 | servo = Servo() 5 | 6 | while True: 7 | print(360 - float(servo.get_angle())) 8 | time.sleep(.1) 9 | -------------------------------------------------------------------------------- /utils/servo_calibration.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import pigpio 4 | 5 | import lib_para_360_servo 6 | 7 | #define GPIO for each servo to read from 8 | gpio_r_r = 14 9 | 10 | #define GPIO for each servo to write to 11 | gpio_r_w = 18 12 | 13 | pi = pigpio.pi() 14 | 15 | #### Calibrate servos, speed = 0.2 and -0.2 16 | #choose gpio_l_w/gpio_l_r (left wheel), or gpio_r_w/gpio_r_r 17 | #(right wheel) accordingly 18 | 19 | servo = lib_para_360_servo.write_pwm(pi = pi, gpio = gpio_r_w) 20 | 21 | #buffer time for initializing everything 22 | time.sleep(1) 23 | servo.set_speed(0.2) 24 | wheel = lib_para_360_servo.calibrate_pwm(pi = pi, gpio = gpio_r_r) 25 | servo.set_speed(0) 26 | #http://abyz.me.uk/rpi/pigpio/python.html#stop 27 | pi.stop() -------------------------------------------------------------------------------- /utils/servo_install.py: -------------------------------------------------------------------------------- 1 | from servo import Servo 2 | 3 | servo = Servo() 4 | 5 | _ = input("Remove servo assembly from hob body. Press enter to continue _") 6 | print("Aligning servo...") 7 | servo.rotate(350) 8 | print("Rotate hob knob anticlockwise to 0, then rotate forward aprrox 10 degrees") 9 | _ = input(" Press enter to continue _") 10 | _ = input("Reattatch servo assembly to hob. Press enter to finish _") 11 | -------------------------------------------------------------------------------- /utils/type_angle.py: -------------------------------------------------------------------------------- 1 | from servo import Servo 2 | import time 3 | 4 | servo = Servo() 5 | 6 | while True: 7 | x = float(input("Type angle: ")) 8 | servo.safe_rotate(x) 9 | time.sleep(.1) 10 | -------------------------------------------------------------------------------- /utils/type_setpoint.py: -------------------------------------------------------------------------------- 1 | from servo import Servo 2 | import time 3 | 4 | servo = Servo() 5 | 6 | while True: 7 | x = float(input("Type setpoint: ")) 8 | servo.update_setpoint(x) 9 | time.sleep(.1) 10 | --------------------------------------------------------------------------------