├── .gitignore ├── LICENSE ├── Makefile ├── README.rst ├── docs ├── .gitignore ├── Makefile ├── conf.py ├── design-notes.rst ├── functional-api.rst ├── images │ ├── solar-heater-state-machine.png │ └── solar-water-heater.png ├── index.rst ├── intro.rst ├── make.bat ├── micropython.rst ├── more-examples.rst ├── output_things.rst ├── ports.rst ├── reference.rst └── tutorial.rst ├── examples ├── esp8266 │ ├── lux_sensor_demo.py │ └── main_adxl345_upython.py ├── event_library_comparison │ ├── .gitignore │ ├── README.rst │ ├── asyncawait.py │ ├── event.py │ └── things.py ├── functional_api_example.py ├── kalman_model.py ├── multi_port_example.py ├── predix_example.py ├── rpi │ ├── README.rst │ ├── dist_lux_rpi.py │ ├── dist_lux_server.py │ └── lux_sensor_example.py ├── simple_csv_reader.py ├── solar_heater_scenario.py └── tutorial.py ├── micropython ├── README.rst ├── logger.py ├── mqtt_writer.py ├── neopixel_writer.py ├── sensors │ ├── README.rst │ ├── adc_esp8266.py │ ├── adxl345_upy.py │ ├── mcp9808.py │ └── tsl2591.py ├── tests │ ├── README.rst │ ├── network.py │ ├── runtests.sh │ ├── test_base.py │ ├── test_end_to_end.py │ ├── test_fast.py │ ├── test_scheduler.py │ ├── test_utils.py │ ├── ubinascii.py │ ├── ucollections.py │ ├── umqtt │ │ ├── __init__.py │ │ └── robust.py │ └── utime.py ├── thingflow.py └── wifi.py ├── setup.py ├── tests ├── .gitignore ├── README.rst ├── example_config_for_tests.py ├── runtests.sh ├── test_base.py ├── test_blocking_input_thing.py ├── test_blocking_output_thing.py ├── test_bokeh.py ├── test_csv_adapters.py ├── test_descheduling.py ├── test_external_event_stream.py ├── test_fatal_error_handling.py ├── test_fatal_error_in_private_loop.py ├── test_functional_api.py ├── test_influxdb.py ├── test_iterable_as_output_thing.py ├── test_linq.py ├── test_mqtt.py ├── test_mqtt_async.py ├── test_multiple_output_ports.py ├── test_pandas.py ├── test_postgres_adapters.py ├── test_predix.py ├── test_rpi_adapters.py ├── test_scheduler_cancel.py ├── test_solar_heater_scenario.py ├── test_tcp_stream.py ├── test_timeout.py ├── test_tracing.py ├── test_transducer.py └── utils.py └── thingflow ├── __init__.py ├── adapters ├── __init__.py ├── bokeh.py ├── csv.py ├── generic.py ├── influxdb.py ├── mqtt.py ├── mqtt_async.py ├── pandas.py ├── postgres.py ├── predix.py ├── rpi │ ├── __init__.py │ └── gpio.py └── tcpstreamer.py ├── base.py ├── filters ├── __init__.py ├── buffer.py ├── combinators.py ├── dispatch.py ├── first.py ├── json.py ├── map.py ├── never.py ├── output.py ├── scan.py ├── select.py ├── skip.py ├── some.py ├── take.py ├── timeout.py ├── transducer.py └── where.py ├── internal ├── __init__.py └── basic.py └── sensors ├── __init__.py └── rpi ├── __init__.py ├── adxl345_py3.py ├── arduino.py ├── gpio.py ├── lux_sensor.py └── sensor.py /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | MANIFEST 3 | dist/ 4 | build/ 5 | __pycache__ 6 | *.pyc 7 | thingflow.egg-info/ 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for antevents - just a simple wrapper over shell commands 2 | 3 | help: 4 | @echo "make targets are: dist tests clean help" 5 | 6 | dist: 7 | python3 setup.py sdist 8 | python3 setup.py bdist_wheel 9 | 10 | tests: 11 | cd tests; ./runtests.sh 12 | 13 | clean: 14 | rm -f MANIFEST 15 | rm -rf dist/ build/ 16 | rm -rf antevents.egg-info/ 17 | find . -name '*~' -delete 18 | rm -rf tests/*.err tests/*.out 19 | 20 | .PHONY: help dist tests clean 21 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | ThingFlow 3 | =========== 4 | 5 | ThingFlow is a (Python3) framework for building IoT event 6 | processing dataflows. [#]_ The goal of this framework is to support the 7 | creation of robust IoT systems from reusable components. These systems must 8 | account for noisy/missing sensor data, distributed computation, and the need for 9 | local (near the data source) processing. 10 | 11 | The source repository for ThingFlow-python is at https://github.com/mpi-sws-rse/thingflow-python. 12 | 13 | Introduction 14 | ============ 15 | The fundamental abstractions in ThingFlow are: 16 | 17 | 1. *sensors*, which provide a means 18 | to sample a changing value representing some quanity in the physical world, 19 | 2. *event streams*, which are push-based sequences of sensor data readings, and 20 | 3. *things*, which are reusable components to generate, transform, or consume the events on these streams. 21 | 22 | Things can have simple, stateless logic (e.g. filter events based 23 | on a predicate) or implement more complex, stateful algorithms, such as 24 | Kalman filters or machine learning. Using ThingFlow, you describe the flow of 25 | data through these things rather than programming low-level behaviors. 26 | 27 | Although ThingFlow presents a simple dataflow model to the user, internally it 28 | uses an event-driven programming model, building on 29 | Python's ``asyncio`` module. In addition to being a natural programming model for 30 | realtime sensor data, it reduces the potential resource consumption of Ant 31 | Events programs. The details of event scheduling are handled by the framework. 32 | Separate threads may be used on the "edges" of a dataflow, where elements 33 | frequently interact with external components that have blocking APIs. 34 | 35 | ThingFlow integrates with standard Python 36 | data analytics frameworks, including NumPy_, Pandas_, and scikit-learn_. This 37 | allows dataflows involving complex elements to be developed and refined offline 38 | and then deployed in an IoT environment using the same code base. 39 | 40 | We call the implementation described here "ThingFlow-Python", as it should be 41 | possible to port the ideas of ThingFlow to other languages. Currently, one such 42 | port exists: "ThingFlow-MicroPython". This is a port ThingFlow to MicroPython, 43 | a limited version of Python 3 that runs "bare metal" on embadded devices. The 44 | ThingFlow-MicroPython port is included in the ThingFlow-Python repository 45 | under the subdirector ``micropython``. It is documented in 46 | `a chapter `_ 47 | of the documentation. 48 | 49 | 50 | .. _NumPy: http://www.numpy.org/ 51 | .. _Pandas: http://pandas.pydata.org/ 52 | .. _scikit-learn: http://scikit-learn.org/stable/ 53 | 54 | Example 55 | ------- 56 | To give the flavor of ThingFlow, below is a short code snippet for the 57 | Raspberry Pi that reads a light sensor and then turns on an LED if the running 58 | average of the last five readings is greater than some threshold:: 59 | 60 | lux = SensorAsOutputThing(LuxSensor()) 61 | lux.map(lambda e: e.val).running_avg(5).map(lambda v: v > THRESHOLD)\ 62 | .GpioPinOut() 63 | scheduler.schedule_periodic(lux, 60.0) 64 | scheduler.run_forever() 65 | 66 | The first line instantiates a light sensor object and wraps it in an *output thing* 67 | to handle sampling and progagation of events. 68 | 69 | The next two lines 70 | create a pipeline of things to process the data from the sensor. We call things 71 | which have a single input and output *filters*, as they can be composed to process 72 | a stream of events. 73 | The ``map`` filter extracts the data value from the sensor event, the 74 | ``running_avg`` filter averages the last five values, and the next ``map`` filter converts 75 | the value to a a boolean based on the threshold. The ``GpioPinOut`` thing is 76 | an *adapter* to the outside world. It turns on the LED based on the value of 77 | its input boolean value. 78 | 79 | Finally, the last two lines of the example schedule the sensor to be sampled 80 | at a sixty second interval and then start the scheduler's main loop. 81 | 82 | Dependencies 83 | ------------ 84 | The ThingFlow proper is self-contained. You do not need any dependencies other 85 | than Python 3 (3.4 or later). Specific adapters and sensors may have additional 86 | dependencies (e.g. the MQTT adapters depend on MQTT client libraries). 87 | 88 | Documentation 89 | ------------- 90 | Documentation is hosted online at http://thingflow-python.readthedocs.io. 91 | 92 | The source tree for the documentation is in the ``docs`` subdirectory - it is 93 | built using `Sphinx `_. If you have Sphinx 94 | installed locally (along with the "Read the Docs" theme), you can also build it 95 | directly on your machine. 96 | 97 | 98 | .. [#] *ThingFlow* was originally known as *AntEvents*. 99 | 100 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | _build/ 2 | -------------------------------------------------------------------------------- /docs/images/solar-heater-state-machine.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpi-sws-rse/thingflow-python/c191a8fedac6a962994945830c872e957f929e29/docs/images/solar-heater-state-machine.png -------------------------------------------------------------------------------- /docs/images/solar-water-heater.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpi-sws-rse/thingflow-python/c191a8fedac6a962994945830c872e957f929e29/docs/images/solar-water-heater.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. ThingFlow-Python documentation master file, created by 2 | sphinx-quickstart on Mon Mar 27 14:14:59 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | ThingFlow-Python 7 | ================ 8 | ThingFlow is a (Python3) framework for building IoT event 9 | processing dataflows. [#]_ The goal of this framework is to support the 10 | creation of robust IoT systems from reusable components. These systems must 11 | account for noisy/missing sensor data, distributed computation, and the need for 12 | local (near the data source) processing. 13 | 14 | The source repository for ThingFlow-python is at https://github.com/mpi-sws-rse/thingflow-python. 15 | 16 | Sections 1 and 2 of this documentation cover how to get started. Sections 3 17 | through 5 cover more advanced topics. Section 6 provides some more code examples. 18 | Section 7 covers our port of ThingFlow to 19 | MicroPython on the ESP8266. Section 8 documents some design decisions made during 20 | the evolution of ThingFlow. Finally, Section 9 contains reference 21 | documentation for the full ThingFlow-python API (extracted from the docstrings). 22 | 23 | .. [#] *ThingFlow* was originally known as *AntEvents*. 24 | 25 | Contents: 26 | 27 | .. toctree:: 28 | :maxdepth: 2 29 | 30 | intro 31 | tutorial 32 | output_things 33 | ports 34 | functional-api 35 | more-examples 36 | micropython 37 | design-notes 38 | reference 39 | 40 | 41 | Indices and tables 42 | ================== 43 | 44 | * :ref:`genindex` 45 | * :ref:`modindex` 46 | * :ref:`search` 47 | 48 | -------------------------------------------------------------------------------- /docs/reference.rst: -------------------------------------------------------------------------------- 1 | .. _reference: 2 | 3 | 9. ThingFlow-Python API Reference 4 | ================================= 5 | 6 | .. automodule:: thingflow 7 | :members: 8 | 9 | thingflow.base 10 | -------------- 11 | 12 | .. automodule:: thingflow.base 13 | :members: 14 | 15 | 16 | thingflow.sensors 17 | ----------------- 18 | The sensors are not included in the auto-generated 19 | documentation, as importing the code requires external 20 | libraries (not possible for automated documentation generation). 21 | Here is a list of available sensor modules in the ThingFlow-Python 22 | distribution: 23 | 24 | * ``rpi.adxl345_py3`` - interface to the adxl345 accelerometer 25 | * ``rpi.arduino`` - interface an Arduino to the Raspberry Pi 26 | * ``rpi.gpio`` - read from the Raspberry Pi GPIO pins 27 | * ``lux_sensor`` - read from a TSL2591 lux sensor 28 | 29 | Please see the source code for more details on these sensors. 30 | 31 | thingflow.filters 32 | ----------------- 33 | 34 | .. automodule:: thingflow.filters 35 | :members: 36 | 37 | thingflow.filters.buffer 38 | ~~~~~~~~~~~~~~~~~~~~~~~~ 39 | .. automodule:: thingflow.filters.buffer 40 | :members: 41 | 42 | thingflow.filters.combinators 43 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 44 | .. automodule:: thingflow.filters.combinators 45 | :members: 46 | 47 | thingflow.filters.dispatch 48 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 49 | .. automodule:: thingflow.filters.dispatch 50 | :members: 51 | 52 | thingflow.filters.first 53 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 54 | .. automodule:: thingflow.filters.first 55 | :members: 56 | 57 | thingflow.filters.json 58 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 59 | .. automodule:: thingflow.filters.json 60 | :members: 61 | 62 | thingflow.filters.map 63 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 64 | .. automodule:: thingflow.filters.map 65 | :members: 66 | 67 | thingflow.filters.never 68 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 69 | .. automodule:: thingflow.filters.never 70 | :members: 71 | 72 | thingflow.filters.output 73 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 74 | .. automodule:: thingflow.filters.output 75 | :members: 76 | 77 | thingflow.filters.scan 78 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 79 | .. automodule:: thingflow.filters.scan 80 | :members: 81 | 82 | thingflow.filters.select 83 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 84 | .. automodule:: thingflow.filters.select 85 | :members: 86 | 87 | thingflow.filters.skip 88 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 89 | .. automodule:: thingflow.filters.skip 90 | :members: 91 | 92 | thingflow.filters.some 93 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 94 | .. automodule:: thingflow.filters.some 95 | :members: 96 | 97 | thingflow.filters.take 98 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 99 | .. automodule:: thingflow.filters.take 100 | :members: 101 | 102 | thingflow.filters.timeout 103 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 104 | .. automodule:: thingflow.filters.timeout 105 | :members: 106 | 107 | thingflow.filters.transducer 108 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 109 | .. automodule:: thingflow.filters.transducer 110 | :members: 111 | 112 | thingflow.filters.where 113 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 114 | .. automodule:: thingflow.filters.where 115 | :members: 116 | 117 | 118 | thingflow.adapters 119 | ------------------ 120 | .. automodule:: thingflow.adapters 121 | :members: 122 | 123 | thingflow.adapters.csv 124 | ~~~~~~~~~~~~~~~~~~~~~~~~ 125 | .. automodule:: thingflow.adapters.csv 126 | :members: 127 | 128 | thingflow.adapters.generic 129 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 130 | .. automodule:: thingflow.adapters.generic 131 | :members: 132 | 133 | Other Adapters 134 | ~~~~~~~~~~~~~~ 135 | Many adapters are not included in the auto-generated documentation, as 136 | importing the code requires external libraries (not possible for the 137 | auto document generation). Here is a list of additional adapters 138 | in the ThingFlow-Python distirbution: 139 | 140 | * ``bokeh`` - interface to the Bokeh visualization framework 141 | * ``influxdb`` - interface to the InfluxDb time series database 142 | * ``mqtt`` - interface to MQTT via ``paho.mqtt`` 143 | * ``mqtt_async`` - interface to MQTT via ``hbmqtt`` 144 | * ``pandas`` - convert ThingFlow events to Pandas ``Series`` data arrays 145 | * ``predix`` - send and query data with the GE Predix Time Series API 146 | * ``postgres`` - interface to the PostgreSQL database 147 | * ``rpi.gpio`` - output on the Raspberry Pi GPIO pins 148 | 149 | 150 | 151 | Please see the source code for more details 152 | on these adapters. 153 | -------------------------------------------------------------------------------- /examples/esp8266/lux_sensor_demo.py: -------------------------------------------------------------------------------- 1 | # Simple demo of reading the tsl2591 lux sensor from the 2 | # ESP8266 running micropython. 3 | 4 | from thingflow import * 5 | from tsl2591 import Tsl2591 6 | tsl = Tsl2591('lux-1') 7 | tsl.sample() 8 | sched = Scheduler() 9 | 10 | class Output: 11 | def on_next(self, x): 12 | print(x) 13 | def on_completed(): 14 | pass 15 | def on_error(self, e): 16 | pass 17 | 18 | sched.schedule_sensor(tsl, 2.0, Output()) 19 | sched.run_forever() 20 | -------------------------------------------------------------------------------- /examples/esp8266/main_adxl345_upython.py: -------------------------------------------------------------------------------- 1 | """For AntEvents MicroPython port, running on Feather Huzzah (ESP8266) 2 | 3 | Script used on the ESP8266 to collect ADXL345 (accelerometer) data and send it 4 | to an MQTT broker. 5 | 6 | To use it, the parameters in the file need to be set (WIFI_SID, WIFI_PW, BROKER) and 7 | the file renamed to 'main.py' which will be run when the ESP8266 is booted. Mpfshell 8 | is recommended for transferring MicroPython files to the ESP8266. 9 | 10 | """ 11 | from thingflow import Scheduler 12 | from adxl345_upy import ADXL345_upy 13 | from mqtt_writer import MQTTWriter 14 | from wifi import wifi_connect 15 | import os 16 | 17 | # Params to set 18 | WIFI_SID="" 19 | WIFI_PW="" 20 | SENSOR_ID="accelerometer" 21 | BROKER='' 22 | 23 | wifi_connect(WIFI_SID, WIFI_PW) 24 | sensor = ADXL345_upy(sensor_id=2) 25 | writer = MQTTWriter(SENSOR_ID, BROKER, 1883, 'sensor/1') 26 | sched = Scheduler() 27 | frequency = 5 #Hz 28 | sched.schedule_sensor(sensor, 1/frequency, writer) 29 | try: 30 | os.stat('stop-sampling.txt') 31 | print("Saw stop-sampling.txt, will skip rest") 32 | except: 33 | print("Starting sensor sampling") 34 | sched.run_forever() 35 | -------------------------------------------------------------------------------- /examples/event_library_comparison/.gitignore: -------------------------------------------------------------------------------- 1 | raw_data.csv 2 | -------------------------------------------------------------------------------- /examples/event_library_comparison/asyncawait.py: -------------------------------------------------------------------------------- 1 | """This version uses the async and await calls. 2 | """ 3 | from statistics import median 4 | import json 5 | import asyncio 6 | import random 7 | import time 8 | import hbmqtt.client 9 | 10 | from thingflow.base import SensorEvent 11 | 12 | URL = "mqtt://localhost:1883" 13 | 14 | class RandomSensor: 15 | def __init__(self, sensor_id, mean=100.0, stddev=20.0, stop_after_events=None): 16 | self.sensor_id = sensor_id 17 | self.mean = mean 18 | self.stddev = stddev 19 | self.stop_after_events = stop_after_events 20 | if stop_after_events is not None: 21 | def generator(): 22 | for i in range(stop_after_events): 23 | yield round(random.gauss(mean, stddev), 1) 24 | else: # go on forever 25 | def generator(): 26 | while True: 27 | yield round(random.gauss(mean, stddev), 1) 28 | self.generator = generator() 29 | 30 | def sample(self): 31 | return self.generator.__next__() 32 | 33 | def __repr__(self): 34 | if self.stop_after_events is None: 35 | return 'RandomSensor(%s, mean=%s, stddev=%s)' % \ 36 | (self.sensor_id, self.mean, self.stddev) 37 | else: 38 | return 'RandomSensor(%s, mean=%s, stddev=%s, stop_after_events=%s)' % \ 39 | (self.sensor_id, self.mean, self.stddev, self.stop_after_events) 40 | 41 | class PeriodicMedianTransducer: 42 | """Emit an event once every ``period`` input events. 43 | The value is the median of the inputs received since the last 44 | emission. 45 | """ 46 | def __init__(self, period=5): 47 | self.period = period 48 | self.samples = [None for i in range(period)] 49 | self.events_since_last = 0 50 | self.last_event = None # this is used in emitting the last event 51 | 52 | def step(self, v): 53 | self.samples[self.events_since_last] = v.val 54 | self.events_since_last += 1 55 | if self.events_since_last==self.period: 56 | val = median(self.samples) 57 | event = SensorEvent(sensor_id=v.sensor_id, ts=v.ts, val=val) 58 | self.events_since_last = 0 59 | return event 60 | else: 61 | self.last_event = v # save in case we complete before completing a period 62 | return None 63 | 64 | def complete(self): 65 | if self.events_since_last>0: 66 | # if we have some partial state, we emit one final event that 67 | # averages whatever we saw since the last emission. 68 | return SensorEvent(sensor_id=self.last_event.sensor_id, 69 | ts=self.last_event.ts, 70 | val=median(self.samples[0:self.events_since_last])) 71 | 72 | 73 | def csv_writer(evt): 74 | print("csv_writer(%s)" % repr(evt)) 75 | 76 | class MqttWriter: 77 | def __init__(self, url, topic, event_loop): 78 | self.url = url 79 | self.topic = topic 80 | self.client = hbmqtt.client.MQTTClient(loop=event_loop) 81 | self.connected = False 82 | 83 | def _to_message(self, msg): 84 | return bytes(json.dumps((msg.sensor_id, msg.ts, msg.val),), encoding='utf-8') 85 | 86 | async def send(self, msg): 87 | if not self.connected: 88 | print("attempting connection") 89 | await self.client.connect(self.url) 90 | self.connected = True 91 | print("connected") 92 | print("sending %s" % self._to_message(msg)) 93 | await self.client.publish(self.topic, self._to_message(msg)) 94 | 95 | async def disconnect(self): 96 | if self.connected: 97 | await self.client.disconnect() 98 | 99 | 100 | async def sample_and_process(sensor, mqtt_writer, xducer): 101 | try: 102 | sample = sensor.sample() 103 | except StopIteration: 104 | final_event = xducer.complete() 105 | if final_event: 106 | await mqtt_writer.send(final_event) 107 | print("disconnecting") 108 | await mqtt_writer.disconnect() 109 | return False 110 | event = SensorEvent(sensor_id=sensor.sensor_id, ts=time.time(), val=sample) 111 | csv_writer(event) 112 | median_event = xducer.step(event) 113 | if median_event: 114 | await mqtt_writer.send(median_event) 115 | return True 116 | 117 | 118 | 119 | sensor = RandomSensor('sensor-2', stop_after_events=12) 120 | transducer = PeriodicMedianTransducer(5) 121 | event_loop = asyncio.get_event_loop() 122 | writer = MqttWriter(URL, sensor.sensor_id, event_loop) 123 | 124 | def loop(): 125 | coro = sample_and_process(sensor, writer, transducer) 126 | task = event_loop.create_task(coro) 127 | def done_callback(f): 128 | exc = f.exception() 129 | if exc: 130 | raise exc 131 | elif f.result()==False: 132 | print("all done, no more callbacks to schedule") 133 | event_loop.stop() 134 | else: 135 | event_loop.call_later(0.5, loop) 136 | task.add_done_callback(done_callback) 137 | 138 | 139 | event_loop.call_soon(loop) 140 | event_loop.run_forever() 141 | 142 | 143 | print("that's all folks") 144 | -------------------------------------------------------------------------------- /examples/event_library_comparison/things.py: -------------------------------------------------------------------------------- 1 | """ 2 | Comparing ThingFlow to generic asyncio programming. 3 | 4 | This is the ThingFlow version. 5 | """ 6 | import asyncio 7 | import random 8 | from statistics import median 9 | from thingflow.base import InputThing, SensorEvent, Scheduler, SensorAsOutputThing 10 | from thingflow.filters.transducer import Transducer 11 | import thingflow.filters.combinators 12 | import thingflow.adapters.csv 13 | from thingflow.adapters.mqtt_async import mqtt_async_send 14 | import thingflow.filters.output 15 | 16 | URL = "mqtt://localhost:1883" 17 | 18 | class RandomSensor: 19 | def __init__(self, sensor_id, mean=100.0, stddev=20.0, stop_after_events=None): 20 | self.sensor_id = sensor_id 21 | self.mean = mean 22 | self.stddev = stddev 23 | self.stop_after_events = stop_after_events 24 | if stop_after_events is not None: 25 | def generator(): 26 | for i in range(stop_after_events): 27 | yield round(random.gauss(mean, stddev), 1) 28 | else: # go on forever 29 | def generator(): 30 | while True: 31 | yield round(random.gauss(mean, stddev), 1) 32 | self.generator = generator() 33 | 34 | def sample(self): 35 | return self.generator.__next__() 36 | 37 | def __repr__(self): 38 | if self.stop_after_events is None: 39 | return 'RandomSensor(%s, mean=%s, stddev=%s)' % \ 40 | (self.sensor_id, self.mean, self.stddev) 41 | else: 42 | return 'RandomSensor(%s, mean=%s, stddev=%s, stop_after_events=%s)' % \ 43 | (self.sensor_id, self.mean, self.stddev, self.stop_after_events) 44 | 45 | class PeriodicMedianTransducer(Transducer): 46 | """Emit an event once every ``period`` input events. 47 | The value is the median of the inputs received since the last 48 | emission. 49 | """ 50 | def __init__(self, period=5): 51 | self.period = period 52 | self.samples = [None for i in range(period)] 53 | self.events_since_last = 0 54 | self.last_event = None # this is used in emitting the last event 55 | 56 | def step(self, v): 57 | self.samples[self.events_since_last] = v.val 58 | self.events_since_last += 1 59 | if self.events_since_last==self.period: 60 | val = median(self.samples) 61 | event = SensorEvent(sensor_id=v.sensor_id, ts=v.ts, val=val) 62 | self.events_since_last = 0 63 | return event 64 | else: 65 | self.last_event = v # save in case we complete before completing a period 66 | return None 67 | 68 | def complete(self): 69 | if self.events_since_last>0: 70 | # if we have some partial state, we emit one final event that 71 | # averages whatever we saw since the last emission. 72 | return SensorEvent(sensor_id=self.last_event.sensor_id, 73 | ts=self.last_event.ts, 74 | val=median(self.samples[0:self.events_since_last])) 75 | 76 | SENSOR_ID = 'sensor-1' 77 | scheduler = Scheduler(asyncio.get_event_loop()) 78 | sensor = SensorAsOutputThing(RandomSensor(SENSOR_ID, mean=10, stddev=5, stop_after_events=12)) 79 | sensor.csv_writer('raw_data.csv').connect(lambda x: print("raw data: %s" % repr(x))) 80 | sensor.transduce(PeriodicMedianTransducer()).mqtt_async_send(URL, SENSOR_ID, scheduler).output() 81 | scheduler.schedule_periodic(sensor, 0.5) 82 | scheduler.run_forever() 83 | print("that's all folks") 84 | -------------------------------------------------------------------------------- /examples/functional_api_example.py: -------------------------------------------------------------------------------- 1 | """This is a demonstration of the fuctional API for filters. It is 2 | based on examples/rpi/lux_sensor_example.py. See docs/functional-api.rst 3 | for details. 4 | """ 5 | import asyncio 6 | import random 7 | random.seed() 8 | 9 | from thingflow.base import InputThing, Scheduler 10 | from thingflow.filters.output import output 11 | from thingflow.filters.select import map 12 | from thingflow.adapters.csv import csv_writer 13 | from thingflow.filters.combinators import passthrough 14 | 15 | class DummyLuxSensor: 16 | def __init__(self, sensor_id, mean=300, stddev=100, stop_after=5): 17 | """Rather than use the real RPI sensor here, we will just 18 | define one that generates random numbers. 19 | """ 20 | self.sensor_id = sensor_id 21 | self.mean = mean 22 | self.stddev = stddev 23 | self.events_left = stop_after 24 | 25 | def sample(self): 26 | if self.events_left>0: 27 | data = random.gauss(self.mean, self.stddev) 28 | self.events_left -= 1 29 | return data 30 | else: 31 | raise StopIteration 32 | 33 | def __repr__(self): 34 | return "DummyLuxSensor(%s, %s, %s)" % \ 35 | (self.sensor_id, self.mean, self.stddev) 36 | 37 | class DummyLed(InputThing): 38 | def on_next(seelf, x): 39 | if x: 40 | print("LED ON") 41 | else: 42 | print("LED OFF") 43 | 44 | def __repr__(self): 45 | return 'DummyLed' 46 | 47 | THRESHOLD = 300 48 | 49 | # Instantiate the sensor and use the functional API to build a flow 50 | lux = DummyLuxSensor("lux-1") 51 | scheduler = Scheduler(asyncio.get_event_loop()) 52 | scheduler.schedule_sensor(lux, 1.0, 53 | passthrough(output()), 54 | passthrough(csv_writer('/tmp/lux.csv')), 55 | map(lambda event:event.val > THRESHOLD), 56 | passthrough(lambda v: print('ON' if v else 'OFF')), 57 | DummyLed(), print_downstream=True) 58 | scheduler.run_forever() 59 | print("That's all folks") 60 | -------------------------------------------------------------------------------- /examples/kalman_model.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import numpy as np 3 | from sklearn import linear_model 4 | 5 | # For Kalman filtering 6 | from filterpy.kalman import KalmanFilter 7 | from filterpy.common import Q_discrete_white_noise 8 | 9 | from thingflow.base import OutputThing, InputThing, from_iterable, Scheduler 10 | 11 | class SGDLinearRegressionModel(OutputThing, InputThing): 12 | def __init__(self): 13 | OutputThing.__init__(self, ports=['train', 'observe', 'predict']) 14 | self.clf = linear_model.SGDRegressor() 15 | 16 | def on_train_next(self, x): 17 | print("On train next called") 18 | # training input: train the model 19 | xx = np.asarray(x[0]) 20 | yy = np.asarray(x[1]) 21 | self.clf.partial_fit(xx, yy) 22 | 23 | def on_train_error(self, x): 24 | print("On train error called") 25 | self.on_error(x) 26 | 27 | def on_train_completed(self): 28 | print("On train completed called") 29 | self.on_completed() 30 | 31 | def on_observe_next(self, x): 32 | print("On observe next called") 33 | xx = np.asarray(x) 34 | p = self.clf.predict(xx) 35 | self._dispatch_next(p, port='predict') 36 | def on_observe_error(self, x): 37 | self.on_error(x) 38 | def on_observe_completed(self): 39 | self.on_completed() 40 | 41 | class FilterModel(OutputThing, InputThing): 42 | def __init__(self, filter): 43 | OutputThing.__init__(self, ports=['observe', 'predict']) 44 | self.filter = filter 45 | 46 | def on_observe_next(self, measurement): 47 | print("On observerain next called") 48 | # training input: train the model 49 | self.filter.predict() 50 | self.filter.update(measurement) 51 | self._dispatch_next(self.filter.x, port='predict') 52 | 53 | def on_observe_error(self, x): 54 | print("On observe error called") 55 | self.on_error(x) 56 | 57 | def on_observe_completed(self): 58 | print("On observe completed called") 59 | self.on_completed() 60 | 61 | 62 | class KalmanFilterModel(FilterModel): 63 | """Implements Kalman filters using filterpy. 64 | x' = Fx + Bu + w 65 | y = H x + ww 66 | """ 67 | def __init__(self, dim_state, dim_control, dim_measurement, 68 | initial_state_mean, initial_state_covariance, 69 | matrix_F, matrix_B, 70 | process_noise_Q, 71 | matrix_H, measurement_noise_R): 72 | filter = KalmanFilter(dim_x=dim_state, dim_u=dim_control, dim_z=dim_measurement) 73 | filter.x = initial_state_mean 74 | filter.P = initial_state_covariance 75 | 76 | filter.Q = process_noise_Q 77 | 78 | filter.F = matrix_F 79 | filter.B = matrix_B 80 | filter.H = matrix_H 81 | 82 | filter.R = measurement_noise_R # covariance matrix 83 | super().__init__(filter) 84 | 85 | 86 | 87 | def main_linear(): 88 | obs_stream = from_iterable(iter([ [ [ [1.0, 1.0], [2.0, 2.0]], [1.0, 2.0] ], [ [ [6.0, 6.0], [9.0, 9.0]], [6.0, 9.0] ] ])) 89 | pred_stream = from_iterable(iter([ [3.0, 3.0] ])) 90 | model = SGDLinearRegressionModel() 91 | obs_stream.connect(model, port_mapping=('default', 'train')) 92 | obs_stream.connect(print) 93 | 94 | pred_stream.connect(model, port_mapping=('default', 'observe')) 95 | model.connect(print, port_mapping=('predict', 'default')) 96 | scheduler = Scheduler(asyncio.get_event_loop()) 97 | scheduler.schedule_periodic(obs_stream, 1) 98 | scheduler.schedule_periodic(pred_stream, 5) 99 | scheduler.run_forever() 100 | 101 | def main_kalman(): 102 | dim_x = 2 103 | dim_u = 1 104 | dim_z = 1 105 | initial_state_mean = np.array([ [1.0] , [0.0] ]) 106 | initial_state_covariance = 1000 * np.eye(dim_x) 107 | 108 | F = np.array([ [ 1., 1.], [0., 1.] ]) 109 | B = np.zeros((2, 1) ) 110 | Q = Q_discrete_white_noise(dim=2, dt=0.1, var=0.13) 111 | 112 | H = np.array([[1.,0.]]) 113 | R = 5 * np.eye(1) 114 | 115 | model = KalmanFilterModel(dim_x, dim_u, dim_z, initial_state_mean, initial_state_covariance, 116 | F, B, Q, H, R) 117 | measurement_stream = from_iterable(iter([ [ 1.0 ], [0.0] ])) 118 | # measurement_stream = from_iterable(iter([ np.array([ [1.0, 1.0] ]) ])) 119 | measurement_stream.connect(model, port_mapping=('default', 'observe')) 120 | model.connect(print, port_mapping=('predict', 'default')) 121 | 122 | scheduler = Scheduler(asyncio.get_event_loop()) 123 | scheduler.schedule_periodic(measurement_stream, 1) 124 | scheduler.run_forever() 125 | 126 | 127 | def main(): 128 | main_kalman() 129 | 130 | if __name__ == '__main__': 131 | main() 132 | -------------------------------------------------------------------------------- /examples/multi_port_example.py: -------------------------------------------------------------------------------- 1 | """A simple example to demonstrate a thing with multiple ports. The 2 | thing samples values from a sensor and sends them on different output 3 | ports depending on the divisibility of the value. See docs/ports.rst 4 | for a more detailed explanation. 5 | """ 6 | import random 7 | import asyncio 8 | from thingflow.base import OutputThing, InputThing, Scheduler,\ 9 | SensorAsOutputThing 10 | 11 | class MultiPortOutputThing(OutputThing, InputThing): 12 | def __init__(self, previous_in_chain): 13 | super().__init__(ports=['divisible_by_two', 'divisible_by_three', 14 | 'other']) 15 | # connect to the previous filter 16 | self.disconnect_from_upstream = previous_in_chain.connect(self) 17 | 18 | def on_next(self, x): 19 | val = int(round(x.val)) 20 | if (val%2)==0: 21 | self._dispatch_next(val, port='divisible_by_two') 22 | if (val%3)==0: 23 | self._dispatch_next(val, port='divisible_by_three') 24 | if (val%3)!=0 and (val%2)!=0: 25 | self._dispatch_next(val, port='other') 26 | 27 | def on_completed(self): 28 | self._dispatch_completed(port='divisible_by_two') 29 | self._dispatch_completed(port='divisible_by_three') 30 | self._dispatch_completed(port='other') 31 | 32 | def on_error(self, e): 33 | self._dispatch_error(e, port='divisible_by_two') 34 | self._dispatch_error(e, port='divisible_by_three') 35 | self._dispatch_error(e, port='other') 36 | 37 | def __repr__(self): 38 | return 'MultiPortOutputThing()' 39 | 40 | class RandomSensor: 41 | def __init__(self, sensor_id, mean=100.0, stddev=20.0, stop_after_events=None): 42 | self.sensor_id = sensor_id 43 | self.mean = mean 44 | self.stddev = stddev 45 | self.stop_after_events = stop_after_events 46 | if stop_after_events is not None: 47 | def generator(): 48 | for i in range(stop_after_events): 49 | yield random.gauss(mean, stddev) 50 | else: # go on forever 51 | def generator(): 52 | while True: 53 | yield random.gauss(mean, stddev) 54 | self.generator = generator() 55 | 56 | def sample(self): 57 | return self.generator.__next__() 58 | 59 | def __repr__(self): 60 | if self.stop_after_events is None: 61 | return 'RandomSensor(%s, mean=%s, stddev=%s)' % \ 62 | (self.sensor_id, self.mean, self.stddev) 63 | else: 64 | return 'RandomSensor(%s, mean=%s, stddev=%s, stop_after_events=%s)' % \ 65 | (self.sensor_id, self.mean, self.stddev, self.stop_after_events) 66 | 67 | 68 | scheduler = Scheduler(asyncio.get_event_loop()) 69 | sensor = SensorAsOutputThing(RandomSensor(1, mean=10, stddev=5, 70 | stop_after_events=10)) 71 | mtthing = MultiPortOutputThing(sensor) 72 | mtthing.connect(lambda v: print("even: %s" % v), 73 | port_mapping=('divisible_by_two', 'default')) 74 | mtthing.connect(lambda v: print("divisible by three: %s" % v), 75 | port_mapping=('divisible_by_three', 'default')) 76 | mtthing.connect(lambda v: print("not divisible: %s" % v), 77 | port_mapping=('other', 'default')) 78 | mtthing.print_downstream() 79 | scheduler.schedule_recurring(sensor) 80 | scheduler.run_forever() 81 | 82 | -------------------------------------------------------------------------------- /examples/predix_example.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example of Predix Time Series APIs 3 | 4 | This sends a sequence of data points via PredixWriter and then 5 | queries then back via PredixReader. 6 | 7 | For hints on configuring Predix, see 8 | https://github.com/jfischer/ge-predix-python-timeseries-example 9 | """ 10 | import sys 11 | import argparse 12 | import logging 13 | import time 14 | import asyncio 15 | import random 16 | 17 | random.seed() 18 | 19 | from thingflow.base import Scheduler, SensorAsOutputThing 20 | from thingflow.adapters.predix import PredixWriter, PredixReader, EventExtractor 21 | 22 | logger = logging.getLogger() 23 | logger.setLevel(logging.DEBUG) 24 | logger.addHandler(logging.StreamHandler()) 25 | logging.basicConfig(level=logging.DEBUG) 26 | 27 | 28 | TEST_SENSOR1 = 'test-sensor-1' 29 | 30 | class TestSensor: 31 | """Generate a random value for the specified number of samples. 32 | """ 33 | def __init__(self, sensor_id, num_events): 34 | self.sensor_id = sensor_id 35 | self.events_remaining = num_events 36 | 37 | def sample(self): 38 | if self.events_remaining>0: 39 | self.events_remaining -= 1 40 | return random.gauss(100, 5) 41 | else: 42 | raise StopIteration 43 | 44 | @staticmethod 45 | def output_thing(sensor_id, num_events): 46 | return SensorAsOutputThing(TestSensor(sensor_id, num_events)) 47 | 48 | def run(args, token): 49 | sensor1 = TestSensor.output_thing(TEST_SENSOR1, 5) 50 | writer = PredixWriter(args.ingest_url, args.predix_zone_id, token, 51 | extractor=EventExtractor(attributes={'test':True}), 52 | batch_size=3) 53 | sensor1.connect(writer) 54 | sensor1.connect(print) # also print the event 55 | scheduler = Scheduler(asyncio.get_event_loop()) 56 | scheduler.schedule_periodic(sensor1, 0.5) 57 | 58 | start_time = time.time() 59 | scheduler.run_forever() 60 | 61 | print("Reading back events") 62 | reader1 = PredixReader(args.query_url, args.predix_zone_id, token, 63 | TEST_SENSOR1, 64 | start_time=start_time, 65 | one_shot=True) 66 | reader1.connect(print) 67 | scheduler.schedule_recurring(reader1) 68 | scheduler.run_forever() 69 | 70 | 71 | INGEST_URL = 'wss://gateway-predix-data-services.run.aws-usw02-pr.ice.predix.io/v1/stream/messages' 72 | QUERY_URL='https://time-series-store-predix.run.aws-usw02-pr.ice.predix.io/v1/datapoints' 73 | 74 | 75 | DESCRIPTION = \ 76 | """Example of Predix Time Series adapters for ThingFlow. 77 | Sends a sequence of data points via PredixWriter and then 78 | queries them back via PredixReader.""" 79 | 80 | def main(argv=sys.argv[1:]): 81 | parser = argparse.ArgumentParser(description=DESCRIPTION) 82 | parser.add_argument("--ingest-url", default=INGEST_URL, 83 | help="Websockets URL for ingest. Default is for Western US datacenter") 84 | parser.add_argument("--query-url", default=QUERY_URL, 85 | help="HTTPS URL for query. Default is for Western US datacenter") 86 | parser.add_argument("--sensor-id", default="sensor-1", 87 | help="Sensor id (tag name) to use. Defaults to 'sensor-1'") 88 | parser.add_argument("predix_zone_id", metavar="PREDIX_ZONE_ID", 89 | help="Zone Id for authentication") 90 | parser.add_argument("token_file", metavar="TOKEN_FILE", 91 | help="Filename of a file containing the bearer token for authentication") 92 | parsed_args = parser.parse_args(args=argv) 93 | try: 94 | with open(parsed_args.token_file, 'r') as tf: 95 | token = tf.read().rstrip() 96 | except: 97 | parser.error("Problem opening/reading token file %s" % parsed_args.token_file) 98 | 99 | 100 | run(parsed_args, token) 101 | print("Test successful.") 102 | return 0 103 | 104 | if __name__=="__main__": 105 | sys.exit(main()) 106 | -------------------------------------------------------------------------------- /examples/rpi/README.rst: -------------------------------------------------------------------------------- 1 | ================================= 2 | Raspberry Pi Light Sensor Example 3 | ================================= 4 | 5 | This directory contains two example ThingFlow applications that read a tsl2591 6 | Lux sensor connected to a Raspberry Pi running Raspbian Linux. To run these 7 | examples, you need to install several packages, as follows:: 8 | 9 | sudo apt-get install build-essential libi2c-dev i2c-tools python-dev libffi-dev 10 | sudo /usr/bin/pip install cffi 11 | git clone https://github.com/maxlklaxl/python-tsl2591.git 12 | cd python-tsl2591; python setup.py install 13 | 14 | Single Process Example 15 | ---------------------- 16 | The script ``lux_sensor_example.py`` is a single process application that 17 | samples from the lux sensor, prints the events obtained, and activates an LED 18 | on the Pi's GPIO bus if a threshold lux value is exceeded. 19 | 20 | Distributed Example 21 | -------------------- 22 | ``dist_lux_rpi.py`` and ``dist_lux_server.py`` implement a distributed Lux data 23 | collector. The ``dist_lux_rpi.py`` script runs on the Pi and has the same 24 | functionality as the single process example. Additionally, it sends the sensor 25 | events to a MQTT broker. The ``dist_lux_server.py`` script runs on a server that 26 | has the PostgreSQL database installed. The server-side script reads events from 27 | the MQTT queue and saves them to the database. 28 | -------------------------------------------------------------------------------- /examples/rpi/dist_lux_rpi.py: -------------------------------------------------------------------------------- 1 | """Demo of lux sensor and led from raspberry pi - distributed version. 2 | This file contains the data capture part that runs on the Raspberry Pi. 3 | """ 4 | import sys 5 | import asyncio 6 | import argparse 7 | 8 | from thingflow.base import Scheduler, SensorAsOutputThing 9 | from thingflow.sensors.rpi.lux_sensor import LuxSensor 10 | from thingflow.adapters.rpi.gpio import GpioPinOut 11 | from thingflow.adapters.mqtt import MQTTWriter 12 | import thingflow.filters.select 13 | import thingflow.filters.json 14 | 15 | 16 | def setup(broker, threshold): 17 | lux = SensorAsOutputThing(LuxSensor()) 18 | lux.connect(print) 19 | led = GpioPinOut() 20 | actions = lux.map(lambda event: event.val > threshold) 21 | actions.connect(led) 22 | actions.connect(lambda v: print('ON' if v else 'OFF')) 23 | lux.to_json().connect(MQTTWriter(broker, topics=[('bogus/bogus', 0)])) 24 | lux.print_downstream() 25 | return (lux, led) 26 | 27 | 28 | def main(argv=sys.argv[1:]): 29 | parser=argparse.ArgumentParser(description='Distributed lux example, data capture process') 30 | parser.add_argument('-i', '--interval', type=float, default=5.0, 31 | help="Sample interval in seconds") 32 | parser.add_argument('-t', '--threshold', type=float, default=25.0, 33 | help="Threshold lux level above which light should be turned on") 34 | parser.add_argument('broker', metavar="BROKER", 35 | type=str, 36 | help="hostname or ip address of mqtt broker") 37 | parsed_args = parser.parse_args(argv) 38 | (lux, led) = setup(parsed_args.broker, parsed_args.threshold) 39 | scheduler = Scheduler(asyncio.get_event_loop()) 40 | stop = scheduler.schedule_periodic_on_separate_thread(lux, 41 | parsed_args.interval) 42 | print("starting run...") 43 | try: 44 | scheduler.run_forever() 45 | except KeyboardInterrupt: 46 | led.on_completed() 47 | stop() 48 | return 0 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /examples/rpi/dist_lux_server.py: -------------------------------------------------------------------------------- 1 | """Demo of lux sensor and led from raspberry pi 2 | Distributed version - server side: read from an mqtt message queue 3 | and save the datainto a postgres database. We assume the database has 4 | a schema "iot". 5 | 6 | Here is the sql to setup the database table (via psql): 7 | drop table if exists events; 8 | drop sequence if exists events_seq; 9 | create sequence events_seq; 10 | create table events (id bigint NOT NULL DEFAULT nextval('events_seq'), ts timestamp NOT NULL, sensor_id integer NOT NULL, val double precision NOT NULL); 11 | """ 12 | import sys 13 | import asyncio 14 | import getpass 15 | 16 | from thingflow.base import Scheduler, SensorEvent 17 | from thingflow.adapters.mqtt import MQTTReader 18 | from thingflow.adapters.postgres import PostgresWriter, SensorEventMapping 19 | import thingflow.filters.select 20 | import thingflow.filters.json 21 | 22 | connect_string="dbname=iot user=%s" % getpass.getuser() 23 | 24 | mapping = SensorEventMapping('events') 25 | 26 | def setup(host): 27 | mqtt = MQTTReader(host, topics=[('bogus/bogus', 2)]) 28 | decoded = mqtt.select(lambda m:(m.payload).decode("utf-8")) \ 29 | .from_json(constructor=SensorEvent) 30 | scheduler = Scheduler(asyncio.get_event_loop()) 31 | decoded.connect(PostgresWriter(scheduler, connect_string, mapping)) 32 | decoded.output() 33 | mqtt.print_downstream() 34 | return mqtt, scheduler 35 | 36 | 37 | def main(host): 38 | mqtt, scheduler = setup(host) 39 | stop = scheduler.schedule_on_private_event_loop(mqtt) 40 | print("starting run...") 41 | try: 42 | scheduler.run_forever() 43 | except KeyboardInterrupt: 44 | stop() 45 | return 0 46 | 47 | if __name__ == '__main__': 48 | if len(sys.argv)!=2: 49 | print("%s BROKER" % sys.argv[0]) 50 | sys.exit(1) 51 | host=sys.argv[1] 52 | main(host) 53 | -------------------------------------------------------------------------------- /examples/rpi/lux_sensor_example.py: -------------------------------------------------------------------------------- 1 | """Demo of lux sensor and led from raspberry pi 2 | """ 3 | import sys 4 | import asyncio 5 | import os.path 6 | 7 | from thingflow.base import Scheduler, SensorAsOutputThing 8 | from thingflow.sensors.rpi.lux_sensor import LuxSensor 9 | from thingflow.adapters.rpi.gpio import GpioPinOut 10 | import thingflow.adapters.csv 11 | import thingflow.filters.select 12 | 13 | 14 | 15 | 16 | def setup(threshold=25): 17 | lux = SensorAsOutputThing(LuxSensor()) 18 | lux.connect(print) 19 | lux.csv_writer(os.path.expanduser('~/lux.csv')) 20 | led = GpioPinOut() 21 | actions = lux.map(lambda event: event.val > threshold) 22 | actions.connect(led) 23 | actions.connect(lambda v: print('ON' if v else 'OFF')) 24 | lux.print_downstream() 25 | return (lux, led) 26 | 27 | 28 | def main(argv=sys.argv[1:]): 29 | if len(argv)!=2: 30 | print("%s threshold interval" % sys.argv[0]) 31 | return 1 32 | threshold = float(argv[0]) 33 | interval = float(argv[1]) 34 | print("%f seconds interval and an led threshold of %f lux" % 35 | (interval, threshold)) 36 | (lux, led) = setup(threshold) 37 | scheduler = Scheduler(asyncio.get_event_loop()) 38 | stop = scheduler.schedule_periodic_on_separate_thread(lux, interval) 39 | print("starting run...") 40 | try: 41 | scheduler.run_forever() 42 | except KeyboardInterrupt: 43 | led.on_completed() 44 | stop() 45 | return 0 46 | 47 | if __name__ == '__main__': 48 | main() 49 | -------------------------------------------------------------------------------- /examples/simple_csv_reader.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is the example from output_things.rst. It reads a CSV-formatted spreadsheet 3 | file and generates an event from each line. We call output_things that pull 4 | data from an external source "readers". 5 | 6 | To run this script:: 7 | 8 | python simple_csv_reader.py CSV_FILE 9 | 10 | For your csv file, here is some sample data: 11 | ts,id,value 12 | 1490576783,sensor-1,1 13 | 1490576784,sensor-1,1 14 | 1490576785,sensor-1,3 15 | 1490576786,sensor-1,4 16 | 17 | There is a more flexible csv reader class defined in 18 | thingflow.adapters.csv. 19 | """ 20 | 21 | import csv 22 | import sys 23 | import asyncio 24 | from thingflow.base import OutputThing, DirectOutputThingMixin, Scheduler,\ 25 | SensorEvent, FatalError 26 | import thingflow.filters.output # load the output method on the output_thing 27 | 28 | class SimpleCsvReader(OutputThing, DirectOutputThingMixin): 29 | """A simple csv file reader. We assume that each row contains 30 | a timestamp, a sensor id, and a value. 31 | 32 | We could save some work here by subclassing from 33 | thingflow.generic.DirectReader. 34 | """ 35 | def __init__(self, filename, has_header_row=True): 36 | super().__init__() # Make sure the output_thing class is initialized 37 | self.filename = filename 38 | self.file = open(filename, 'r', newline='') 39 | self.reader = csv.reader(self.file) 40 | if has_header_row: 41 | # swallow up the header row so it is not passed as data 42 | try: 43 | self.reader.__next__() 44 | except Exception as e: 45 | raise FatalError("Problem reading header row of csv file %s: %s" % 46 | (filename, e)) 47 | 48 | def _observe(self): 49 | try: 50 | row = self.reader.__next__() 51 | event = SensorEvent(ts=float(row[0]), sensor_id=row[1], 52 | val=float(row[2])) 53 | self._dispatch_next(event) 54 | except StopIteration: 55 | self.file.close() 56 | self._dispatch_completed() 57 | except FatalError: 58 | self._close() 59 | raise 60 | except Exception as e: 61 | self.file.close() 62 | self._dispatch_error(e) 63 | 64 | 65 | # If we are running this as a script, read events from the specified 66 | # file and print them via output(). 67 | if __name__ == '__main__': 68 | # check command line arguments 69 | if len(sys.argv)!=2: 70 | # did not provide filename or provided too many arguments 71 | sys.stderr.write("%s FILENAME\n" % sys.argv[0]) 72 | if len(sys.argv)==1: 73 | sys.stderr.write(" FILENAME is a required parameter\n") 74 | sys.exit(1) 75 | 76 | reader = SimpleCsvReader(sys.argv[1]) 77 | reader.output() 78 | scheduler = Scheduler(asyncio.get_event_loop()) 79 | scheduler.schedule_recurring(reader) 80 | scheduler.run_forever() 81 | sys.exit(0) 82 | 83 | 84 | -------------------------------------------------------------------------------- /examples/tutorial.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is an example thingflow program that is described in tutorial.rst. 3 | """ 4 | 5 | # First, let's define a sensor that generates a random number each time 6 | # it is sampled. 7 | 8 | import random 9 | random.seed() 10 | from thingflow.base import SensorAsOutputThing 11 | 12 | 13 | 14 | class RandomSensor: 15 | def __init__(self, sensor_id, mean, stddev, stop_after): 16 | """This sensor will signal it is completed after the 17 | specified number of events have been sampled. 18 | """ 19 | self.sensor_id = sensor_id 20 | self.mean = mean 21 | self.stddev = stddev 22 | self.events_left = stop_after 23 | 24 | def sample(self): 25 | if self.events_left>0: 26 | data = random.gauss(self.mean, self.stddev) 27 | self.events_left -= 1 28 | return data 29 | else: 30 | raise StopIteration 31 | 32 | def __str__(self): 33 | return "RandomSensor(%s, %s, %s)" % \ 34 | (self.sensor_id, self.mean, self.stddev) 35 | 36 | 37 | # Instantiate our sensor 38 | MEAN = 100 39 | STDDEV = 10 40 | sensor = SensorAsOutputThing(RandomSensor(1, MEAN, STDDEV, stop_after=5)) 41 | 42 | 43 | # Now, we will define a pretend LED as a subscriber. Each time is it passed 44 | # True, it will print 'On'. Each time it is passed False, it will print 'Off'. 45 | from thingflow.base import InputThing 46 | class LED(InputThing): 47 | def on_next(self, x): 48 | if x: 49 | print("On") 50 | else: 51 | print("Off") 52 | 53 | def on_error(self, e): 54 | print("Got an error: %s" % e) 55 | 56 | def on_completed(self): 57 | print("LED Completed") 58 | 59 | def __str__(self): 60 | return 'LED' 61 | 62 | # instantiate an LED 63 | led = LED() 64 | 65 | 66 | # Now, build a pipeline to sample events returned from the sensor, 67 | # convert to a boolean based on whether the value is greater than 68 | # the mean, and output to the LED. 69 | import thingflow.filters.map 70 | sensor.map(lambda evt: evt.val > MEAN).connect(led) 71 | 72 | # If you want to see the raw value of each sensor, just add the output() element 73 | import thingflow.filters.output 74 | sensor.output() 75 | 76 | # Call a debug method on the base output_thing class to see the element tree rooted 77 | # at sensor. 78 | sensor.print_downstream() 79 | 80 | # Now, we need to schedule the sensor to be sampled 81 | import asyncio 82 | from thingflow.base import Scheduler 83 | scheduler = Scheduler(asyncio.get_event_loop()) 84 | scheduler.schedule_periodic(sensor, 1.0) # sample once a second 85 | scheduler.run_forever() # run until all sensors complete 86 | print("That's all folks!") 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /micropython/logger.py: -------------------------------------------------------------------------------- 1 | # A really simple rotating logger 2 | import time 3 | class Logger(object): 4 | DEBUG = 10 5 | INFO = 20 6 | WARNING = 30 7 | WARN = WARNING 8 | ERROR = 40 9 | 10 | def __init__(self, outputfile, max_len, level=INFO, 11 | interactive=False): 12 | # Specify a logfile, the max length of the file before rotating, 13 | # the minimum level to log, and whether this is being run interactivly. 14 | # If it is interactive, we also print messages in addition to saving 15 | # them to the file. 16 | self.outputfile = outputfile 17 | self.backupfile = outputfile + '.1' 18 | self.max_len = max_len 19 | self.level = level 20 | self.interactive = interactive 21 | self.size_written = 0 22 | import os 23 | try: 24 | os.stat(self.outputfile) 25 | # If we get here there is an old log file in place. 26 | # We force a rotation. 27 | self._rotate() 28 | except: 29 | self.fileobj = open(outputfile, 'w') 30 | 31 | def _write(self, levelname, msg): 32 | lt = time.localtime() 33 | data = '%d-%02d-%02d %02d:%02d:%02d [%s] %s\n' % \ 34 | (lt[0], lt[1], lt[2], lt[3], lt[4], lt[5], levelname, msg) 35 | if (len(data) + self.size_written)>self.max_len: 36 | self.fileobj.close() 37 | self._rotate() 38 | self.fileobj.write(data) 39 | self.fileobj.flush() 40 | self.size_written += len(data) 41 | if self.interactive: 42 | print(data, end="") 43 | 44 | def _rotate(self): 45 | import os 46 | try: 47 | os.stat(self.backupfile) 48 | # this only gets run if the file exists 49 | os.remove(self.backupfile) 50 | except: 51 | pass 52 | print("running rotate") 53 | os.rename(self.outputfile, self.backupfile) 54 | self.size_written = 0 55 | self.fileobj = open(self.outputfile, 'w') 56 | 57 | def debug(self, msg): 58 | if self.level <= Logger.DEBUG: 59 | self._write('DBG', msg) 60 | def info(self, msg): 61 | if self.level <= Logger.INFO: 62 | self._write('INF', msg) 63 | def warning(self, msg): 64 | if self.level <= Logger.WARNING: 65 | self._write('WRN', msg) 66 | 67 | warn = warning 68 | 69 | def error(self, msg): 70 | if self.level <= Logger.ERROR: 71 | self._write('ERR', msg) 72 | 73 | def set_level(self, level): 74 | self.level = level 75 | 76 | def close(self): 77 | self.fileobj.close() 78 | self.fileobj = None 79 | 80 | _logger = None 81 | 82 | 83 | def initialize_logging(filename, max_len=32000, level=Logger.INFO, 84 | interactive=False): 85 | global _logger 86 | if _logger!=None: 87 | raise Exception("Logger was already initialized!") 88 | _logger = Logger(filename, max_len, level, interactive) 89 | 90 | 91 | def get_logger(): 92 | global _logger 93 | if _logger!=None: 94 | return _logger 95 | else: 96 | raise Exception("Logger was not yet initialized!") 97 | 98 | def close_logging(): 99 | global _logger 100 | if _logger: 101 | _logger.close() 102 | _logger = None 103 | -------------------------------------------------------------------------------- /micropython/mqtt_writer.py: -------------------------------------------------------------------------------- 1 | # Writer interface over umqtt API. 2 | 3 | from umqtt.robust import MQTTClient 4 | import json 5 | 6 | class MQTTWriter: 7 | __slots__ = ('host', 'port', 'topic', 'client') 8 | def __init__(self, name, host, port, topic): 9 | self.topic = topic 10 | self.host = host 11 | self.port = port 12 | self.client = MQTTClient(name, host, port) 13 | self._connect() 14 | 15 | def _connect(self): 16 | print("Connecting to %s:%s" % (self.host, self.port)) 17 | self.client.connect() 18 | print("Connection successful") 19 | 20 | def on_next(self, x): 21 | data = bytes(json.dumps(x), 'utf-8') 22 | self.client.publish(bytes(self.topic, 'utf-8'), data) 23 | 24 | def on_completed(self): 25 | print("mqtt_completed, disconnecting") 26 | self.client.disconnect() 27 | 28 | def on_error(self, e): 29 | print("mqtt on_error: %s, disconnecting" %e) 30 | self.client.disconnect() 31 | 32 | -------------------------------------------------------------------------------- /micropython/neopixel_writer.py: -------------------------------------------------------------------------------- 1 | """Control NeoPixel-style light strips from ThingFlow. 2 | 3 | num_pixels and bytes_per_pixel will vary, depending on your light strip. 4 | Set pinno to the gpio pin number where you connected the data line of 5 | the NeoPixel. 6 | 7 | If you have bytes_per_pixel=3, the events will be of the form: 8 | (pixelno, r, g, b) 9 | 10 | If you have bytes_per_pixel=4, the events will be of the form: 11 | (pixelno, r, g, b, w) 12 | """ 13 | 14 | from machine import Pin 15 | from neopixel import NeoPixel 16 | 17 | class NeoPixelWriter: 18 | def __init__(self, num_pixels=10, bytes_per_pixel=4, pinno=15): 19 | pin = Pin(pinno, Pin.OUT) 20 | self.np = NeoPixel(pin, num_pixels, bpp=bytes_per_pixel) 21 | self.bytes_per_pixel = bytes_per_pixel 22 | self.tuple_len = bytes_per_pixel+1 23 | 24 | def on_next(self, x): 25 | """The event should be a tuple/list where the first element 26 | is the pixel number and the rest are the settings for that pixel 27 | OR it can be a standard (sensor_id, ts, event) tuple, where the control 28 | message is in the third element. 29 | """ 30 | if len(x)==3 and (isinstance(x[2], tuple) or isinstance(x[2], list)) and \ 31 | len(x[2])==self.tuple_len: 32 | x = x[2] # the control message is embedded in a standard triple 33 | elif len(x)!=self.tuple_len: 34 | raise Exception("expecting a tuple of length %d" % self.tuple_len) 35 | pixel = x[0] 36 | self.np[pixel] = x[1:] 37 | self.np.write() 38 | 39 | def on_error(self, e): 40 | pass 41 | 42 | def on_completed(self): 43 | pass 44 | -------------------------------------------------------------------------------- /micropython/sensors/README.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | Sensors for ThingFlow, Micropython Port 3 | ======================================== 4 | 5 | These are typically copied directly to the root directory on the micropython 6 | system. Due to memory limitations, the comments are pretty sparse. 7 | 8 | Some sensors are adaptations of third party code. Those cases are noted below. 9 | Those files are made available under their original open source licenses. 10 | 11 | tsl2591.py 12 | ---------- 13 | A sensor for the TSL2591 light sensor breakout board from Adafruit. This is a 14 | port to Micropython of `python-tsl2591 `_. 15 | The port has its own github repo at https://github.com/jfischer/micropython-tsl2591. 16 | The file is included here for convenience. 17 | 18 | mcp9808.py 19 | ---------- 20 | A sensor for the MCP9808 temperature sensor breakout board from Adafuit. This 21 | is a port to MicroPython of 22 | `Adafruit-MCP9808 `__. 23 | 24 | adxl345_upy.py 25 | -------------- 26 | Sensor for ADXL345 digital accelerometer. The python library is originally from 27 | https://github.com/pimoroni/adxl345-python with edits for Micropython (that does 28 | not have the smbus module) and Python 3 (print statement). 29 | 30 | adc_esp8266.py 31 | -------------- 32 | Analog to digital sensor for the esp8266 microcontroller. Original 33 | implementation from https://github.com/mzdaniel/micropython-iot 34 | The file is included here for convenience. 35 | -------------------------------------------------------------------------------- /micropython/sensors/adc_esp8266.py: -------------------------------------------------------------------------------- 1 | '''Analog to Digital sensor (in ADC0) for the esp8266 microcontroller''' 2 | 3 | # author: Daniel Mizyrycki 4 | # license: MIT 5 | # repository: https://github.com/mzdaniel/micropython-iot 6 | 7 | from machine import ADC 8 | 9 | class ADCSensor: 10 | def __init__(self, sensor_id='adc', min_rd=0, max_rd=1024, 11 | min_val=0, max_val=1): 12 | '''Initialize sensor 13 | 14 | min_rd and max_rd are used in sample for sensor calibration 15 | min_val and max_val are the sample limits 16 | ''' 17 | self.sensor_id = sensor_id 18 | self.min_rd = min_rd 19 | self.max_rd = max_rd 20 | self.min_val = min_val 21 | self.max_val = max_val 22 | self.coef = (max_val - min_val) / (max_rd - min_rd) 23 | self.adc = ADC(0) 24 | 25 | def read(self) -> int: 26 | '''Get a sensor reading using Micropython API 27 | 28 | Return 0-1024 direct ADC (0~3.3v) reading 29 | ''' 30 | return self.adc.read() 31 | 32 | def sample(self) -> float: 33 | '''Get an ADC interpolated reading using ThingFlow sensor API 34 | 35 | Return min_val~max_val 36 | ''' 37 | reading = self.read() 38 | return self.min_val + (reading - self.min_rd) * self.coef 39 | 40 | def __repr__(self): 41 | return "ADCSensor('%s')" % self.sensor_id 42 | -------------------------------------------------------------------------------- /micropython/sensors/adxl345_upy.py: -------------------------------------------------------------------------------- 1 | # ADXL345 Python library for Raspberry Pi 2 | # 3 | # author: Jonathan Williamson 4 | # license: BSD, see LICENSE.txt included in this package 5 | # 6 | # This is a Raspberry Pi Python implementation to help you get started with 7 | # the Adafruit Triple Axis ADXL345 breakout board: 8 | # http://shop.pimoroni.com/products/adafruit-triple-axis-accelerometer 9 | # 10 | # Minor edit to print statement for Python 3 and AntEvents API changes (need sensor_id) 11 | # Edits for MicroPython (no smbus module) 12 | 13 | from machine import I2C, Pin 14 | from time import sleep 15 | 16 | # select the correct i2c bus for this revision of Raspberry Pi 17 | #revision = ([l[12:-1] for l in open('/proc/cpuinfo','r').readlines() if l[:8]=="Revision"]+['0000'])[0] 18 | #bus = smbus.SMBus(1 if int(revision, 16) >= 4 else 0) 19 | bus = I2C(scl = Pin(5), sda = Pin(4), freq = 100000) 20 | 21 | # ADXL345 constants 22 | EARTH_GRAVITY_MS2 = 9.80665 23 | SCALE_MULTIPLIER = 0.004 24 | 25 | DATA_FORMAT = 0x31 26 | BW_RATE = 0x2C 27 | POWER_CTL = 0x2D 28 | 29 | BW_RATE_1600HZ = [0x0F] 30 | BW_RATE_800HZ = [0x0E] 31 | BW_RATE_400HZ = [0x0D] 32 | BW_RATE_200HZ = [0x0C] 33 | BW_RATE_100HZ = [0x0B] 34 | BW_RATE_50HZ = [0x0A] 35 | BW_RATE_25HZ = [0x09] 36 | 37 | RANGE_2G = 0x00 38 | RANGE_4G = 0x01 39 | RANGE_8G = 0x02 40 | RANGE_16G = 0x03 41 | 42 | MEASURE = [0x08] 43 | AXES_DATA = 0x32 44 | 45 | class ADXL345_upy: 46 | 47 | address = None 48 | 49 | def __init__(self, sensor_id, address = 0x53): 50 | self.sensor_id = sensor_id 51 | self.address = address 52 | self.setBandwidthRate(BW_RATE_100HZ) 53 | self.setRange(RANGE_2G) 54 | self.enableMeasurement() 55 | 56 | def enableMeasurement(self): 57 | bus.writeto_mem(self.address, POWER_CTL, bytearray(MEASURE)) 58 | 59 | def setBandwidthRate(self, rate_flag): 60 | bus.writeto_mem(self.address, BW_RATE, bytearray(rate_flag)) 61 | 62 | # set the measurement range for 10-bit readings 63 | def setRange(self, range_flag): 64 | value = bus.readfrom_mem(self.address, DATA_FORMAT,1) 65 | 66 | val2 = value[0] 67 | val2 &= ~0x0F; 68 | val2 |= range_flag; 69 | val2 |= 0x08; 70 | buf = [val2] 71 | 72 | bus.writeto_mem(self.address, DATA_FORMAT, bytearray(buf)) 73 | 74 | # returns the current reading from the sensor for each axis 75 | # 76 | # parameter gforce: 77 | # False (default): result is returned in m/s^2 78 | # True : result is returned in gs 79 | def sample(self, gforce = False): 80 | #bytes = bus.read_i2c_block_data(self.address, AXES_DATA, 6) 81 | bytes = bus.readfrom_mem(self.address, AXES_DATA, 6) 82 | 83 | x = bytes[0] | (bytes[1] << 8) 84 | if(x & (1 << 16 - 1)): 85 | x = x - (1<<16) 86 | 87 | y = bytes[2] | (bytes[3] << 8) 88 | if(y & (1 << 16 - 1)): 89 | y = y - (1<<16) 90 | 91 | z = bytes[4] | (bytes[5] << 8) 92 | if(z & (1 << 16 - 1)): 93 | z = z - (1<<16) 94 | 95 | x = x * SCALE_MULTIPLIER 96 | y = y * SCALE_MULTIPLIER 97 | z = z * SCALE_MULTIPLIER 98 | 99 | if gforce == False: 100 | x = x * EARTH_GRAVITY_MS2 101 | y = y * EARTH_GRAVITY_MS2 102 | z = z * EARTH_GRAVITY_MS2 103 | 104 | x = round(x, 4) 105 | y = round(y, 4) 106 | z = round(z, 4) 107 | 108 | return {"x": x, "y": y, "z": z} 109 | 110 | if __name__ == "__main__": 111 | # if run directly we'll just create an instance of the class and output 112 | # the current readings 113 | adxl345 = ADXL345() 114 | 115 | axes = adxl345.sample(True) 116 | print("ADXL345 on address 0x%x:" % (adxl345.address)) 117 | print(" x = %.3fG" % ( axes['x'] )) 118 | print(" y = %.3fG" % ( axes['y'] )) 119 | print(" z = %.3fG" % ( axes['z'] )) 120 | -------------------------------------------------------------------------------- /micropython/sensors/mcp9808.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2014 Adafruit Industries 2 | # Author: Tony DiCola 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | # 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | # Ported from Adafruit code for Raspberry Pi 23 | # https://github.com/adafruit/Adafruit_Python_MCP9808/ 24 | 25 | 26 | # Default I2C address for device. 27 | MCP9808_I2CADDR_DEFAULT = 0x18 28 | 29 | # Register addresses. 30 | MCP9808_REG_CONFIG = 0x01 31 | MCP9808_REG_UPPER_TEMP = 0x02 32 | MCP9808_REG_LOWER_TEMP = 0x03 33 | MCP9808_REG_CRIT_TEMP = 0x04 34 | MCP9808_REG_AMBIENT_TEMP = 0x05 35 | MCP9808_REG_MANUF_ID = 0x06 36 | MCP9808_REG_DEVICE_ID = 0x07 37 | 38 | # Configuration register values. 39 | MCP9808_REG_CONFIG_SHUTDOWN = 0x0100 40 | MCP9808_REG_CONFIG_CRITLOCKED = 0x0080 41 | MCP9808_REG_CONFIG_WINLOCKED = 0x0040 42 | MCP9808_REG_CONFIG_INTCLR = 0x0020 43 | MCP9808_REG_CONFIG_ALERTSTAT = 0x0010 44 | MCP9808_REG_CONFIG_ALERTCTRL = 0x0008 45 | MCP9808_REG_CONFIG_ALERTSEL = 0x0002 46 | MCP9808_REG_CONFIG_ALERTPOL = 0x0002 47 | MCP9808_REG_CONFIG_ALERTMODE = 0x0001 48 | 49 | from machine import I2C, Pin 50 | 51 | def _bytes_to_int_be(data): 52 | return data[1] + (data[0]<<8) 53 | 54 | class Mcp9808(object): 55 | """Class to represent an Adafruit MCP9808 precision temperature measurement 56 | board. 57 | """ 58 | def __init__(self, sensor_id='mcp9808', 59 | address=MCP9808_I2CADDR_DEFAULT, scl_pinno=5, sda_pinno=4, 60 | i2c=None): 61 | """Initialize MCP9808 device on the specified I2C address and bus number. 62 | Address defaults to 0x18 and bus number defaults to the appropriate bus 63 | for the hardware. 64 | """ 65 | self.sensor_id = sensor_id 66 | if i2c is None: 67 | self.i2c = I2C(scl=Pin(scl_pinno, Pin.IN),sda=Pin(sda_pinno, Pin.IN)) 68 | else: 69 | self.i2c = i2c 70 | self.address=address 71 | assert self.begin(), "Invalid values read from I2C bus for MCP9808" 72 | 73 | def _read_uint16(self, register): 74 | return _bytes_to_int_be(self.i2c.readfrom_mem(self.address,register, 75 | 2)) 76 | def begin(self): 77 | """Start taking temperature measurements. Returns True if the device is 78 | intialized, False otherwise. 79 | """ 80 | # Check manufacturer and device ID match expected values. 81 | mid = self._read_uint16(MCP9808_REG_MANUF_ID) 82 | did = self._read_uint16(MCP9808_REG_DEVICE_ID) 83 | print('Read manufacturer ID: {0:04X}'.format(mid)) 84 | print('Read device ID: {0:04X}'.format(did)) 85 | return mid == 0x0054 and did == 0x0400 86 | 87 | def sample(self): 88 | """Read sensor and return its value in degrees celsius.""" 89 | # Read temperature register value. 90 | t = self._read_uint16(MCP9808_REG_AMBIENT_TEMP) 91 | # Scale and convert to signed value. 92 | temp = (t & 0x0FFF) / 16.0 93 | if t & 0x1000: 94 | temp -= 256.0 95 | return temp 96 | -------------------------------------------------------------------------------- /micropython/sensors/tsl2591.py: -------------------------------------------------------------------------------- 1 | # tsl2591 lux sensor interface 2 | import time 3 | 4 | VISIBLE = 2 5 | INFRARED = 1 6 | FULLSPECTRUM = 0 7 | 8 | ADDR = 0x29 9 | READBIT = 0x01 10 | COMMAND_BIT = 0xA0 11 | CLEAR_BIT = 0x40 12 | WORD_BIT = 0x20 13 | BLOCK_BIT = 0x10 14 | ENABLE_POWERON = 0x01 15 | ENABLE_POWEROFF = 0x00 16 | ENABLE_AEN = 0x02 17 | ENABLE_AIEN = 0x10 18 | CONTROL_RESET = 0x80 19 | LUX_DF = 408.0 20 | LUX_COEFB = 1.64 21 | LUX_COEFC = 0.59 22 | LUX_COEFD = 0.86 23 | 24 | REGISTER_ENABLE = 0x00 25 | REGISTER_CONTROL = 0x01 26 | REGISTER_THRESHHOLDL_LOW = 0x02 27 | REGISTER_THRESHHOLDL_HIGH = 0x03 28 | REGISTER_THRESHHOLDH_LOW = 0x04 29 | REGISTER_THRESHHOLDH_HIGH = 0x05 30 | REGISTER_INTERRUPT = 0x06 31 | REGISTER_CRC = 0x08 32 | REGISTER_ID = 0x0A 33 | REGISTER_CHAN0_LOW = 0x14 34 | REGISTER_CHAN0_HIGH = 0x15 35 | REGISTER_CHAN1_LOW = 0x16 36 | REGISTER_CHAN1_HIGH = 0x17 37 | INTEGRATIONTIME_100MS = 0x00 38 | INTEGRATIONTIME_200MS = 0x01 39 | INTEGRATIONTIME_300MS = 0x02 40 | INTEGRATIONTIME_400MS = 0x03 41 | INTEGRATIONTIME_500MS = 0x04 42 | INTEGRATIONTIME_600MS = 0x05 43 | 44 | GAIN_LOW = 0x00 45 | GAIN_MED = 0x10 46 | GAIN_HIGH = 0x20 47 | GAIN_MAX = 0x30 48 | 49 | def _bytes_to_int(data): 50 | return data[0] + (data[1]<<8) 51 | 52 | from machine import I2C, Pin 53 | class SMBusEmulator: 54 | __slots__ = ('i2c',) 55 | def __init__(self, scl_pinno=5, sda_pinno=4): 56 | self.i2c = I2C(scl=Pin(scl_pinno, Pin.IN), 57 | sda=Pin(sda_pinno, Pin.IN)) 58 | 59 | def write_byte_data(self, addr, cmd, val): 60 | buf = bytes([cmd, val]) 61 | self.i2c.writeto(addr, buf) 62 | 63 | def read_word_data(self, addr, cmd): 64 | assert cmd < 256 65 | buf = bytes([cmd]) 66 | self.i2c.writeto(addr, buf) 67 | data = self.i2c.readfrom(addr, 4) 68 | return _bytes_to_int(data) 69 | 70 | SENSOR_ADDRESS=0x29 71 | 72 | class Tsl2591: 73 | def __init__( 74 | self, 75 | sensor_id='tsl2591', 76 | integration=INTEGRATIONTIME_100MS, 77 | gain=GAIN_LOW 78 | ): 79 | self.sensor_id = sensor_id 80 | self.bus = SMBusEmulator() 81 | self.integration_time = integration 82 | self.gain = gain 83 | self.set_timing(self.integration_time) 84 | self.set_gain(self.gain) 85 | self.disable() 86 | 87 | def get_i2c(self): 88 | return self.bus.i2c 89 | 90 | def set_timing(self, integration): 91 | self.enable() 92 | self.integration_time = integration 93 | self.bus.write_byte_data( 94 | SENSOR_ADDRESS, 95 | COMMAND_BIT | REGISTER_CONTROL, 96 | self.integration_time | self.gain 97 | ) 98 | self.disable() 99 | 100 | def set_gain(self, gain): 101 | self.enable() 102 | self.gain = gain 103 | self.bus.write_byte_data( 104 | SENSOR_ADDRESS, 105 | COMMAND_BIT | REGISTER_CONTROL, 106 | self.integration_time | self.gain 107 | ) 108 | self.disable() 109 | 110 | def calculate_lux(self, full, ir): 111 | if (full == 0xFFFF) | (ir == 0xFFFF): 112 | return 0 113 | 114 | case_integ = { 115 | INTEGRATIONTIME_100MS: 100., 116 | INTEGRATIONTIME_200MS: 200., 117 | INTEGRATIONTIME_300MS: 300., 118 | INTEGRATIONTIME_400MS: 400., 119 | INTEGRATIONTIME_500MS: 500., 120 | INTEGRATIONTIME_600MS: 600., 121 | } 122 | if self.integration_time in case_integ.keys(): 123 | atime = case_integ[self.integration_time] 124 | else: 125 | atime = 100. 126 | 127 | case_gain = { 128 | GAIN_LOW: 1., 129 | GAIN_MED: 25., 130 | GAIN_HIGH: 428., 131 | GAIN_MAX: 9876., 132 | } 133 | 134 | if self.gain in case_gain.keys(): 135 | again = case_gain[self.gain] 136 | else: 137 | again = 1. 138 | 139 | cpl = (atime * again) / LUX_DF 140 | lux1 = (full - (LUX_COEFB * ir)) / cpl 141 | 142 | lux2 = ((LUX_COEFC * full) - (LUX_COEFD * ir)) / cpl 143 | 144 | return max([lux1, lux2]) 145 | 146 | def enable(self): 147 | self.bus.write_byte_data( 148 | SENSOR_ADDRESS, 149 | COMMAND_BIT | REGISTER_ENABLE, 150 | ENABLE_POWERON | ENABLE_AEN | ENABLE_AIEN 151 | ) 152 | 153 | def disable(self): 154 | self.bus.write_byte_data( 155 | SENSOR_ADDRESS, 156 | COMMAND_BIT | REGISTER_ENABLE, 157 | ENABLE_POWEROFF 158 | ) 159 | 160 | def get_full_luminosity(self): 161 | self.enable() 162 | time.sleep(0.120*self.integration_time+1) 163 | full = self.bus.read_word_data( 164 | SENSOR_ADDRESS, COMMAND_BIT | REGISTER_CHAN0_LOW 165 | ) 166 | ir = self.bus.read_word_data( 167 | SENSOR_ADDRESS, COMMAND_BIT | REGISTER_CHAN1_LOW 168 | ) 169 | self.disable() 170 | return full, ir 171 | 172 | def get_luminosity(self, channel): 173 | full, ir = self.get_full_luminosity() 174 | if channel == FULLSPECTRUM: 175 | return full 176 | elif channel == INFRARED: 177 | return ir 178 | elif channel == VISIBLE: 179 | return full - ir 180 | else: 181 | return 0 182 | 183 | def sample(self): 184 | full, ir = self.get_full_luminosity() 185 | return self.calculate_lux(full, ir) 186 | 187 | -------------------------------------------------------------------------------- /micropython/tests/README.rst: -------------------------------------------------------------------------------- 1 | These tests are designed to be run on a desktop python3 install, not 2 | on the ESP8266 itself. The 8266 does not have the testing libraries to 3 | run these tests (and perhaps not enough memory). 4 | 5 | The idea is that you develop on a desktop, run the tests, and then copy 6 | the files over to the ESP8266 device. 7 | 8 | -------------------------------------------------------------------------------- /micropython/tests/network.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Emulation of ESP8266 network layer 4 | """ 5 | 6 | STA_IF = 1 7 | 8 | class WLAN: 9 | def __init__(self, interface_id): 10 | self.interface_id = interface_id 11 | self.is_active = False 12 | self.connected = False 13 | self.essid = None 14 | 15 | def active(self, make_active): 16 | if make_active: 17 | self.is_active = True 18 | else: 19 | self.is_active = False 20 | self.connected = False 21 | self.essid = None 22 | 23 | def isconnected(self): 24 | return self.connected 25 | 26 | def connect(self, essid, password): 27 | self.connected = True 28 | self.essid = essid 29 | 30 | def ifconfig(self): 31 | return self.essid 32 | -------------------------------------------------------------------------------- /micropython/tests/runtests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Really simple test runner. 3 | # Runs a bunch of python unit tests as standalone programs 4 | # and collects the results. 5 | 6 | ########################### 7 | # Environment setup 8 | ########################### 9 | # Try to guess which python points to python 3 10 | if [[ "`which python3`" == "" ]]; then 11 | PYTHON=`which python` 12 | else 13 | PYTHON=`which python3` 14 | fi 15 | if [[ "$PYTHON" == "" ]]; then 16 | echo "Could not find python!" 17 | exit 1 18 | fi 19 | # verify that it is a python3 20 | $PYTHON -c "import sys; sys.exit(0 if sys.version.startswith('3.') else 1)" 21 | if [[ "$?" == 1 ]]; then 22 | echo "Wrong version of python, need python 3.x, got `$PYTHON --version`" 23 | exit 1 24 | fi 25 | echo "Using python at $PYTHON" 26 | # set python path if necessary 27 | if [[ "$PYTHONPATH" == "" ]]; then 28 | export PYTHONPATH=`cd ..; pwd` 29 | echo "Set PYTHONPATH to $PYTHONPATH" 30 | fi 31 | 32 | # Counts of each test result type 33 | OK=0 34 | SKIPPED=0 35 | FAILED=0 36 | ERROR=0 37 | 38 | # Run a single test and update the counts. Takes one argument: the test name. 39 | # A .py will be appended to get the python filename. The standard output 40 | # goes into $TEST.out and the standard error to $TEST.err. These are not 41 | # kept unless the test fails. 42 | function runtest { 43 | TEST=$1 44 | echo -n "Running $TEST" 45 | $PYTHON $TEST.py >$TEST.out 2>$TEST.err 46 | rc=$? 47 | if [[ "$rc" == 0 ]]; then 48 | # got a success. Now check whether skipped. 49 | tail -1 $TEST.err | grep -q 'OK (skipped' 50 | skiprc=$? 51 | if [[ "$skiprc" == "0" ]]; then 52 | echo " SKIPPED" 53 | SKIPPED=$((SKIPPED+1)) 54 | rm $TEST.err $TEST.out 55 | else 56 | tail -1 $TEST.err | grep -q 'OK' 57 | okrc=$? 58 | if [[ "$okrc" == "0" ]]; then 59 | echo " OK" 60 | OK=$((OK+1)) 61 | rm $TEST.err $TEST.out 62 | else 63 | # did not find the OK 64 | echo " UNKNOWN!" 65 | ERROR=$((ERROR+1)) 66 | fi # okrc 67 | fi # skiprc 68 | else # non-zero return code 69 | tail -1 $TEST.err | grep -q 'FAILED' 70 | failrc=$? 71 | if [[ "$failrc" == "0" ]]; then 72 | echo " FAILED" 73 | FAILED=$((FAILED+1)) 74 | else 75 | echo " ERROR" 76 | ERROR=$((ERROR+1)) 77 | fi # failrc 78 | fi # rc 79 | } 80 | 81 | ########################### 82 | # Run the tests 83 | ########################### 84 | rm -f *.err *.out 85 | echo ">>>>>>>>>>>>>>>>>>>> Starting Tests" 86 | 87 | runtest test_base 88 | runtest test_scheduler 89 | runtest test_utils 90 | runtest test_end_to_end 91 | 92 | 93 | echo ">>>>>>>>>>>>>>>>>>>> Finished Tests" 94 | echo "$OK Tests successful." 95 | echo "$SKIPPED Tests skipped." 96 | echo "$FAILED Tests failed." 97 | echo "$ERROR Tests had errors." 98 | exit $((FAILED + ERROR)) 99 | -------------------------------------------------------------------------------- /micropython/tests/test_base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """These tests are designed to be run on a desktop. You can use 4 | them to validate the system before deploying to 8266. They use stub 5 | sensors. 6 | 7 | Test the core thingflow functionality. 8 | """ 9 | 10 | import sys 11 | import os 12 | import os.path 13 | import time 14 | 15 | try: 16 | from thingflow import * 17 | except ImportError: 18 | sys.path.append(os.path.abspath('../')) 19 | from thingflow import * 20 | 21 | import unittest 22 | 23 | 24 | class DummySensor: 25 | __slots__ = ('value_stream', 'sample_time', 'idx', 'sensor_id') 26 | def __init__(self, value_stream, sample_time=0): 27 | self.value_stream = value_stream 28 | self.idx = 0 29 | self.sample_time = sample_time 30 | self.sensor_id = 1 31 | 32 | def sample(self): 33 | if self.idx==len(self.value_stream): 34 | raise StopIteration() 35 | else: 36 | if self.sample_time > 0: 37 | print("Sensor simulating a sample time of %d seconds with a sleep" % 38 | self.sample_time) 39 | time.sleep(self.sample_time) 40 | val = self.value_stream[self.idx] 41 | self.idx += 1 42 | return val 43 | 44 | def __str__(self): 45 | return 'DummySensor' 46 | 47 | 48 | class ValidationInputThing: 49 | """Compare the values in a event stream to the expected values. 50 | Use the test_case for the assertions (for proper error reporting in a unit 51 | test). 52 | """ 53 | def __init__(self, expected_stream, test_case, 54 | extract_value_fn=lambda event:event[2]): 55 | self.expected_stream = expected_stream 56 | self.next_idx = 0 57 | self.test_case = test_case 58 | self.extract_value_fn = extract_value_fn 59 | self.completed = False 60 | 61 | def on_next(self, x): 62 | tc = self.test_case 63 | tc.assertLess(self.next_idx, len(self.expected_stream), 64 | "Got an event after reaching the end of the expected stream") 65 | expected = self.expected_stream[self.next_idx] 66 | actual = self.extract_value_fn(x) 67 | tc.assertEqual(actual, expected, 68 | "Values for element %d of event stream mismatch" % self.next_idx) 69 | print(x) 70 | self.next_idx += 1 71 | 72 | def on_completed(self): 73 | tc = self.test_case 74 | tc.assertEqual(self.next_idx, len(self.expected_stream), 75 | "Got on_completed() before end of stream") 76 | self.completed = True 77 | 78 | def on_error(self, exc): 79 | tc = self.test_case 80 | tc.assertTrue(False, 81 | "Got an unexpected on_error call with parameter: %s" % exc) 82 | 83 | 84 | class TestBase(unittest.TestCase): 85 | def test_base(self): 86 | expected = [1, 2, 3, 4, 5] 87 | sensor = DummySensor(expected) 88 | output_thing = SensorAsOutputThing(sensor) 89 | validator = ValidationInputThing(expected, self) 90 | output_thing.connect(validator) 91 | scheduler = Scheduler() 92 | scheduler.schedule_periodic(output_thing, 1) 93 | scheduler.run_forever() 94 | self.assertTrue(validator.completed) 95 | 96 | def test_schedule_sensor(self): 97 | expected = [1, 2, 3, 4, 5] 98 | sensor = DummySensor(expected) 99 | validator = ValidationInputThing(expected, self) 100 | scheduler = Scheduler() 101 | scheduler.schedule_sensor(sensor, 1, validator) 102 | scheduler.run_forever() 103 | self.assertTrue(validator.completed) 104 | 105 | def test_nonzero_sample_time(self): 106 | """Sensor sample time is greater than the interval between samples! 107 | """ 108 | expected = [1, 2, 3, 4, 5] 109 | sensor = DummySensor(expected, sample_time=2) 110 | output_thing = SensorAsOutputThing(sensor) 111 | validator = ValidationInputThing(expected, self) 112 | output_thing.connect(validator) 113 | scheduler = Scheduler() 114 | scheduler.schedule_periodic(output_thing, 1) 115 | scheduler.run_forever() 116 | self.assertTrue(validator.completed) 117 | 118 | def test_subsecond_schedule_interval(self): 119 | expected = [1, 2, 3, 4, 5] 120 | sensor = DummySensor(expected) 121 | output_thing = SensorAsOutputThing(sensor) 122 | validator = ValidationInputThing(expected, self) 123 | output_thing.connect(validator) 124 | scheduler = Scheduler() 125 | scheduler.schedule_periodic(output_thing, 0.25) 126 | start = time.time() 127 | scheduler.run_forever() 128 | stop = time.time() 129 | elapsed = stop - start 130 | print("elapsed was %s" % round(elapsed, 2)) 131 | self.assertTrue((elapsed>1.0) and (elapsed<2.0), 132 | "Elapsed time should be between 1 and 2 seconds, was %s" % elapsed) 133 | self.assertTrue(validator.completed) 134 | 135 | 136 | if __name__ == '__main__': 137 | unittest.main() 138 | -------------------------------------------------------------------------------- /micropython/tests/test_end_to_end.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """These tests are designed to be run on a desktop. You can use 4 | them to validate the system before deploying to 8266. They use stub 5 | sensors. 6 | 7 | Test end-to-end functionality where we sample a sensor event and write to a 8 | queue. This only works if you have a broker at localhost:1883. 9 | 10 | To validate that the messages are being received, go to the commandline and 11 | run: 12 | mosquitto_sub -t test 13 | """ 14 | 15 | MQTT_PORT=1883 16 | 17 | import sys 18 | import os 19 | import os.path 20 | import time 21 | 22 | try: 23 | from thingflow import * 24 | except ImportError: 25 | sys.path.append(os.path.abspath('../')) 26 | from thingflow import * 27 | 28 | 29 | import unittest 30 | 31 | from mqtt_writer import MQTTWriter 32 | 33 | class DummySensor(object): 34 | def __init__(self, sensor_id, value_stream, sample_time=0): 35 | self.sensor_id = sensor_id 36 | self.value_stream = value_stream 37 | self.idx = 0 38 | self.sample_time = sample_time 39 | 40 | def sample(self): 41 | if self.idx==len(self.value_stream): 42 | raise StopIteration() 43 | else: 44 | if self.sample_time > 0: 45 | print("Sensor simulating a sample time of %d seconds with a sleep" % 46 | self.sample_time) 47 | time.sleep(self.sample_time) 48 | val = self.value_stream[self.idx] 49 | self.idx += 1 50 | return val 51 | 52 | def __str__(self): 53 | return 'DummySensor' 54 | 55 | class ValidationInputThing: 56 | """Compare the values in a event stream to the expected values. 57 | Use the test_case for the assertions (for proper error reporting in a unit 58 | test). 59 | """ 60 | def __init__(self, expected_stream, test_case, 61 | extract_value_fn=lambda event:event[2]): 62 | self.expected_stream = expected_stream 63 | self.next_idx = 0 64 | self.test_case = test_case 65 | self.extract_value_fn = extract_value_fn 66 | self.completed = False 67 | 68 | def on_next(self, x): 69 | tc = self.test_case 70 | tc.assertLess(self.next_idx, len(self.expected_stream), 71 | "Got an event after reaching the end of the expected stream") 72 | expected = self.expected_stream[self.next_idx] 73 | actual = self.extract_value_fn(x) 74 | tc.assertEqual(actual, expected, 75 | "Values for element %d of event stream mismatch" % self.next_idx) 76 | self.next_idx += 1 77 | 78 | def on_completed(self): 79 | tc = self.test_case 80 | tc.assertEqual(self.next_idx, len(self.expected_stream), 81 | "Got on_completed() before end of stream") 82 | self.completed = True 83 | 84 | def on_error(self, exc): 85 | tc = self.test_case 86 | tc.assertTrue(False, 87 | "Got an unexpected on_error call with parameter: %s" % exc) 88 | 89 | 90 | def is_broker_running(): 91 | import subprocess 92 | rc = subprocess.call("netstat -an | grep %d" % MQTT_PORT, shell=True) 93 | if rc==0: 94 | return True 95 | else: 96 | return False 97 | 98 | 99 | @unittest.skipUnless(is_broker_running(), 100 | "Did not find a broker listening on port %d" % MQTT_PORT) 101 | class TestEndToEnd(unittest.TestCase): 102 | def test_publish_sensor(self): 103 | expected = [1, 2, 3, 4, 5] 104 | sensor = DummySensor('lux-1', expected) 105 | output_thing = SensorAsOutputThing(sensor) 106 | validator = ValidationInputThing(expected, self) 107 | output_thing.connect(validator) 108 | self.writer = MQTTWriter('thingflow', 'localhost', MQTT_PORT, 'test') 109 | output_thing.connect(self.writer) 110 | scheduler = Scheduler() 111 | scheduler.schedule_periodic(output_thing, 1) 112 | scheduler.run_forever() 113 | self.assertTrue(validator.completed) 114 | 115 | if __name__ == '__main__': 116 | unittest.main() 117 | -------------------------------------------------------------------------------- /micropython/tests/test_fast.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | This isn't actually a unit test - it is for validating fast sampling on the 5 | actual ESP8266. It samples a dummy sensor at 10Hz and pushes to an 6 | MQTT queue. To run it: 7 | 8 | 0. Set up an mqtt broker (mosquitto) 9 | 1. Replace the CHANGE_ME values with the appropriate values for your environment 10 | 2. Copy the script to the ESP8266 (e.g. via mpfsheel) and import the module. 11 | """ 12 | from thingflow import Scheduler 13 | from mqtt_writer import MQTTWriter 14 | from wifi import wifi_connect 15 | import uos 16 | import ustruct 17 | 18 | 19 | # Params to set 20 | WIFI_SID="CHANGE_ME" 21 | WIFI_PW="CHANGE_ME" 22 | SENSOR_ID="front-room" 23 | BROKER='CHANGE_ME' 24 | 25 | class DummySensor: 26 | def __init__(self, sensor_id): 27 | self.sensor_id = sensor_id 28 | 29 | def sample(self): 30 | return ustruct.unpack('@H', uos.urandom(2))[0] 31 | 32 | 33 | wifi_connect(WIFI_SID, WIFI_PW) 34 | sensor = DummySensor(sensor_id=SENSOR_ID) 35 | writer = MQTTWriter(SENSOR_ID, BROKER, 1883, 'remote-sensors') 36 | sched = Scheduler() 37 | sched.schedule_sensor(sensor, 0.1, writer) 38 | print("Starting sensor sampling") 39 | sched.run_forever() 40 | 41 | -------------------------------------------------------------------------------- /micropython/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """These tests are designed to be run on a desktop. You can use 4 | them to validate the system before deploying to 8266. They use stub 5 | sensors. 6 | 7 | Test wifi.py and logger.py 8 | """ 9 | 10 | import sys 11 | import os 12 | import os.path 13 | 14 | try: 15 | import wifi 16 | import logger 17 | except ImportError: 18 | sys.path.append(os.path.abspath('../')) 19 | import wifi 20 | import logger 21 | 22 | 23 | import unittest 24 | 25 | 26 | class TestLogging(unittest.TestCase): 27 | def _cleanup(self): 28 | logger.close_logging() 29 | for f in ['test.log', 'test.log.1']: 30 | if os.path.exists(f): 31 | os.remove(f) 32 | 33 | def setUp(self): 34 | self._cleanup() 35 | 36 | def tearDown(self): 37 | self._cleanup() 38 | 39 | def test_logging(self): 40 | self.assertTrue(not os.path.exists('test.log')) 41 | logger.initialize_logging('test.log', max_len=1024, interactive=True) 42 | l = logger.get_logger() 43 | self.assertTrue(os.path.exists('test.log')) 44 | self.assertTrue(not os.path.exists('test.log.1')) 45 | l.debug('debug msg') 46 | l.info('info msg') 47 | l.warn('warn') 48 | l.error('error') 49 | l.info('d'*1024) # force a rollover 50 | l.info('new file') 51 | self.assertTrue(os.path.exists('test.log')) 52 | self.assertTrue(os.path.exists('test.log.1')) 53 | 54 | def test_wifi_connect(self): 55 | wifi.wifi_connect('foo', 'bar') 56 | 57 | if __name__ == '__main__': 58 | unittest.main() 59 | -------------------------------------------------------------------------------- /micropython/tests/ubinascii.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Emulation for micropython ubinascii module. 4 | """ 5 | 6 | from binascii import hexlify 7 | -------------------------------------------------------------------------------- /micropython/tests/ucollections.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Emulation for micropython's ucollections 4 | """ 5 | from collections import namedtuple 6 | 7 | -------------------------------------------------------------------------------- /micropython/tests/umqtt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mpi-sws-rse/thingflow-python/c191a8fedac6a962994945830c872e957f929e29/micropython/tests/umqtt/__init__.py -------------------------------------------------------------------------------- /micropython/tests/umqtt/robust.py: -------------------------------------------------------------------------------- 1 | """Fake mqtt interface - this simulates the api provided 2 | by micropython. We use paho.mqtt to talk to the broker. 3 | """ 4 | import paho.mqtt.client 5 | 6 | class MQTTClient: 7 | def __init__(self, name, host, port): 8 | self.client = paho.mqtt.client.Client(name) 9 | self.host = host 10 | self.port = port 11 | 12 | def connect(self): 13 | self.client.connect(self.host, self.port) 14 | self.client.loop_start() 15 | 16 | 17 | def disconnect(self): 18 | self.client.disconnect() 19 | self.client.loop_stop(force=False) 20 | 21 | def publish(self, topic, data): 22 | topic = str(topic, encoding='utf-8') # paho wants a string 23 | print("publishing %s on %s" % (repr(data), repr(topic))) 24 | self.client.publish(topic, data) 25 | 26 | -------------------------------------------------------------------------------- /micropython/tests/utime.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Emuation for micropython's utime module 4 | """ 5 | from time import time, sleep 6 | 7 | def ticks_ms(): 8 | return int(round(time()*1000)) 9 | 10 | def sleep_ms(ms): 11 | sleep(ms/1000) 12 | 13 | def ticks_diff(old, new): 14 | return new - old 15 | 16 | -------------------------------------------------------------------------------- /micropython/wifi.py: -------------------------------------------------------------------------------- 1 | import network 2 | import utime 3 | 4 | def wifi_connect(essid, password): 5 | # Connect to the wifi. Based on the example in the micropython 6 | # documentation. 7 | wlan = network.WLAN(network.STA_IF) 8 | wlan.active(True) 9 | if not wlan.isconnected(): 10 | print('connecting to network ' + essid + '...') 11 | wlan.connect(essid, password) 12 | # connect() appears to be async - waiting for it to complete 13 | while not wlan.isconnected(): 14 | print('waiting for connection...') 15 | utime.sleep(4) 16 | print('checking connection...') 17 | print('Wifi connect successful, network config: %s' % repr(wlan.ifconfig())) 18 | else: 19 | # Note that connection info is stored in non-volatile memory. If 20 | # you are connected to the wrong network, do an explicity disconnect() 21 | # and then reconnect. 22 | print('Wifi already connected, network config: %s' % repr(wlan.ifconfig())) 23 | 24 | def wifi_disconnect(): 25 | # Disconnect from the current network. You may have to 26 | # do this explicitly if you switch networks, as the params are stored 27 | # in non-volatile memory. 28 | wlan = network.WLAN(network.STA_IF) 29 | if wlan.isconnected(): 30 | print("Disconnecting...") 31 | wlan.disconnect() 32 | else: 33 | print("Wifi not connected.") 34 | 35 | def disable_wifi_ap(): 36 | # Disable the built-in access point. 37 | wlan = network.WLAN(network.AP_IF) 38 | wlan.active(False) 39 | print('Disabled access point, network status is %s' % 40 | wlan.status()) 41 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2016,2017 by MPI-SWS and Data-Ken Research. 3 | # Licensed under the Apache 2.0 License. 4 | """Setup script for thingflow distribution. Note that we only 5 | package up the python code. The tests, docs, and examples 6 | are all kept only in the full source repository. 7 | """ 8 | 9 | import sys 10 | sys.path.insert(0, 'thingflow') 11 | from thingflow import __version__ 12 | 13 | #We try setuptools first (which has more features), and 14 | # fallback to distutils if setuptools was not installed. 15 | try: 16 | from setuptools import setup 17 | except ImportError: 18 | print("Did not find setuptools, using distutils instead") 19 | from distutils.core import setup 20 | 21 | DESCRIPTION =\ 22 | """ 23 | ThingFlow is a (Python3) framework for building IOT event processing 24 | dataflows. The goal of this framework is to support the 25 | creation of robust IoT systems from reusable components. These systems must 26 | account for noisy/missing sensor data, distributed computation, and the 27 | need for local (near the data source) processing. 28 | 29 | ThingFlow is pure Python (3.4 or later). The packaged distribution 30 | (e.g. on PyPi) only includes the core Python code. The source repository at 31 | https://github.com/mpi-sws-rse/thingflow-python contains the core Python 32 | code plus the documentation, examples, and tests. There is also a port 33 | of ThingFlow for micropython available in the source repo. 34 | """ 35 | 36 | setup(name='thingflow', 37 | version=__version__, 38 | description="Event Stream processing library for IOT", 39 | long_description=DESCRIPTION, 40 | license="Apache 2.0", 41 | author="MPI-SWS and Data-Ken Research", 42 | author_email="info@thingflow.io", 43 | maintainer='Jeff Fischer', 44 | maintainer_email='jeff+thingflow@data-ken.org', 45 | url='https://github.com/mpi-sws-rse/thingflow-python', 46 | packages=['thingflow', 'thingflow.internal', 'thingflow.filters', 47 | 'thingflow.sensors', 'thingflow.sensors.rpi', 48 | 'thingflow.adapters', 'thingflow.adapters.rpi'], 49 | classifiers = [ 50 | 'Development Status :: 4 - Beta', 51 | 'License :: OSI Approved :: Apache Software License', 52 | 'Programming Language :: Python :: 3.4', 53 | 'Programming Language :: Python :: 3.5', 54 | 'Programming Language :: Python :: 3.6', 55 | 'Operating System :: OS Independent', 56 | 'Intended Audience :: Developers' , 57 | ], 58 | keywords = ['events', 'iot', 'sensors'], 59 | ) 60 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | config_for_tests.py 2 | *.err 3 | *.out 4 | -------------------------------------------------------------------------------- /tests/README.rst: -------------------------------------------------------------------------------- 1 | ============================ 2 | Antevents Tests 3 | ============================ 4 | 5 | This directory contains unit tests for the antlab infrastructure and 6 | adapters. Use the script runtests.sh to run all the tests. If will stop 7 | on the first error it encounters (as signified by a non-zero return code 8 | from the test program). 9 | 10 | As much as possible, the tests are standalone and do not require external 11 | dependencies. However, tests of specific adapters will often require some 12 | software to be installed and configured. To support this, we do the following: 13 | 14 | 1. Any configuration variables (e.g. usernames, passwords, connect strings) 15 | go into the file config_for_tests.py. This file is NOT checked into 16 | git, since it may contain sensitive data. Instead copy the file 17 | example_config_for_tests.py to config_for_tests.py and adjust it for 18 | your environment. 19 | 2. Tests with external dependencies use the @unittest.skipUnless decorator 20 | to check for the dependencies and skip the test if the requirements are 21 | not met. 22 | 23 | Dependencies 24 | ------------- 25 | Here are the commands used to install all the dependencies on Ubuntu:: 26 | 27 | sudo apt-get install postgresql 28 | sudo apt-get install libpq-dev python3-dev 29 | pip install psycopg2 30 | # In the following, replace USER with your linux user name 31 | sudo -u postgres psql <${TEST}.out 2>${TEST}.err 61 | rc=$? 62 | if [[ "$rc" == 0 ]]; then 63 | # got a success. Now check whether skipped. 64 | tail -1 $TEST.err | grep -q 'OK (skipped' 65 | skiprc=$? 66 | if [[ "$skiprc" == "0" ]]; then 67 | echo -e " ${YELLOW}SKIPPED${NC}" 68 | SKIPPED=$((SKIPPED+1)) 69 | rm $TEST.err $TEST.out 70 | else 71 | tail -1 $TEST.err | grep -q 'OK' 72 | okrc=$? 73 | if [[ "$okrc" == "0" ]]; then 74 | echo -e " ${GREEN}OK${NC}" 75 | OK=$((OK+1)) 76 | rm $TEST.err $TEST.out 77 | else 78 | # did not find the OK 79 | echo -e " ${RED}UNKNOWN!${NC}" 80 | ERROR=$((ERROR+1)) 81 | fi # okrc 82 | fi # skiprc 83 | else # non-zero return code 84 | tail -1 $TEST.err | grep -q 'FAILED' 85 | failrc=$? 86 | if [[ "$failrc" == "0" ]]; then 87 | echo -e " ${RED}FAILED${NC}" 88 | FAILED=$((FAILED+1)) 89 | else 90 | echo -e " ${RED}ERROR${NC}" 91 | ERROR=$((ERROR+1)) 92 | fi # failrc 93 | fi # rc 94 | } 95 | 96 | 97 | ########################### 98 | # validate command line arguments 99 | ########################### 100 | if [[ "$#" == "0" ]]; then 101 | TESTS_TO_RUN=$TESTS 102 | else 103 | ARGS=${@:1} 104 | for tst in $ARGS; do 105 | found=0 106 | for chktst in $TESTS; do 107 | if [[ "$chktst" == "$tst" ]]; then 108 | found=1 109 | break 110 | fi 111 | done 112 | if [[ $found != 1 ]]; then 113 | echo -e "${RED}ERROR: $tst is not a known test.${NC}" 114 | echo "Valid tests are: $TESTS" 115 | exit 1 116 | fi 117 | done 118 | TESTS_TO_RUN=$ARGS 119 | fi 120 | 121 | 122 | ########################### 123 | # Run the tests 124 | ########################### 125 | rm -f *.err *.out 126 | 127 | echo ">>>>>>>>>>>>>>>>>>>> Starting Tests" 128 | 129 | 130 | for tst in $TESTS_TO_RUN; do 131 | runtest ${tst} 132 | done 133 | 134 | echo ">>>>>>>>>>>>>>>>>>>> Finished Tests" 135 | if [[ "$OK" == "0" ]]; then 136 | echo -e "${RED}0 Tests successful.${NC}" 137 | else 138 | echo -e "${GREEN}$OK Tests successful.${NC}" 139 | fi 140 | if [[ "$SKIPPED" == "0" ]]; then 141 | echo -e "${GREEN}0 Tests skipped.${NC}" 142 | else 143 | echo -e "${YELLOW}$SKIPPED Tests skipped.${NC}" 144 | fi 145 | if [[ "$FAILED" == "0" ]]; then 146 | echo -e "${GREEN}0 Tests failed.${NC}" 147 | else 148 | echo -e "${RED}$FAILED Tests failed.${NC}" 149 | fi 150 | if [[ "$ERROR" == "0" ]]; then 151 | echo -e "${GREEN}0 Tests had errors.${NC}" 152 | else 153 | echo -e "${RED}$ERROR Tests had errors.${NC}" 154 | fi 155 | exit $((FAILED + ERROR)) 156 | -------------------------------------------------------------------------------- /tests/test_blocking_input_thing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Run a subscriber that blocks in its send call. It gets a separate dedicated thread. 4 | """ 5 | 6 | import unittest 7 | import asyncio 8 | import time 9 | from thingflow.base import BlockingInputThing, Scheduler 10 | from utils import make_test_output_thing_from_vallist 11 | 12 | values = [ 1, 2, 3, 4, 5 ] 13 | 14 | class TestInputThing(BlockingInputThing): 15 | def __init__(self, scheduler, expected_sequence, test_case): 16 | self.tc = test_case 17 | self.expected_sequence = expected_sequence 18 | self.idx = 0 19 | self.completed = False 20 | super().__init__(scheduler) 21 | 22 | def _on_next(self, port, x): 23 | assert port=='default' 24 | print("TestInputThing._on_next(%s)" % x.__repr__()) 25 | self.tc.assertTrue(self.idx < len(self.expected_sequence), 26 | "Received an event %s, but already at end of expected sequence" % 27 | x.__repr__()) 28 | self.tc.assertEqual(self.expected_sequence[self.idx], x[2], 29 | "Expected and actual values do not match for item %d" % self.idx) 30 | self.idx += 1 31 | 32 | def _on_completed(self, port): 33 | assert port=='default' 34 | self.tc.assertEqual(len(self.expected_sequence), self.idx, 35 | "Received on_completed when not at end of expected sequence") 36 | self.completed = True 37 | 38 | def _on_error(self, port, e): 39 | raise Exception("Should not get an on_error event. Got exception %s" % e) 40 | 41 | 42 | class TestCase(unittest.TestCase): 43 | def test(self): 44 | scheduler = Scheduler(asyncio.get_event_loop()) 45 | sensor = make_test_output_thing_from_vallist(1, values) 46 | scheduler.schedule_periodic(sensor, 1) 47 | blocking_subscriber = TestInputThing(scheduler, values, self) 48 | sensor.connect(blocking_subscriber) 49 | scheduler.run_forever() 50 | self.assertTrue(blocking_subscriber.completed) 51 | 52 | if __name__ == '__main__': 53 | unittest.main() 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /tests/test_blocking_output_thing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Run an output_thing that might block in a separate background thread 4 | """ 5 | import time 6 | import unittest 7 | 8 | from thingflow.base import OutputThing, DirectOutputThingMixin, InputThing,\ 9 | Scheduler 10 | from thingflow.filters.combinators import passthrough 11 | from thingflow.filters.output import output 12 | from utils import ValidationInputThing 13 | 14 | import asyncio 15 | 16 | EVENTS = 4 17 | 18 | 19 | class BlockingOutputThing(OutputThing, DirectOutputThingMixin): 20 | def __init__(self): 21 | super().__init__() 22 | self.event_count = 0 23 | 24 | def _observe(self): 25 | self.event_count += 1 26 | time.sleep(0.5) # simulate a blocking call 27 | self._dispatch_next(self.event_count) 28 | 29 | 30 | class StopLoopAfter(InputThing): 31 | def __init__(self, stop_after, cancel_thunk): 32 | self.events_left = stop_after 33 | self.cancel_thunk = cancel_thunk 34 | 35 | def on_next(self, x): 36 | self.events_left -= 1 37 | if self.events_left == 0: 38 | print("Requesting stop of event loop") 39 | self.cancel_thunk() 40 | 41 | class BlockingSensor: 42 | def __init__(self, sensor_id, stop_after): 43 | self.sensor_id = sensor_id 44 | self.stop_after = stop_after 45 | self.event_count = 0 46 | 47 | def sample(self): 48 | if self.event_count==self.stop_after: 49 | raise StopIteration 50 | self.event_count += 1 51 | time.sleep(0.5) # simulate a blocking call 52 | return self.event_count 53 | 54 | def __repr__(self): 55 | return "BlockingSensor(%s, stop_after=%s)" % (self.sensor_id, 56 | self.stop_after) 57 | 58 | 59 | class TestCase(unittest.TestCase): 60 | def test_blocking_output_thing(self): 61 | o = BlockingOutputThing() 62 | o.output() 63 | scheduler = Scheduler(asyncio.get_event_loop()) 64 | c = scheduler.schedule_periodic_on_separate_thread(o, 1) 65 | vs = ValidationInputThing([i+1 for i in range(EVENTS)], self, 66 | extract_value_fn=lambda v:v) 67 | o.connect(vs) 68 | o.connect(StopLoopAfter(EVENTS, c)) 69 | o.print_downstream() 70 | scheduler.run_forever() 71 | print("that's it") 72 | 73 | def test_blocking_sensor(self): 74 | s = BlockingSensor(1, stop_after=EVENTS) 75 | scheduler = Scheduler(asyncio.get_event_loop()) 76 | scheduler.schedule_sensor_on_separate_thread(s, 1, 77 | passthrough(output()), 78 | ValidationInputThing([i+1 for i in range(EVENTS)], self, 79 | extract_value_fn=lambda v:v), 80 | make_event_fn=lambda s, v: v) 81 | scheduler.run_forever() 82 | print("that's it") 83 | 84 | if __name__ == '__main__': 85 | unittest.main() 86 | 87 | -------------------------------------------------------------------------------- /tests/test_bokeh.py: -------------------------------------------------------------------------------- 1 | # Make sure bokeh server is running 2 | # bokeh serve 3 | 4 | import asyncio 5 | from utils import ValueListSensor, ValidationSubscriber 6 | from antevents.base import Scheduler, SensorPub, SensorEvent 7 | 8 | from antevents.linq.map import map 9 | from antevents.adapters.bokeh import bokeh_output_streaming, BokehStreamer, BokehPlot, BokehPlotManager 10 | import datetime, time 11 | 12 | def mk_csv(): 13 | sid = 'temp' 14 | val = 0 15 | csv = [ ] 16 | for i in range(3): 17 | ts = time.mktime(datetime.datetime.now().timetuple()) 18 | dt = ts 19 | val = val + 1 20 | csv.append([SensorEvent(ts=ts, sensor_id=sid, val=val)]) 21 | time.sleep(1) 22 | return csv 23 | 24 | def debug(): 25 | csv = mk_csv() 26 | bokeh_output(csv) 27 | bokeh_output_streaming(csv) 28 | 29 | value_stream = [10, 13, 20, 20, 19, 19, 20, 21, 28, 28, 23, 21, 21, 18, 19, 16, 21, 30 | 10, 13, 20, 20, 19, 19, 20, 21, 28, 28, 23, 21, 21, 18, 19, 16, 21] 31 | value_stream2 = [2, 3, 2, 2, 9, 9, 2, 1, 8, 8, 3, 2, 1, 8, 9, 6, 2, 2, 3, 4, 5, 6, 7, 8, 2, 3, 4, 32 | 6, 2, 3, 2, 2, 9, 9, 2, 1, 8, 8, 3, 2, 1, 8, 9, 6, 2, 2, 3, 4, 5, 6, 7, 8, 2, 3, 4, 6, 33 | 6, 2, 3, 2, 2, 9, 9, 2, 1, 8, 8, 3, 2, 1, 8, 9, 6, 2, 2, 3, 4, 5, 6, 7, 8, 2, 3, 4, 6, 34 | 6, 2, 3, 2, 2, 9, 9, 2, 1, 8, 8, 3, 2, 1, 8, 9, 6, 2, 2, 3, 4, 5, 6, 7, 8, 2, 3, 4, 6, 35 | 6, 2, 3, 2, 2, 9, 9, 2, 1, 8, 8, 3, 2, 1, 8, 9, 6, 2, 2, 3, 4, 5, 6, 7, 8, 2, 3, 4, 6] 36 | 37 | def test_bokeh_output(): 38 | loop = asyncio.get_event_loop() 39 | s = ValueListSensor(1, value_stream) 40 | p = SensorPub(s) 41 | b = BokehStreamer([ SensorEvent(ts=0,val=10,sensor_id="temp" ) ], io_loop=loop) 42 | p.subscribe(b) 43 | 44 | scheduler = Scheduler(loop) 45 | scheduler.schedule_periodic(p, 0.5) # sample twice every second 46 | scheduler.run_forever() 47 | self.assertTrue(vo.completed, 48 | "Schedule exited before validation observer completed") 49 | print("That's all folks") 50 | 51 | def test_bokeh_manager(): 52 | loop = asyncio.get_event_loop() 53 | s1 = ValueListSensor(1, value_stream) 54 | p1 = SensorPub(s1) 55 | s2 = ValueListSensor(1, value_stream2) 56 | p2 = SensorPub(s2) 57 | 58 | bm = BokehPlotManager() 59 | bplot1 = BokehPlot('Sensor1', y_axis_label='value') 60 | bplot2 = BokehPlot('Sensor2', y_axis_label='value') 61 | bm.register(bplot1) 62 | bm.register(bplot2) 63 | p1.map(lambda v: ('Sensor1', v) ).subscribe(bm) 64 | p2.map(lambda v: ('Sensor2', v) ).subscribe(bm) 65 | bm.start() 66 | 67 | scheduler = Scheduler(loop) 68 | scheduler.schedule_periodic(p1, 1.0) # sample every second 69 | scheduler.schedule_periodic(p2, 0.5) # sample twice every second 70 | scheduler.run_forever() 71 | # self.assertTrue(vo.completed, 72 | # "Schedule exited before validation observer completed") 73 | print("That's all folks") 74 | 75 | if __name__ == "__main__": 76 | # test_bokeh_output() 77 | test_bokeh_manager() 78 | -------------------------------------------------------------------------------- /tests/test_csv_adapters.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Verify the csv reader/writer through a round trip 4 | """ 5 | 6 | import unittest 7 | import time 8 | from tempfile import NamedTemporaryFile 9 | import os 10 | import asyncio 11 | import datetime 12 | 13 | from thingflow.base import Scheduler, IterableAsOutputThing, SensorEvent 14 | from thingflow.adapters.csv import CsvReader, default_event_mapper 15 | import thingflow.filters.dispatch 16 | from utils import make_test_output_thing, CaptureInputThing, \ 17 | SensorEventValidationInputThing 18 | 19 | NUM_EVENTS=5 20 | 21 | class TestCases(unittest.TestCase): 22 | def test_default_mapper(self): 23 | """Verify the class that maps between an event and a sensor 24 | """ 25 | event = SensorEvent(ts=time.time(), sensor_id=1, val=123.456) 26 | row = default_event_mapper.event_to_row(event) 27 | event2 = default_event_mapper.row_to_event(row) 28 | self.assertEqual(event2, event, 29 | "Round-tripped event does not match original event") 30 | 31 | def test_file_write_read(self): 32 | tf = NamedTemporaryFile(mode='w', delete=False) 33 | tf.close() 34 | try: 35 | sensor = make_test_output_thing(1, stop_after_events=NUM_EVENTS) 36 | capture = CaptureInputThing() 37 | sensor.connect(capture) 38 | sensor.csv_writer(tf.name) 39 | scheduler = Scheduler(asyncio.get_event_loop()) 40 | scheduler.schedule_recurring(sensor) 41 | print("Writing sensor events to temp file") 42 | scheduler.run_forever() 43 | self.assertTrue(capture.completed, "CaptureInputThing did not complete") 44 | self.assertEqual(len(capture.events), NUM_EVENTS, 45 | "number of events captured did not match generated events") 46 | reader = CsvReader(tf.name) 47 | vs = SensorEventValidationInputThing(capture.events, self) 48 | reader.connect(vs) 49 | scheduler.schedule_recurring(reader) 50 | print("reading sensor events back from temp file") 51 | scheduler.run_forever() 52 | self.assertTrue(vs.completed, "ValidationInputThing did not complete") 53 | finally: 54 | os.remove(tf.name) 55 | 56 | # data for rollover test 57 | ROLLING_FILE1 = 'dining-room-2015-01-01.csv' 58 | ROLLING_FILE2 = 'dining-room-2015-01-02.csv' 59 | FILES = [ROLLING_FILE1, ROLLING_FILE2] 60 | def make_ts(day, hr, minute): 61 | return (datetime.datetime(2015, 1, day, hr, minute) - datetime.datetime(1970,1,1)).total_seconds() 62 | EVENTS = [SensorEvent('dining-room', make_ts(1, 11, 1), 1), 63 | SensorEvent('dining-room', make_ts(1, 11, 2), 2), 64 | SensorEvent('dining-room', make_ts(2, 11, 1), 3), 65 | SensorEvent('dining-room', make_ts(2, 11, 2), 4)] 66 | 67 | # data for dispatch test 68 | sensor_ids = ['dining-room', 'living-room'] 69 | ROLLING_FILE3 = 'living-room-2015-01-01.csv' 70 | ROLLING_FILE4 = 'living-room-2015-01-02.csv' 71 | FILES2 = [ROLLING_FILE1, ROLLING_FILE2, ROLLING_FILE3, ROLLING_FILE4] 72 | EVENTS2 = [SensorEvent('dining-room', make_ts(1, 11, 1), 1), 73 | SensorEvent('living-room', make_ts(1, 11, 2), 2), 74 | SensorEvent('living-room', make_ts(2, 11, 1), 3), 75 | SensorEvent('dining-room', make_ts(2, 11, 2), 4)] 76 | def make_rule(sensor_id): 77 | return (lambda evt: evt.sensor_id==sensor_id, sensor_id) 78 | dispatch_rules = [make_rule(s) for s in sensor_ids] 79 | 80 | 81 | class TestRollingCsvWriter(unittest.TestCase): 82 | def _cleanup(self): 83 | for f in FILES2: 84 | if os.path.exists(f): 85 | os.remove(f) 86 | 87 | def setUp(self): 88 | self._cleanup() 89 | 90 | def tearDown(self): 91 | self._cleanup() 92 | 93 | def test_rollover(self): 94 | def generator(): 95 | for e in EVENTS: 96 | yield e 97 | sensor = IterableAsOutputThing(generator(), name='sensor') 98 | sensor.rolling_csv_writer('.', 'dining-room') 99 | vs = SensorEventValidationInputThing(EVENTS, self) 100 | sensor.connect(vs) 101 | scheduler = Scheduler(asyncio.get_event_loop()) 102 | scheduler.schedule_recurring(sensor) 103 | scheduler.run_forever() 104 | for f in FILES: 105 | self.assertTrue(os.path.exists(f), 'did not find file %s' % f) 106 | print("found log file %s" % f) 107 | 108 | def test_dispatch(self): 109 | """Test a scenario where we dispatch to one of several writers 110 | depending on the sensor id. 111 | """ 112 | def generator(): 113 | for e in EVENTS2: 114 | yield e 115 | sensor = IterableAsOutputThing(generator(), name='sensor') 116 | dispatcher = sensor.dispatch(dispatch_rules) 117 | for s in sensor_ids: 118 | dispatcher.rolling_csv_writer('.', s, sub_port=s) 119 | dispatcher.connect(lambda x: self.assertTrue(False, "bad dispatch of %s" % x)) 120 | scheduler = Scheduler(asyncio.get_event_loop()) 121 | scheduler.schedule_recurring(sensor) 122 | scheduler.run_forever() 123 | for f in FILES2: 124 | self.assertTrue(os.path.exists(f), 'did not find file %s' % f) 125 | cnt = 0 126 | with open(f, 'r') as fobj: 127 | for line in fobj: 128 | cnt +=1 129 | self.assertEqual(2, cnt, "File %s did not have 2 lines" % f) 130 | print("found log file %s" % f) 131 | 132 | 133 | 134 | if __name__ == '__main__': 135 | unittest.main() 136 | 137 | 138 | -------------------------------------------------------------------------------- /tests/test_descheduling.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test that, when all downstream input things are disconnected, 3 | the schedule deschedules the OutputThing. 4 | """ 5 | 6 | import asyncio 7 | import unittest 8 | from thingflow.base import InputThing, SensorAsOutputThing, Scheduler 9 | 10 | from utils import RandomSensor 11 | 12 | 13 | class PrintAndDeschedule(InputThing): 14 | def __init__(self, prev_in_chain, num_events=5): 15 | self.disconnect = prev_in_chain.connect(self) 16 | self.num_events = num_events 17 | self.prev_in_chain = prev_in_chain 18 | self.count = 0 19 | 20 | def on_next(self, x): 21 | print(x) 22 | self.count+=1 23 | if self.count==self.num_events: 24 | print("Disconnecting from sensor") 25 | self.disconnect() 26 | print(self.prev_in_chain.__connections__) 27 | 28 | def on_completed(self): 29 | print("on_completed") 30 | 31 | def on_error(self, e): 32 | print("on_error(%s)" % e) 33 | 34 | class TestDescheduling(unittest.TestCase): 35 | def test_recurring(self): 36 | s = SensorAsOutputThing(RandomSensor(1)) 37 | PrintAndDeschedule(s) 38 | scheduler = Scheduler(asyncio.get_event_loop()) 39 | scheduler.schedule_recurring(s) 40 | scheduler.run_forever() 41 | print("Exited successfully") 42 | 43 | 44 | def test_periodic(self): 45 | s = SensorAsOutputThing(RandomSensor(1)) 46 | PrintAndDeschedule(s) 47 | scheduler = Scheduler(asyncio.get_event_loop()) 48 | scheduler.schedule_periodic(s, 0.25) 49 | scheduler.run_forever() 50 | print("Exited successfully") 51 | 52 | if __name__ == '__main__': 53 | unittest.main() 54 | 55 | 56 | -------------------------------------------------------------------------------- /tests/test_external_event_stream.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Run an observable that has its own event loop. 4 | We use the MQTT reader with the mock client to test this 5 | (does not require an mqtt broker). 6 | """ 7 | import thingflow.filters.output 8 | from thingflow.base import Scheduler, InputThing 9 | from thingflow.adapters.mqtt import MQTTReader, MockMQTTClient 10 | 11 | import unittest 12 | import asyncio 13 | 14 | 15 | class StopLoopAfter(InputThing): 16 | def __init__(self, stop_after, cancel_thunk): 17 | self.events_left = stop_after 18 | self.cancel_thunk = cancel_thunk 19 | 20 | def on_next(self, x): 21 | self.events_left -= 1 22 | if self.events_left == 0: 23 | print("Requesting stop of event loop") 24 | self.cancel_thunk() 25 | 26 | class TestExternalEventStream(unittest.TestCase): 27 | def test_case(self): 28 | """Just run the reader in its own event loop. We stop everything after 4 29 | events. 30 | """ 31 | s = Scheduler(asyncio.get_event_loop()) 32 | m = MQTTReader("localhost", topics=[('bogus/bogus', 0),], 33 | mock_class=MockMQTTClient) 34 | m.output() 35 | c = s.schedule_on_private_event_loop(m) 36 | m.connect(StopLoopAfter(4, c)) 37 | m.print_downstream() 38 | s.run_forever() 39 | print("that's it") 40 | 41 | 42 | if __name__ == '__main__': 43 | unittest.main() 44 | 45 | -------------------------------------------------------------------------------- /tests/test_fatal_error_handling.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Test that a fatal error causes the scheduler to exit. 4 | """ 5 | from thingflow.base import * 6 | from utils import make_test_output_thing 7 | 8 | import sys 9 | import asyncio 10 | import unittest 11 | 12 | class DieAfter(InputThing): 13 | def __init__(self, num_events): 14 | self.events_left = num_events 15 | 16 | def on_next(self, x): 17 | self.events_left -= 1 18 | if self.events_left == 0: 19 | print("throwing fatal error") 20 | raise FatalError("this is a fatal error") 21 | 22 | class TestFatalErrorHandling(unittest.TestCase): 23 | def test_case(self): 24 | sensor = make_test_output_thing(1) 25 | sensor.connect(print) 26 | sensor2 = make_test_output_thing(2) 27 | sensor2.connect(print) 28 | s = Scheduler(asyncio.get_event_loop()) 29 | s.schedule_periodic(sensor, 1) 30 | s.schedule_periodic(sensor2, 1) 31 | sensor.connect(DieAfter(4)) 32 | sensor.print_downstream() 33 | try: 34 | s.run_forever() 35 | except FatalError: 36 | print("got to end with fatal error thrown as expected") 37 | else: 38 | self.assertFalse(1, "Did not get to a fatal error") 39 | 40 | if __name__ == '__main__': 41 | unittest.main() 42 | -------------------------------------------------------------------------------- /tests/test_fatal_error_in_private_loop.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Run a private event loop and then throw a fatal error in it to verify that 4 | we shut down cleanly and don't lose the error. 5 | """ 6 | 7 | import thingflow.filters.output 8 | from thingflow.base import Scheduler, OutputThing, EventLoopOutputThingMixin, FatalError 9 | 10 | import unittest 11 | import asyncio 12 | s = Scheduler(asyncio.get_event_loop()) 13 | 14 | import time 15 | 16 | class TestOutputThing(OutputThing, EventLoopOutputThingMixin): 17 | def __init__(self): 18 | super().__init__() 19 | 20 | def _observe_event_loop(self): 21 | print("starting event loop") 22 | for i in range(4): 23 | if self.stop_requested: 24 | break 25 | self._dispatch_next(i) 26 | time.sleep(1) 27 | raise FatalError("testing the fatal error") 28 | 29 | class TestFatalErrorInPrivateLoop(unittest.TestCase): 30 | def test_case(self): 31 | m = TestOutputThing() 32 | m.output() 33 | c = s.schedule_on_private_event_loop(m) 34 | m.print_downstream() 35 | try: 36 | s.run_forever() 37 | except FatalError: 38 | print("we got the fatal error as expected") 39 | else: 40 | print("The event loop exited without throwing a fatal error!") 41 | self.assertFalse(1, "The event loop exited without throwing a fatal error!") 42 | 43 | if __name__ == '__main__': 44 | unittest.main() 45 | 46 | 47 | -------------------------------------------------------------------------------- /tests/test_functional_api.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Tests for the functional-style api 4 | """ 5 | import asyncio 6 | import unittest 7 | import sys 8 | 9 | from utils import ValueListSensor, ValidationInputThing 10 | 11 | from thingflow.base import Scheduler, SensorAsOutputThing 12 | from thingflow.filters.output import output 13 | from thingflow.filters.select import map 14 | from thingflow.filters.where import where 15 | from thingflow.filters.combinators import passthrough, compose 16 | 17 | lux_data = [100, 200, 300, 450, 100, 200, 600] 18 | 19 | 20 | class TestFunctionalApi(unittest.TestCase): 21 | """These are tests of the various features of the 22 | functional-style API. We want to make certain they 23 | work as advertised. 24 | """ 25 | def test_complex_workflow(self): 26 | THRESHOLD = 300 27 | lux = ValueListSensor('lux-1', lux_data) 28 | scheduler = Scheduler(asyncio.get_event_loop()) 29 | vs1 = ValidationInputThing(lux_data, self) 30 | vs2 = ValidationInputThing([False, False, False, True, False, False, True], 31 | self, 32 | extract_value_fn=lambda v: v) 33 | scheduler.schedule_sensor(lux, 0.5, 34 | passthrough(output()), 35 | passthrough(vs1), 36 | map(lambda event:event.val > THRESHOLD), 37 | passthrough(lambda v: print('ON' if v else 'OFF')), 38 | vs2, print_downstream=True) 39 | scheduler.run_forever() 40 | self.assertTrue(vs1.completed) 41 | self.assertTrue(vs2.completed) 42 | print("That's all folks") 43 | 44 | def test_thunk_builder_handling(self): 45 | """We have logic where we can pass a thunk builder into a combinator and 46 | it will do the right thing. Check that it works""" 47 | scheduler = Scheduler(asyncio.get_event_loop()) 48 | lux = ValueListSensor('lux-2', lux_data) 49 | vs = ValidationInputThing(lux_data, self) 50 | scheduler.schedule_sensor(lux, 0.5, 51 | passthrough(output()), # output() evaluates to a thunk 52 | passthrough(output), # output is a thunk builder 53 | passthrough(output(sys.stdout)), # output can take an arg 54 | vs, print_downstream=True) 55 | scheduler.run_forever() 56 | self.assertTrue(vs.completed) 57 | 58 | def test_passthrough_as_a_method(self): 59 | """Verify that, when passthrough is used as a method, it can still take 60 | thunks. 61 | """ 62 | scheduler = Scheduler(asyncio.get_event_loop()) 63 | luxpub = SensorAsOutputThing(ValueListSensor('lux-2', lux_data)) 64 | vs1 = ValidationInputThing([450, 600], self) 65 | vs2 = ValidationInputThing(lux_data, self) 66 | luxpub.passthrough(compose(where(lambda evt: evt.val>300), vs1)).connect(vs2) 67 | scheduler.schedule_periodic(luxpub, 0.5) 68 | scheduler.run_forever() 69 | self.assertTrue(vs1.completed) 70 | self.assertTrue(vs2.completed) 71 | 72 | if __name__ == '__main__': 73 | unittest.main() 74 | 75 | -------------------------------------------------------------------------------- /tests/test_influxdb.py: -------------------------------------------------------------------------------- 1 | # Make sure influxdb is running 2 | 3 | try: 4 | from influxdb import InfluxDBClient 5 | from thingflow.adapters.influxdb import InfluxDBWriter,\ 6 | InfluxDBReader 7 | PREREQS_AVAILABLE=True 8 | except ImportError: 9 | PREREQS_AVAILABLE = False 10 | 11 | try: 12 | from config_for_tests import INFLUXDB_USER, INFLUXDB_PASSWORD 13 | except ImportError: 14 | INFLUXDB_USER=None 15 | INFLUXDB_PASSWORD=None 16 | try: 17 | from config_for_tests import INFLUXDB_DATABASE 18 | except ImportError: 19 | INFLUXDB_DATABASE='thingflow' # the default 20 | 21 | import asyncio 22 | import datetime, time 23 | from collections import namedtuple 24 | import unittest 25 | 26 | from utils import ValueListSensor 27 | from thingflow.base import Scheduler, SensorAsOutputThing, \ 28 | SensorEvent, CallableAsInputThing 29 | 30 | 31 | 32 | Sensor = namedtuple('Sensor', ['series_name', 'fields', 'tags']) 33 | 34 | value_stream = [10, 13, 20, 20, 19, 19, 20, 21, 28, 28, 23, 21, 21, 18, 19, 16, 21, 35 | 10, 13, 20, 20, 19, 19, 20, 21, 28, 28, 23, 21, 21, 18, 19, 16, 21] 36 | value_stream2 = [2, 3, 2, 2, 9, 9, 2, 1, 8, 8, 3, 2, 1, 8, 9, 6, 2, 2, 3, 4, 5, 6, 7, 8, 2, 3, 4, 37 | 6, 2, 3, 2, 2, 9, 9, 2, 1, 8, 8, 3, 2, 1, 8, 9, 6, 2, 2, 3, 4, 5, 6, 7, 8, 2, 3, 4, 6, 38 | 6, 2, 3, 2, 2, 9, 9, 2, 1, 8, 8, 3, 2, 1, 8, 9, 6, 2, 2, 3, 4, 5, 6, 7, 8, 2, 3, 4, 6, 39 | 6, 2, 3, 2, 2, 9, 9, 2, 1, 8, 8, 3, 2, 1, 8, 9, 6, 2, 2, 3, 4, 5, 6, 7, 8, 2, 3, 4, 6, 40 | 6, 2, 3, 2, 2, 9, 9, 2, 1, 8, 8, 3, 2, 1, 8, 9, 6, 2, 2, 3, 4, 5, 6, 7, 8, 2, 3, 4, 6] 41 | 42 | @unittest.skipUnless(PREREQS_AVAILABLE, 43 | "influxdb client library not installed") 44 | @unittest.skipUnless(INFLUXDB_USER is not None, 45 | "Influxdb not configured in config_for_tests.py") 46 | class TestInflux(unittest.TestCase): 47 | def test_influx_output(self): 48 | loop = asyncio.get_event_loop() 49 | s = ValueListSensor(1, value_stream) 50 | p = SensorAsOutputThing(s) 51 | b = InfluxDBWriter(msg_format=Sensor(series_name='Sensor', fields=['val', 'ts'], tags=['sensor_id']), 52 | generate_timestamp=False, 53 | username=INFLUXDB_USER, 54 | password=INFLUXDB_PASSWORD, 55 | database=INFLUXDB_DATABASE) 56 | p.connect(b) 57 | 58 | scheduler = Scheduler(loop) 59 | scheduler.schedule_periodic(p, 0.2) # sample five times every second 60 | scheduler.run_forever() 61 | 62 | # Now play back 63 | rs = self.c.query('SELECT * FROM Sensor;').get_points() 64 | for d in rs: 65 | print(d) 66 | 67 | # Play back using an output thing 68 | p = InfluxDBReader('SELECT * FROM Sensor;', 69 | database=INFLUXDB_DATABASE, 70 | username=INFLUXDB_USER, 71 | password=INFLUXDB_PASSWORD) 72 | p.connect(CallableAsInputThing(print)) 73 | 74 | scheduler = Scheduler(loop) 75 | scheduler.schedule_periodic(p, 0.2) # sample five times every second 76 | scheduler.run_forever() 77 | print("That's all folks") 78 | 79 | def setUp(self): 80 | self.c = InfluxDBClient(database=INFLUXDB_DATABASE, 81 | username=INFLUXDB_USER, 82 | password=INFLUXDB_PASSWORD) 83 | self.c.delete_series(measurement='Sensor') 84 | #self.c.query('DELETE from Sensor;') 85 | 86 | def tearDown(self): 87 | self.c.delete_series(measurement='Sensor') 88 | #self.c.query('DELETE from Sensor;') 89 | 90 | 91 | if __name__ == '__main__': 92 | unittest.main() 93 | -------------------------------------------------------------------------------- /tests/test_iterable_as_output_thing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Test of IterableAsOutputThing. This was originally test_base.py, but we then 4 | added the sensor infrastructure and rewrote the test. This test verfies the 5 | specific IterableAsOutputThing code. 6 | """ 7 | 8 | import asyncio 9 | import unittest 10 | 11 | from thingflow.base import Scheduler, IterableAsOutputThing 12 | from utils import make_test_output_thing_from_vallist, ValidationInputThing 13 | import thingflow.filters.where 14 | import thingflow.filters.output 15 | 16 | value_stream = [ 17 | 20, 18 | 30, 19 | 100, 20 | 120, 21 | 20, 22 | 5, 23 | 2222 24 | ] 25 | 26 | expected_stream = [ 27 | 100, 28 | 120, 29 | 2222 30 | ] 31 | 32 | def predicate(v): 33 | if v[2]>=100.0: 34 | print("v=%s, True" % v[2]) 35 | return True 36 | else: 37 | print("v=%s, False" % v[2]) 38 | return False 39 | 40 | class ErrorIterator: 41 | """An iterator that thows an error after the initial stream 42 | (instead of StopIteration). 43 | """ 44 | def __init__(self, expected_stream): 45 | self.expected_stream = expected_stream 46 | self.idx = 0 47 | 48 | def __iterator__(self): 49 | return self 50 | 51 | def __next__(self): 52 | if self.idx==len(self.expected_stream): 53 | raise Exception("Throwing an exception in ErrorIterator") 54 | else: 55 | v = self.expected_stream[self.idx] 56 | self.idx += 1 57 | return v 58 | 59 | 60 | class TestIterableAsOutputThing(unittest.TestCase): 61 | def test_where(self): 62 | s = make_test_output_thing_from_vallist(1, value_stream) 63 | w = s.where(predicate) 64 | w.output() 65 | vo = ValidationInputThing(expected_stream, self) 66 | w.connect(vo) 67 | scheduler = Scheduler(asyncio.get_event_loop()) 68 | scheduler.schedule_periodic(s, 0.5) # sample twice every second 69 | s.print_downstream() 70 | scheduler.run_forever() 71 | self.assertTrue(vo.completed, 72 | "Schedule exited before validation observer completed") 73 | self.assertTrue(vo.completed) 74 | print("That's all folks") 75 | 76 | def test_error_handling(self): 77 | """This is a non-fatal error, so we should just print the error and 78 | exit cleanly without propagating the exception. 79 | """ 80 | s = IterableAsOutputThing(ErrorIterator(expected_stream)) 81 | s.output() 82 | scheduler = Scheduler(asyncio.get_event_loop()) 83 | scheduler.schedule_periodic(s, 0.5) # sample twice every second 84 | s.print_downstream() 85 | scheduler.run_forever() 86 | 87 | 88 | if __name__ == '__main__': 89 | unittest.main() 90 | 91 | -------------------------------------------------------------------------------- /tests/test_linq.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Tests of the linq apis. Pretty much still manually verified, although running 4 | it as a part of the automated test suite makes a decent regression test. 5 | """ 6 | import asyncio 7 | import unittest 8 | 9 | from thingflow.base import * 10 | from utils import make_test_output_thing, make_test_output_thing_from_vallist,\ 11 | ValidationInputThing 12 | import thingflow.filters.where 13 | import thingflow.filters.output 14 | import thingflow.filters 15 | from thingflow.filters.never import Never 16 | 17 | def pp_buf(x): 18 | print("Buffered output: ", x) 19 | print("\n") 20 | 21 | 22 | class TestLinq(unittest.TestCase): 23 | def test_case(self): 24 | """Rupak, if you want to test more, just add it here or add additional 25 | methods starting with test_ 26 | """ 27 | loop = asyncio.get_event_loop() 28 | 29 | s = make_test_output_thing(1, stop_after_events=5) 30 | 31 | t = s.skip(2).some(lambda x: x[2]>100) 32 | 33 | s.connect(print) 34 | t.connect(print) 35 | 36 | scheduler = Scheduler(loop) 37 | scheduler.schedule_periodic(s, 2) # sample once every 2 seconds 38 | 39 | 40 | u = s.take_last(3).scan(lambda a, x: a+x[2], 0) 41 | u.connect(print) 42 | v = s.take_last(3).reduce(lambda a, x: a+x[2], 0) 43 | v.connect(print) 44 | 45 | w = s.buffer_with_time(5, scheduler) 46 | w.connect(pp_buf) 47 | # w = Never() 48 | # w.connect(print) 49 | # scheduler.schedule_periodic(w, 1) 50 | 51 | s.print_downstream() 52 | 53 | loop.call_later(30, scheduler.stop) 54 | 55 | scheduler.run_forever() 56 | print("That's all folks") 57 | 58 | def test_first(self): 59 | """Test the first() operator 60 | """ 61 | p = make_test_output_thing_from_vallist(1, [1, 2, 3, 4, 5, 6]) 62 | vs = ValidationInputThing([1], self) 63 | p.first().connect(vs) 64 | scheduler = Scheduler(asyncio.get_event_loop()) 65 | scheduler.schedule_recurring(p) 66 | scheduler.run_forever() 67 | self.assertTrue(vs.completed) 68 | 69 | 70 | if __name__ == '__main__': 71 | unittest.main() 72 | 73 | -------------------------------------------------------------------------------- /tests/test_mqtt.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Test mqtt broker 4 | 5 | In addition to testing mqtt publish/subscribe functionality, this runs a 6 | output_thing that has its own event loop. 7 | 8 | To run the test, you will need the paho-mqtt client and the mosquitto broker. 9 | You can get the client via: 10 | 11 | pip install paho-mqtt 12 | 13 | On Debian-based linuxes, you can get the broker via: 14 | 15 | sudo apt-get install mosquitto 16 | 17 | We assume that the broker is listening on localhost:1883. 18 | 19 | """ 20 | 21 | import unittest 22 | import sys 23 | import thingflow.filters.output 24 | import thingflow.filters.json 25 | import thingflow.filters.select 26 | from thingflow.base import Scheduler, InputThing, SensorEvent, ScheduleError, ExcInDispatch 27 | from thingflow.adapters.mqtt import MQTTReader, MQTTWriter 28 | from utils import make_test_output_thing_from_vallist, ValidationInputThing 29 | 30 | try: 31 | import paho.mqtt 32 | MQTT_CLIENT_AVAILABLE = True 33 | except ImportError: 34 | MQTT_CLIENT_AVAILABLE = False 35 | 36 | MQTT_PORT=1883 37 | 38 | import asyncio 39 | 40 | sensor_data = [1, 2, 3, 4, 5] 41 | 42 | class StopLoopAfter(InputThing): 43 | def __init__(self, stop_after, cancel_thunk): 44 | self.events_left = stop_after 45 | self.cancel_thunk = cancel_thunk 46 | 47 | def on_next(self, x): 48 | self.events_left -= 1 49 | if self.events_left == 0: 50 | print("Requesting stop of event loop") 51 | self.cancel_thunk() 52 | 53 | def mqtt_msg_to_unicode(m): 54 | v = (m.payload).decode("utf-8") 55 | return v 56 | 57 | 58 | def is_broker_running(): 59 | import subprocess 60 | rc = subprocess.call("netstat -an | grep %d" % MQTT_PORT, shell=True) 61 | if rc==0: 62 | print("MQTT broker running") 63 | return True 64 | else: 65 | print("MQTT broker not running") 66 | return False 67 | 68 | 69 | @unittest.skipUnless(MQTT_CLIENT_AVAILABLE, 70 | "MQTT client not installed for python at %s" % sys.executable) 71 | @unittest.skipUnless(is_broker_running(), 72 | "MQTT broker not running on port %d" % MQTT_PORT) 73 | class TestCase(unittest.TestCase): 74 | def test_mqtt(self): 75 | loop = asyncio.get_event_loop() 76 | s = Scheduler(loop) 77 | sensor = make_test_output_thing_from_vallist(1, sensor_data) 78 | mqtt_writer = MQTTWriter('localhost', topics=[('bogus/bogus',0),]) 79 | sensor.to_json().connect(mqtt_writer) 80 | s.schedule_periodic(sensor, 0.5) 81 | 82 | mqtt_reader = MQTTReader("localhost", topics=[('bogus/bogus', 0),]) 83 | vs = ValidationInputThing(sensor_data, self) 84 | mqtt_reader.take(5).select(mqtt_msg_to_unicode).from_json(constructor=SensorEvent) \ 85 | .output().connect(vs) 86 | c = s.schedule_on_private_event_loop(mqtt_reader) 87 | stop = StopLoopAfter(5, c) 88 | mqtt_reader.connect(stop) 89 | mqtt_reader.print_downstream() 90 | sensor.print_downstream() 91 | s.run_forever() 92 | loop.stop() 93 | self.assertTrue(vs.completed) 94 | print("that's it") 95 | 96 | def test_daniels_bug(self): 97 | """Test bug reported by Daniel (issue #1). If you call the mqtt writer without 98 | serializing the message, you should get a fatal error. 99 | """ 100 | import time 101 | import asyncio 102 | import thingflow.filters.output # This has output side-effect 103 | from thingflow.base import Scheduler, from_list 104 | from thingflow.adapters.mqtt import MQTTReader, MQTTWriter 105 | from collections import namedtuple 106 | 107 | StripEvent = namedtuple('StripEvent', ['strip_id', 'ts', 'val']) 108 | 109 | strip_events = ( 110 | StripEvent('strip-1', 1500000000, 50), 111 | StripEvent('strip-1', 1500000000, 5), 112 | StripEvent('strip-1', 1500000000, 50)) 113 | 114 | mqtt = MQTTWriter('localhost', topics=[('strip-data', 0),]) 115 | 116 | strip = from_list(strip_events) 117 | strip.connect(mqtt) 118 | strip.output() 119 | 120 | sched = Scheduler(asyncio.get_event_loop()) 121 | sched.schedule_periodic(strip, 1.0) 122 | try: 123 | sched.run_forever() 124 | except ScheduleError as e: 125 | # verify the cause of the error 126 | dispatch_error = e.__cause__ 127 | self.assertTrue(isinstance(dispatch_error, ExcInDispatch), 128 | "expecting cause to be a dispatch error, instead got %s" % repr(dispatch_error)) 129 | orig_error = dispatch_error.__cause__ 130 | self.assertTrue(isinstance(orig_error, TypeError), 131 | "expecting original exception to be a TypeError, intead got %s" % repr(orig_error)) 132 | print("Got expected exception: '%s'" % e) 133 | 134 | if __name__ == '__main__': 135 | unittest.main() 136 | -------------------------------------------------------------------------------- /tests/test_mqtt_async.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Test async version of mqtt libraries. Depends on hbmqtt 4 | (https://github.com/beerfactory/hbmqtt) 5 | 6 | """ 7 | 8 | import unittest 9 | import sys 10 | import asyncio 11 | import string 12 | from random import choice, seed 13 | from thingflow.base import Scheduler, SensorAsOutputThing, SensorEvent 14 | import thingflow.filters.output 15 | import thingflow.filters.combinators 16 | import thingflow.filters.select 17 | from thingflow.filters.transducer import PeriodicMedianTransducer 18 | from utils import ValueListSensor, ValidateAndStopInputThing 19 | 20 | seed() 21 | 22 | try: 23 | import hbmqtt 24 | from thingflow.adapters.mqtt_async import QueueWriter, QueueReader 25 | HBMQTT_AVAILABLE = True 26 | except ImportError: 27 | HBMQTT_AVAILABLE = False 28 | 29 | URL = "mqtt://localhost:1883" 30 | 31 | 32 | VALUES = [ 33 | 1.0, 34 | 2.5, 35 | 3.7, 36 | 4.1, 37 | 8.1, 38 | 0.5, 39 | 6.5, 40 | 4.5, 41 | 3.9, 42 | 6.5 43 | ] 44 | 45 | EXPECTED = [ 46 | 2.5, 47 | 4.1, 48 | 4.5, 49 | 6.5 50 | ] 51 | 52 | def msg_to_event(msg): 53 | return SensorEvent(sensor_id=msg[0], ts=msg[1], val=msg[2]) 54 | 55 | CHARS=string.ascii_letters+string.digits 56 | def get_topic_name(test_class): 57 | return test_class.__class__.__name__ + ''.join([ choice(CHARS) for i in range(5) ]) 58 | 59 | @unittest.skipUnless(HBMQTT_AVAILABLE, 60 | "HBMQTT library not installed for python at %s" % 61 | sys.executable) 62 | class TestCase(unittest.TestCase): 63 | def setUp(self): 64 | # Creating a new event loop each test case does not seem to work. 65 | # I think it is due to hbmqtt not cleaning up some state in the asyncio 66 | # layer. 67 | #self.loop = asyncio.new_event_loop() 68 | self.loop = asyncio.get_event_loop() 69 | self.sched = Scheduler(self.loop) 70 | def tearDown(self): 71 | pass 72 | #self.loop.stop() 73 | #self.loop.close() 74 | 75 | def test_client_only(self): 76 | SENSOR_ID='sensor-1' 77 | TOPIC=get_topic_name(self) 78 | sensor = SensorAsOutputThing(ValueListSensor(SENSOR_ID, VALUES)) 79 | td = sensor.transduce(PeriodicMedianTransducer(period=3)) 80 | qw = QueueWriter(td, URL, TOPIC, self.sched) 81 | qw.output() 82 | self.sched.schedule_periodic(sensor, 0.5) 83 | self.sched.run_forever() 84 | self.assertFalse(qw.has_pending_requests(), 85 | "QueueWriter has pending requests: %s" % qw.dump_state()) 86 | print("test_client_only completed") 87 | 88 | def send_and_recv_body(self, sleep_timeout): 89 | SENSOR_ID='sensor-1' 90 | TOPIC=get_topic_name(self) 91 | sensor = SensorAsOutputThing(ValueListSensor(SENSOR_ID, VALUES)) 92 | td = sensor.transduce(PeriodicMedianTransducer(period=3)) 93 | qw = QueueWriter(td, URL, TOPIC, self.sched) 94 | qw.output() 95 | qr = QueueReader(URL, TOPIC, self.sched, timeout=sleep_timeout) 96 | self.sched.schedule_periodic(sensor, 0.5) 97 | stop_qr = self.sched.schedule_on_main_event_loop(qr) 98 | vs = ValidateAndStopInputThing(EXPECTED, self, stop_qr) 99 | qr.select(msg_to_event).connect(vs) 100 | self.sched.run_forever() 101 | self.assertFalse(qw.has_pending_requests(), 102 | "QueueWriter has pending requests: %s" % qw.dump_state()) 103 | self.assertEqual(qr.state, QueueReader.FINAL_STATE) 104 | self.assertEqual(vs.next_idx, len(EXPECTED)) 105 | print("send_and_recv_bod(%s) completed" % sleep_timeout) 106 | 107 | def test_short_timeout(self): 108 | self.send_and_recv_body(0.1) 109 | 110 | def test_long_timeout(self): 111 | self.send_and_recv_body(3.0) 112 | 113 | 114 | if __name__ == '__main__': 115 | unittest.main() 116 | 117 | -------------------------------------------------------------------------------- /tests/test_multiple_output_ports.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | Build a filter that takes an input stream and dispatches to one of several 5 | output ports based on the input value. 6 | """ 7 | 8 | import asyncio 9 | import unittest 10 | 11 | from thingflow.base import OutputThing, InputThing, Scheduler 12 | from utils import make_test_output_thing 13 | import thingflow.filters.where 14 | import thingflow.filters.output 15 | 16 | class SplitOutputThing(OutputThing, InputThing): 17 | """Here is a filter that takes a sequence of sensor events as its input 18 | and the splits it into one of three output ports: 'below' if the 19 | value is below one standard deviation from the mean, 'above' 20 | if the value is above one standard deviation from the mean, and 21 | 'within' if the value is within a standard deviation from the mean. 22 | """ 23 | def __init__(self, mean=100.0, stddev=20.0): 24 | OutputThing.__init__(self, ports=['above', 'below', 'within']) 25 | self.mean = mean 26 | self.stddev = stddev 27 | 28 | def on_next(self, x): 29 | val = x[2] 30 | if val < (self.mean-self.stddev): 31 | #print("split: value=%s dispatching to below" % val) 32 | self._dispatch_next(val, port='below') 33 | elif val > (self.mean+self.stddev): 34 | #print("split: value=%s dispatching to above" % val) 35 | self._dispatch_next(val, port='above') 36 | else: 37 | #print("split: value=%s dispatching to within" % val) 38 | self._dispatch_next(val, port='within') 39 | 40 | def __str__(self): 41 | return "SplitOutputThing" 42 | 43 | class TestMultiplePubports(unittest.TestCase): 44 | def test_case(self): 45 | sensor = make_test_output_thing(1, stop_after_events=10) 46 | split= SplitOutputThing() 47 | sensor.connect(split) 48 | split.connect(lambda x: print("above:%s" % x), 49 | port_mapping=('above','default')) 50 | split.connect(lambda x: print("below:%s" % x), 51 | port_mapping=('below', 'default')) 52 | split.connect(lambda x: print("within:%s" % x), 53 | port_mapping=('within', 'default')) 54 | 55 | scheduler = Scheduler(asyncio.get_event_loop()) 56 | scheduler.schedule_periodic(sensor, 1) 57 | 58 | sensor.print_downstream() 59 | scheduler.run_forever() 60 | print("that's all") 61 | 62 | if __name__ == '__main__': 63 | unittest.main() 64 | 65 | 66 | -------------------------------------------------------------------------------- /tests/test_pandas.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test the adapter to pandas (data analysis library) 3 | """ 4 | 5 | import asyncio 6 | import unittest 7 | 8 | try: 9 | import pandas 10 | PANDAS_AVAILABLE=True 11 | except: 12 | PANDAS_AVAILABLE = False 13 | 14 | from utils import ValueListSensor 15 | from thingflow.base import Scheduler, SensorAsOutputThing 16 | 17 | value_stream = [ 18 | 20, 19 | 30, 20 | 100, 21 | 120, 22 | 20, 23 | 5, 24 | 2222 25 | ] 26 | 27 | @unittest.skipUnless(PANDAS_AVAILABLE, "pandas library not installed") 28 | class TestPandas(unittest.TestCase): 29 | def test_pandas(self): 30 | s = ValueListSensor(1, value_stream) 31 | p = SensorAsOutputThing(s) 32 | import thingflow.adapters.pandas 33 | import numpy 34 | w =thingflow.adapters.pandas.PandasSeriesWriter() 35 | p.connect(w) 36 | sch = Scheduler(asyncio.get_event_loop()) 37 | sch.schedule_recurring(p) 38 | sch.run_forever() 39 | self.assertTrue(w.result is not None, "Result of pandas never set") 40 | # now we verify each element 41 | for (i, v) in enumerate(value_stream): 42 | pv = w.result[i] 43 | self.assertTrue(isinstance(pv, numpy.int64), 44 | "Expecting pandas value '%s' to be numpy.int64, but instead was %s" % 45 | (pv, repr(type(pv)))) 46 | self.assertEqual(v, pv, 47 | "Pandas value '%s' not equal to original value '%s'" % 48 | (repr(pv), repr(v))) 49 | print("Validate pandas array") 50 | 51 | if __name__ == '__main__': 52 | unittest.main() 53 | 54 | 55 | -------------------------------------------------------------------------------- /tests/test_postgres_adapters.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | Test the postgres adapters 5 | """ 6 | try: 7 | import psycopg2 8 | PREREQS_AVAILABLE = True 9 | except ImportError: 10 | PREREQS_AVAILABLE = False 11 | try: 12 | from config_for_tests import POSTGRES_DBNAME, POSTGRES_USER 13 | except ImportError: 14 | POSTGRES_DBNAME=None 15 | POSTGRES_USER=None 16 | 17 | # Set the following to True to skip the tearDown(). This is useful when 18 | # trying to debug a failing test but should be left at False in 19 | # production. 20 | DEBUG_MODE = False 21 | 22 | import asyncio 23 | import unittest 24 | 25 | from utils import ValueListSensor, SensorEventValidationInputThing 26 | from thingflow.base import Scheduler, InputThing 27 | if PREREQS_AVAILABLE: 28 | from thingflow.adapters.postgres import PostgresWriter, SensorEventMapping,\ 29 | create_sensor_table, delete_sensor_table, PostgresReader 30 | from thingflow.filters.output import output 31 | from thingflow.filters.combinators import parallel 32 | 33 | 34 | sensor_values = [1, 2, 3, 4, 5] 35 | 36 | class CaptureInputThing(InputThing): 37 | def __init__(self): 38 | self.seq = [] 39 | 40 | def on_next(self, x): 41 | self.seq.append(x) 42 | 43 | 44 | 45 | @unittest.skipUnless(PREREQS_AVAILABLE, "postgress client library not installed") 46 | @unittest.skipUnless(POSTGRES_DBNAME and POSTGRES_USER, 47 | "POSTGRES_DBNAME and POSTGRES_USER not defined in config_for_tests") 48 | class TestCase(unittest.TestCase): 49 | def setUp(self): 50 | self.mapping = SensorEventMapping('test_events') 51 | self.connect_string = "dbname=%s user=%s" % (POSTGRES_DBNAME, 52 | POSTGRES_USER) 53 | conn = psycopg2.connect(self.connect_string) 54 | create_sensor_table(conn, 'test_events', drop_if_exists=True) 55 | conn.close() 56 | 57 | def tearDown(self): 58 | if DEBUG_MODE: 59 | print("DEBUG_MODE=True, SKIPPING tearDown()") 60 | return 61 | self.connect_string = "dbname=%s user=%s" % (POSTGRES_DBNAME, 62 | POSTGRES_USER) 63 | conn = psycopg2.connect(self.connect_string) 64 | delete_sensor_table(conn, 'test_events') 65 | conn.close() 66 | 67 | def test_publish_and_subscribe(self): 68 | sensor = ValueListSensor(1, sensor_values) 69 | scheduler = Scheduler(asyncio.get_event_loop()) 70 | pg = PostgresWriter(scheduler, self.connect_string, self.mapping) 71 | capture = CaptureInputThing() 72 | scheduler.schedule_sensor(sensor, 0.5, 73 | parallel(pg, output, capture)) 74 | scheduler.run_forever() 75 | print("finish writing to the database") 76 | row_source = PostgresReader(self.connect_string, self.mapping) 77 | row_source.output() 78 | validate = SensorEventValidationInputThing(capture.seq, self) 79 | row_source.connect(validate) 80 | scheduler.schedule_recurring(row_source) 81 | scheduler.run_forever() 82 | self.assertTrue(validate.completed) 83 | print("finished reading rows") 84 | 85 | 86 | if __name__ == '__main__': 87 | unittest.main() 88 | 89 | -------------------------------------------------------------------------------- /tests/test_rpi_adapters.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests of RPI-specific functionality (adapters and sensors). 3 | """ 4 | 5 | import asyncio 6 | import unittest 7 | import platform 8 | 9 | from utils import ValueListSensor 10 | from thingflow.base import Scheduler, SensorAsOutputThing 11 | import thingflow.filters.map 12 | from thingflow.filters.output import output 13 | from thingflow.filters.combinators import passthrough 14 | from utils import StopAfterN 15 | 16 | MACHINE=platform.machine() 17 | 18 | # Check whether the library for tsl2591 is installed. 19 | # See https://github.com/maxlklaxl/python-tsl2591.git 20 | try: 21 | import tsl2591 22 | TSL2591_INSTALLED=True 23 | except: 24 | TSL2591_INSTALLED=False 25 | 26 | values = [ 27 | 0, 28 | 1, 29 | 0, 30 | 1, 31 | 0, 32 | 1, 33 | 0 34 | ] 35 | 36 | @unittest.skipUnless(MACHINE=="armv7l", 37 | "Tests are specific to RaspberryPi") 38 | class TestRpi(unittest.TestCase): 39 | def test_gpio(self): 40 | import thingflow.adapters.rpi.gpio 41 | o = thingflow.adapters.rpi.gpio.GpioPinOut() 42 | sensor_thing = SensorAsOutputThing(ValueListSensor("sensor-1", values)) 43 | sensor_thing.map(lambda evt: evt.val>0).passthrough(output()).connect(o) 44 | s = Scheduler(asyncio.get_event_loop()) 45 | s.schedule_periodic(sensor_thing, 1.0) 46 | s.run_forever() 47 | 48 | @unittest.skipUnless(TSL2591_INSTALLED, 49 | "TSL2591 sensor library not installed") 50 | def test_tsl2591(self): 51 | import thingflow.sensors.rpi.lux_sensor 52 | sensor = SensorAsOutputThing(thingflow.sensors.rpi.lux_sensor.LuxSensor()) 53 | s = Scheduler(asyncio.get_event_loop()) 54 | stop = s.schedule_periodic(sensor, 1.0) 55 | StopAfterN(sensor, stop, N=4).output() 56 | s.run_forever() 57 | 58 | 59 | 60 | 61 | if __name__ == '__main__': 62 | unittest.main() 63 | 64 | 65 | -------------------------------------------------------------------------------- /tests/test_scheduler_cancel.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Cancel an active schedule. Since this is the last active schedule, it 4 | should cleanly stop the scheduler. 5 | """ 6 | from thingflow.base import * 7 | from utils import make_test_output_thing 8 | 9 | import asyncio 10 | import unittest 11 | 12 | class CallAfter(InputThing): 13 | def __init__(self, num_events, fn): 14 | self.events_left = num_events 15 | self.fn = fn 16 | 17 | def on_next(self, x): 18 | self.events_left -= 1 19 | if self.events_left == 0: 20 | print("calling fn %s" % self.fn) 21 | self.fn() 22 | 23 | class TestSchedulerCancel(unittest.TestCase): 24 | def test_case(self): 25 | sensor = make_test_output_thing(1) 26 | sensor.connect(print) 27 | s = Scheduler(asyncio.get_event_loop()) 28 | cancel_schedule = s.schedule_periodic(sensor, 1) 29 | sensor.connect(CallAfter(4, cancel_schedule)) 30 | sensor.print_downstream() 31 | s.run_forever() 32 | print("got to end") 33 | 34 | if __name__ == '__main__': 35 | unittest.main() 36 | 37 | -------------------------------------------------------------------------------- /tests/test_tcp_stream.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | import asyncio 4 | 5 | from antevents.tcpstreamer import TcpStreamObserver 6 | from antevents.base import make_test_publisher, Scheduler 7 | 8 | 9 | loop = asyncio.get_event_loop() 10 | 11 | s = make_test_publisher(1, stop_after_events=10) 12 | 13 | t = TcpStreamObserver(loop, "localhost", 2991) 14 | 15 | s.subscribe(t) 16 | 17 | scheduler = Scheduler(loop) 18 | scheduler.schedule_periodic(s, 2) # sample once every 2 seconds 19 | 20 | 21 | scheduler.run_forever() 22 | scheduler.stop() 23 | -------------------------------------------------------------------------------- /tests/test_timeout.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Verify that we can set up timeout events 4 | """ 5 | import asyncio 6 | import unittest 7 | from thingflow.base import Scheduler, FunctionFilter 8 | import thingflow.filters.timeout 9 | import thingflow.filters.output 10 | from utils import make_test_output_thing_from_vallist, ValidationInputThing 11 | 12 | def on_next_alternate(self, x): 13 | if self.keep_mode: 14 | print("Sending %s" % x.__repr__()) 15 | self._dispatch_next(x) 16 | else: 17 | print("Dropping %s" % x.__repr__()) 18 | self.countdown -= 1 19 | if self.countdown==0: 20 | self.countdown = self.N 21 | self.keep_mode = not self.keep_mode 22 | 23 | class DropPeriodic(FunctionFilter): 24 | """Allow through N events, drop the next N, and then repeat. 25 | """ 26 | def __init__(self, previous_in_chain, N=1): 27 | self.N = N 28 | self.countdown = N 29 | self.keep_mode = True 30 | super().__init__(previous_in_chain, on_next=on_next_alternate, 31 | name='drop_alternate') 32 | 33 | 34 | class EventWatcher(thingflow.filters.timeout.EventWatcher): 35 | """Repeat the last good event 36 | """ 37 | def __init__(self): 38 | self.last_good_event = None 39 | def on_next(self, x): 40 | self.last_good_event = x 41 | def produce_event_for_timeout(self): 42 | print("producing event for timeout: %s" % 43 | self.last_good_event.__repr__()) 44 | return self.last_good_event 45 | 46 | sensor_values = [ 47 | 1, 48 | 2, 49 | 3, 50 | 4, 51 | 5, 52 | 6 53 | ] 54 | 55 | expected_values = [ 56 | 1, 57 | 1, 58 | 3, 59 | 3, 60 | 5, 61 | 5 62 | ] 63 | 64 | expected_values_multiple_timeouts = [ 65 | 1, 66 | 2, 67 | 2, 68 | 2, 69 | 5, 70 | 6 71 | ] 72 | 73 | class TestTimeouts(unittest.TestCase): 74 | def test_supplying_event_on_timeout(self): 75 | """In this testcase, we drop every other event. 76 | We set the timeout to a bit longer than the event interval of 77 | one second. It then supplies the previous event. The resulting 78 | output stream will show every other value repeated twice. 79 | """ 80 | sensor = make_test_output_thing_from_vallist(1, sensor_values) 81 | drop = DropPeriodic(sensor) 82 | scheduler = Scheduler(asyncio.get_event_loop()) 83 | vo = ValidationInputThing(expected_values, self) 84 | drop.supply_event_when_timeout(EventWatcher(), 85 | scheduler, 1.1).output().connect(vo) 86 | scheduler.schedule_periodic(sensor, 1) 87 | sensor.print_downstream() 88 | scheduler.run_forever() 89 | self.assertTrue(vo.completed, 90 | "Schedule exited before validation observer completed") 91 | 92 | def test_multiple_timeouts(self): 93 | """In this testcase, we pass two events, drop two events, etc. 94 | We set the timeout to a bit longer than the event interval. The last 95 | good value is supplied when the timeout expires. Thus, we should see 96 | two good events, two repeats of the first event, two good events, etc. 97 | """ 98 | sensor = make_test_output_thing_from_vallist(1, sensor_values) 99 | drop = DropPeriodic(sensor, N=2) 100 | scheduler = Scheduler(asyncio.get_event_loop()) 101 | vo = ValidationInputThing(expected_values_multiple_timeouts, self) 102 | drop.supply_event_when_timeout(EventWatcher(), 103 | scheduler, 1.1).output().connect(vo) 104 | scheduler.schedule_periodic(sensor, 1) 105 | sensor.print_downstream() 106 | scheduler.run_forever() 107 | self.assertTrue(vo.completed, 108 | "Schedule exited before validation observer completed") 109 | 110 | 111 | if __name__ == '__main__': 112 | unittest.main() 113 | 114 | -------------------------------------------------------------------------------- /tests/test_tracing.py: -------------------------------------------------------------------------------- 1 | """Test that tracing works properly. 2 | The automated test only verifies that tracing does not cause 3 | a crash. You have to verify the actual messages manually 4 | """ 5 | import unittest 6 | import asyncio 7 | from thingflow.base import * 8 | from utils import ValueListSensor, ValidationInputThing 9 | from thingflow.filters.map import map 10 | from thingflow.filters.output import output 11 | from thingflow.filters.combinators import passthrough 12 | 13 | values = [1,2,3,4,5] 14 | 15 | class TestTracing(unittest.TestCase): 16 | def test_tracing(self): 17 | s = ValueListSensor(1, values) 18 | p = SensorAsOutputThing(s) 19 | v = ValidationInputThing([v+1 for v in values], self, 20 | extract_value_fn=lambda x:x) 21 | p.passthrough(output).map(lambda x : x.val+1).passthrough(output).connect(v) 22 | p.trace_downstream() 23 | scheduler = Scheduler(asyncio.get_event_loop()) 24 | scheduler.schedule_periodic(p, 0.5) # sample twice every second 25 | scheduler.run_forever() 26 | 27 | 28 | if __name__ == '__main__': 29 | unittest.main() 30 | 31 | -------------------------------------------------------------------------------- /tests/test_transducer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | Tests related to the transducers framework and specific transducers that are 5 | defined in thingflow.filters.transducer 6 | """ 7 | 8 | import asyncio 9 | import unittest 10 | from utils import ValueListSensor, ValidationInputThing 11 | from thingflow.base import Scheduler 12 | from thingflow.filters.transducer import SensorSlidingMean, PeriodicMedianTransducer, transduce 13 | from thingflow.filters.combinators import parallel 14 | from thingflow.filters.output import output 15 | 16 | value_stream = [ 17 | 10, 18 | 11, 19 | 9, 20 | 12, 21 | 15, 22 | 6, 23 | 14, 24 | 9 25 | ] 26 | 27 | mean_stream = [ 28 | 10, 29 | 10.5, 30 | 10, 31 | 10.5, 32 | 11.75, 33 | 10.5, 34 | 11.75, 35 | 11.0 36 | ] 37 | 38 | periodic_median_stream = [ 39 | 10, 40 | 12, 41 | 11.5 42 | ] 43 | 44 | 45 | class TestCase(unittest.TestCase): 46 | def setUp(self): 47 | self.scheduler = Scheduler(asyncio.get_event_loop()) 48 | self.sensor = ValueListSensor(1, value_stream) 49 | 50 | def test_sensor_event_sliding_window(self): 51 | vs = ValidationInputThing(mean_stream, self) 52 | self.scheduler.schedule_sensor(self.sensor, 0.1, 53 | transduce(SensorSlidingMean(4)), 54 | parallel(vs, output())) 55 | self.scheduler.run_forever() 56 | self.assertTrue(vs.completed) 57 | 58 | def test_periodic_median_transducer(self): 59 | vs = ValidationInputThing(periodic_median_stream, self) 60 | self.scheduler.schedule_sensor(self.sensor, 0.1, 61 | transduce(PeriodicMedianTransducer(3)), 62 | parallel(vs, output())) 63 | self.scheduler.run_forever() 64 | self.assertTrue(vs.completed) 65 | 66 | if __name__ == '__main__': 67 | unittest.main() 68 | -------------------------------------------------------------------------------- /thingflow/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016, 2017 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | This is the main package for antevents. Directly within this package you fill 5 | find the following module: 6 | 7 | * `base` - the core abstractions and classes of the system. 8 | 9 | The rest of the functionality is in sub-packages: 10 | 11 | * `adapters` - components to read/write events outside the system 12 | * `internal` - some internal definitions 13 | * `filters` - filters that allow linq-style query pipelines over event streams 14 | * `sensors` - interfaces to sensors go here 15 | """ 16 | 17 | __version__ = "2.3.0" 18 | -------------------------------------------------------------------------------- /thingflow/adapters/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | *Adapters* are components that connect ThingFlows to the external 5 | world. *Readers* are event output things which source an event stream 6 | into an ThingFlow process. *Writers* are input things that 7 | translate an event stream to a form used outside of the ThingFlow 8 | process. For example, `CsvReader` is a output thing that reads 9 | events from a CSV-formatted spreadsheet file and `CsvWriter` 10 | is an input thing that writes events to a CSV file. 11 | 12 | Why don't we just call adapters OutputThings and InputThings? We 13 | want to avoid confusion do to the fact that an OutputThing is used to connect 14 | to external inputs while external outputs interface via InputThings. 15 | 16 | """ 17 | -------------------------------------------------------------------------------- /thingflow/adapters/generic.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | Generic reader and writer classes, to be subclassed for specific adapters. 5 | """ 6 | 7 | from thingflow.base import OutputThing, DirectOutputThingMixin, FatalError 8 | 9 | class EventRowMapping: 10 | """Interface that converts between events and "rows" 11 | """ 12 | def event_to_row(self, event): 13 | """Convert an event to the row representation (usually a 14 | list of values). 15 | """ 16 | raise NotImplemented 17 | 18 | def row_to_event(self, row): 19 | """Convert a row to an event. 20 | """ 21 | raise NotImplemented 22 | 23 | 24 | class DirectReader(OutputThing, DirectOutputThingMixin): 25 | """A reader that can be run in the current thread (does not block 26 | indefinitely). Reads rows from the iterable, converts them to events 27 | using the mapping and passes them on. 28 | """ 29 | def __init__(self, iterable, mapper, name=None): 30 | super().__init__() 31 | self.iterable = iterable 32 | self.mapper = mapper 33 | self.name = name 34 | 35 | def _observe(self): 36 | try: 37 | row = self.iterable.__next__() 38 | self._dispatch_next(self.mapper.row_to_event(row)) 39 | except StopIteration: 40 | self._close() 41 | self._dispatch_completed() 42 | except FatalError: 43 | self._close() 44 | raise 45 | except Exception as e: 46 | self._close() 47 | self._dispatch_error(e) 48 | 49 | def _close(self): 50 | """This method is called when we stop the iteration, either due to 51 | reaching the end of the sequence or an error. It can be overridden by 52 | subclasses to clean up any state and release resources (e.g. closing 53 | open files/connections). 54 | """ 55 | pass 56 | 57 | def __str__(self): 58 | if hasattr(self, 'name') and self.name: 59 | return self.name 60 | else: 61 | return super().__str__() 62 | -------------------------------------------------------------------------------- /thingflow/adapters/pandas.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | Pandas (http://pandas.pydata.org) is a data analysis library. 5 | This module contains adapters for converting between thingflow 6 | event streams and Pandas data types. 7 | """ 8 | import datetime 9 | import pandas as pd 10 | 11 | from thingflow.base import InputThing 12 | 13 | class PandasSeriesWriter(InputThing): 14 | """Create a pandas Series object corresponding to the 15 | event stream passed to this subscriber. 16 | """ 17 | def __init__(self, tz=datetime.timezone.utc): 18 | self.data= [] 19 | self.index = [] 20 | self.tz = tz 21 | self.result = None # we will store the series here when done 22 | 23 | def on_next(self, x): 24 | self.data.append(x.val) 25 | self.index.append(datetime.datetime.fromtimestamp(x.ts, tz=self.tz)) 26 | 27 | def on_completed(self): 28 | self.result = pd.Series(self.data, index=self.index) 29 | 30 | 31 | -------------------------------------------------------------------------------- /thingflow/adapters/rpi/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | -------------------------------------------------------------------------------- /thingflow/adapters/rpi/gpio.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | Output on raspberry pi gpio pins 5 | """ 6 | 7 | import RPi.GPIO as gpio 8 | 9 | from thingflow.base import InputThing, SensorEvent 10 | 11 | class GpioPinOut(InputThing): 12 | """Actuator for an output pin on the GPIO bus. 13 | """ 14 | def __init__(self, port=11): 15 | self.port = port 16 | gpio.setmode(gpio.BOARD) 17 | gpio.setup(port, gpio.OUT, initial=gpio.LOW) 18 | self.current_state = False 19 | self.closed = False 20 | 21 | def on_next(self, x): 22 | """If x is a truthy value, we turn the light on 23 | """ 24 | assert not isinstance(x, SensorEvent), "Send a raw value, not a sensor event" 25 | if x and not self.current_state: 26 | gpio.output(self.port, gpio.HIGH) 27 | self.current_state = True 28 | elif (not x) and self.current_state: 29 | gpio.output(self.port, gpio.LOW) 30 | self.current_state = False 31 | 32 | def _cleanup(self): 33 | if not self.closed: 34 | gpio.output(self.port, gpio.LOW) 35 | gpio.cleanup() 36 | self.closed = True 37 | 38 | def on_completed(self): 39 | self._cleanup() 40 | 41 | def on_error(self, e): 42 | self._cleanup() 43 | 44 | def __str__(self): 45 | return "GpioPinOut(port=%s, state=%s)" % \ 46 | (self.port, 'ON' if self.current_state else 'OFF') 47 | 48 | -------------------------------------------------------------------------------- /thingflow/adapters/tcpstreamer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """TODO: This needs to be updated to match the latest base.py! 4 | """ 5 | import asyncio 6 | # log = logging.getLogger(__name__) 7 | # formatter = logging.Formatter("%(asctime)s %(levelname)s " + 8 | # "[%(module)s:%(lineno)d] %(message)s") 9 | # log.setLevel(logging.DEBUG) 10 | 11 | from thingflow.base import InputThing 12 | 13 | clients = {} # task -> (reader, writer) 14 | 15 | 16 | class TcpStreamInputThing(InputThing): 17 | 18 | def __init__(self, loop, host=None, port=2991): 19 | self.server = None 20 | self.task = 0 21 | self.clients = { } 22 | self.host = host 23 | self.port = 2991 24 | 25 | self.loop = loop # TODO REMOVE 26 | # set up tcp stream 27 | self.server = loop.run_until_complete( 28 | asyncio.streams.start_server(self._accept_client, 29 | host, port, loop=loop)) 30 | 31 | def __str__(self): 32 | return "TcpStreamInputThing[{0}, {1}]".format(self.host, self.port) 33 | 34 | def _accept_client(self, client_reader, client_writer): 35 | """ 36 | This method accepts a new client connection and creates a Task 37 | to handle this client. self.clients is updated to keep track 38 | of the new client. 39 | """ 40 | print("Accepting new client") 41 | self.task = self.task + 1 42 | self.clients[self.task] = (client_reader, client_writer) 43 | 44 | def stop(self): 45 | """ 46 | Stops the TCP server, i.e. closes the listening socket(s). 47 | This method runs the loop until the server sockets are closed. 48 | """ 49 | if self.server is not None: 50 | self.server.close() 51 | # TODO: do not have access to loop 52 | self.loop.run_until_complete(self.server.wait_closed()) 53 | self.server = None 54 | 55 | def on_next(self, msg): 56 | # send message on tcp stream 57 | print("tcp: on_next") 58 | for task, (reader, writer) in self.clients.items(): 59 | try: 60 | print("tcp: writing to client") 61 | writer.write(str(msg).encode('utf-8')) 62 | writer.write('\n'.encode('utf-8')) 63 | asyncio.async(writer.drain()) # can this raise exception? 64 | except ConnectionResetError: 65 | print("tcp: client disconnected") 66 | del self.clients[task] 67 | print("tcp: on_next done") 68 | 69 | def on_error(self, e): 70 | # close tcp connection 71 | self.stop() 72 | print(e) 73 | 74 | def on_completed(self): 75 | # close tcp connection 76 | self.stop() 77 | 78 | -------------------------------------------------------------------------------- /thingflow/filters/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016,2017 by MPI-SWS and Data-ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | This sub-module provides a collection of filters for providing linq-style 5 | programming (inspired by RxPy). 6 | 7 | Each function appears as a method on the OutputThing base class, allowing for 8 | easy chaining of calls. For example:: 9 | 10 | sensor.where(lambda x: x > 100).select(lambda x: x*2) 11 | 12 | If the @filtermethod decorator is used, then a standalone function is also 13 | defined that takes all the arguments except the publisher and returns a 14 | function which, when called, takes a publisher and subscribes to the publisher. 15 | We call this returned function a "thunk". Thunks can be used with combinators 16 | (like compose(), parallel(), and passthrough(), all defined in combinators.py) 17 | as well as directly with the scheduler. For example:: 18 | 19 | scheduler.schedule_sensor(sensor, where(lambda x: x> 100), 20 | select(lambda x: x*2)) 21 | 22 | 23 | The implementation code for a linq-style filter typically looks like the 24 | following:: 25 | 26 | @filtermethod(OutputThing) 27 | def example(this, ...): 28 | def _filter(self, x): 29 | .... 30 | return FunctionFilter(this, _filter, name="example") 31 | 32 | Note that, by convention, we use `this` as the first argument of the function, 33 | rather than self. The `this` parameter corresponds to the previous element in 34 | the chain, while the `self` parameter used in the _filter() function represents 35 | the current element in the chain. If you get these mixed up, you can get an 36 | infinite loop! 37 | 38 | In general, a linq-style filter takes the previous OutputThing/filter in a 39 | chain as its first input, parameters to the filter as subsequent inputs, and 40 | returns a OutputThing/filter that should be used as the input to the next step 41 | in the filter chain. 42 | """ 43 | 44 | 45 | 46 | from . import buffer 47 | from . import first 48 | from . import never 49 | from . import output 50 | from . import scan 51 | from . import select 52 | from . import skip 53 | from . import some 54 | from . import take 55 | from . import transducer 56 | from . import timeout 57 | from . import where 58 | from . import combinators 59 | -------------------------------------------------------------------------------- /thingflow/filters/buffer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | import datetime 4 | 5 | from thingflow.base import OutputThing, Filter, FunctionFilter, FatalError, filtermethod 6 | from thingflow.filters.timeout import Timeout, EventWatcher 7 | 8 | @filtermethod(OutputThing) 9 | def buffer_with_count(this, count): 10 | """ 11 | """ 12 | if count < 0: 13 | raise FatalError 14 | 15 | q = [ [] ] 16 | num_seen = [0] 17 | 18 | def on_next(self, x): 19 | num_seen[0] += 1 20 | q[0].append(x) 21 | if num_seen[0] == count: 22 | self._dispatch_next(q[0]) 23 | num_seen[0] = 0 24 | q[0] = [] 25 | 26 | def on_completed(self): 27 | self._dispatch_next(q[0]) 28 | q[0] = [] 29 | self._dispatch_completed() 30 | 31 | def on_error(self, e): 32 | self._dispatch_next(q[0]) 33 | q[0] = [] 34 | self._dispatch_error(e) 35 | 36 | return FunctionFilter(this, on_next, on_error=on_error, on_completed=on_completed, 37 | name="buffer_with_count") 38 | 39 | class BufferEventWatcher(EventWatcher): 40 | def __init__(self): 41 | self.q = [] 42 | def on_next(self, x): 43 | print("on_next called", datetime.datetime.now(), x) 44 | self.q.append(x) 45 | 46 | def produce_event_for_timeout(self): 47 | r = self.q 48 | print("produce_event_for_timeout called", datetime.datetime.now(), r) 49 | self.q = [ ] 50 | return r 51 | 52 | class BufferEventUntilTimeoutOrCount(Filter): 53 | """A class that passes on the events on the default channel to a buffer (maintained 54 | by a BufferEventWatcher). When a timeout fires, the BufferEventWatcher returns 55 | the buffer of all events so far. 56 | """ 57 | def __init__(self, previous_in_chain, event_watcher, scheduler, interval=None, count=None): 58 | self.count = count 59 | self.seen = 0 60 | self.event_watcher = event_watcher 61 | super().__init__(previous_in_chain) 62 | if interval: 63 | self.timeout_thing = \ 64 | Timeout(scheduler, self.event_watcher.produce_event_for_timeout) 65 | self.interval = interval 66 | 67 | if interval: 68 | # pass the timeout_thing's timeout events to my on_timeout_next() 69 | # method 70 | self.timeout_thing.connect(self, 71 | port_mapping=('default','timeout')) 72 | # We start the timeout now. 73 | # This timeout won't start counting down until we start the scheduler. 74 | self.timeout_thing.start(interval) 75 | 76 | def on_next(self, x): 77 | self.seen += 1 78 | self.event_watcher.on_next(x) 79 | if self.count and self.seen == self.count: 80 | e = self.event_watcher.produce_event_for_timeout() 81 | self.seen = 0 82 | self._dispatch_next(e) 83 | if self.interval: 84 | self.timeout_thing.start(self.interval) 85 | 86 | def on_completed(self): 87 | # flush the remaining events from the event buffer 88 | self._dispatch_next(self.event_watcher.produce_event_for_timeout()) 89 | self.event_watcher.close() 90 | if self.interval: 91 | self.timeout_thing.clear() 92 | self._dispatch_completed() 93 | def on_error(self, e): 94 | # flush the remaining events from the event buffer 95 | self._dispatch_next(self.event_watcher.produce_event_for_timeout()) 96 | self.event_watcher.close() 97 | if self.interval: 98 | self.timeout_thing.clear() 99 | self._dispatch_error(e) 100 | 101 | def on_timeout_next(self, x): 102 | """We got the buffered events from the timeout -- send it to the subscribers 103 | and reset the timer 104 | """ 105 | self.timeout_thing.start(self.interval) 106 | self._dispatch_next(x) 107 | 108 | def on_timeout_error(self, e): 109 | raise FatalError("%s.on_timeout_error should not be called" % self) 110 | 111 | def on_timeout_completed(self): 112 | raise FatalError("%s.on_timeout_completed should not be called" % self) 113 | 114 | def __str__(self): 115 | return 'buffer_until_timeout' 116 | 117 | 118 | @filtermethod(OutputThing) 119 | def buffer_with_time(this, interval, scheduler): 120 | if interval < 0: 121 | raise FatalError 122 | e = BufferEventWatcher() 123 | f = BufferEventUntilTimeoutOrCount(this, e, scheduler, interval=interval, count=None) 124 | return f 125 | 126 | 127 | 128 | @filtermethod(OutputThing) 129 | def buffer_with_time_or_count(this, interval, count, scheduler): 130 | if interval <= 0: 131 | raise FatalError 132 | if count <= 0: 133 | raise FatalError 134 | e = BufferEventWatcher() 135 | f = BufferEventUntilTimeoutOrCount(this, e, scheduler, interval=interval, count=count) 136 | return f 137 | 138 | 139 | -------------------------------------------------------------------------------- /thingflow/filters/combinators.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | This module defines combinators for linq-style functions: 5 | compose, parallel, and passthrough. A linq-style function takes the previous 6 | OutputThing/filter in a chain as its first input ("this"), parameters to the 7 | filter as subsequent inputs, and returns a OutputThing/filter that should be 8 | used as the input to the next step in the filter chain. 9 | 10 | We use the term "thunk" for the special case where the linq-style function 11 | takes only a single input - the previous OutputThing/filter in the chain. 12 | The Scheduler.schedule_sensor() method and the functions below can accept 13 | thunks in place filters. If a linq-style filter F was defined using 14 | the @filtermethod decorator, then calling the function directly (not as 15 | a method of a OutputThing) returns a thunk. 16 | """ 17 | 18 | from thingflow.base import OutputThing, filtermethod, _make_thunk, \ 19 | _connect_thunk 20 | 21 | 22 | def compose(*thunks): 23 | """Given a list of thunks and/or filters, compose them 24 | in a sequence and return a thunk. 25 | """ 26 | def apply(this): 27 | p = this 28 | for thunk in thunks: 29 | assert p, \ 30 | "attempted to compose a terminal InputThing/thunk in non-final position" 31 | p = _connect_thunk(p, thunk) 32 | return p 33 | _make_thunk(apply) 34 | return apply 35 | 36 | 37 | def parallel(*connectees): 38 | """Take one or more InputThings/thunks and create a thunk that will 39 | connect all of them to "this" when evaluated. Note that the entire 40 | set of InputThings acts as spurs - the original OutputThing is returned 41 | as the next OutputThing in the chain. 42 | """ 43 | def apply(this): 44 | for c in connectees: 45 | _connect_thunk(this, c) 46 | return this 47 | _make_thunk(apply) 48 | return apply 49 | 50 | 51 | @filtermethod(OutputThing) 52 | def passthrough(this, spur): 53 | """We wish to have a spur off a chain of filters. For example, a writer that 54 | is a InputThing, but does not implement the OutputThing API. Thus, we have no 55 | way to put it in the middle of a chain. Or, we might want to implement a 56 | "fork" in the chain of filters, with two parallel downstream chains. 57 | 58 | passthrough takes "this", the previous OutputThing in the chain, and "spur", 59 | either a InputThing, a thunk (a function that takes the OutputThing as its 60 | single argument), or a plain anonymous function. The spur is connected to 61 | the OutputThing and then the OutputThing is returned to continue the chain. 62 | """ 63 | _connect_thunk(this, spur) 64 | return this 65 | 66 | -------------------------------------------------------------------------------- /thingflow/filters/dispatch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | 4 | from thingflow.base import OutputThing, InputThing, filtermethod 5 | 6 | 7 | class Dispatcher(OutputThing, InputThing): 8 | """Dispatch rules are a list of (predicate, port) pairs. See the 9 | documentation on the dispatch() extension method for details. 10 | """ 11 | def __init__(self, previous_in_chain, dispatch_rules): 12 | ports = [port for (pred, port) in dispatch_rules] + ['default'] 13 | super().__init__(ports=ports) 14 | self.dispatch_rules = dispatch_rules 15 | self.disconnect = previous_in_chain.connect(self) 16 | 17 | def on_next(self, x): 18 | for (pred, port) in self.dispatch_rules: 19 | if pred(x): 20 | self._dispatch_next(x, port=port) 21 | return 22 | self._dispatch_next(x, port='default') # fallthrough case 23 | 24 | def on_completed(self): 25 | for (pred, port) in self.dispatch_rules: 26 | self._dispatch_completed(port=port) 27 | self._dispatch_completed(port='default') 28 | 29 | def on_error(self, e): 30 | for (pred, port) in self.dispatch_rules: 31 | self._dispatch_error(e, port=port) 32 | self._dispatch_error(e, port='default') 33 | 34 | def __str__(self): 35 | return 'dispatch' 36 | 37 | @filtermethod(OutputThing) 38 | def dispatch(this, dispatch_rules): 39 | """Dispatch each incoming event to one output port, according to 40 | the dispatch rules. The rules are a list of (predicate, port) pairs. 41 | The event is dispatched to the first port where the associated predicate 42 | returns True. If no predicates return True, the event is dispatched 43 | to the default port. 44 | """ 45 | return Dispatcher(this, dispatch_rules) 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /thingflow/filters/first.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | 4 | from thingflow.base import OutputThing, filtermethod 5 | import thingflow.filters.take 6 | 7 | @filtermethod(OutputThing) 8 | def first(this): 9 | """Take the first element of the stream. Sends out on_completed after 10 | forwarding the first element. If the stream is empty, we will just 11 | pass on the completed notification we get from the incoming stream. 12 | """ 13 | return this.take(1) 14 | -------------------------------------------------------------------------------- /thingflow/filters/json.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | import json 4 | 5 | from thingflow.base import OutputThing, FunctionFilter, filtermethod 6 | 7 | @filtermethod(OutputThing) 8 | def to_json(this): 9 | """Convert the events in the stream to a json string. 10 | """ 11 | def on_next(self, x): 12 | self._dispatch_next(json.dumps(x)) 13 | 14 | return FunctionFilter(this, on_next=on_next, name='to_json') 15 | 16 | 17 | @filtermethod(OutputThing) 18 | def from_json(this, constructor=None): 19 | """Parse a sequence of json strings. If constructor is specified, the 20 | parsed value is passed as *args to the constructor to return the actual 21 | object. 22 | """ 23 | def on_next(self, x): 24 | obj = json.loads(x) 25 | if constructor: 26 | obj = constructor(*obj) 27 | self._dispatch_next(obj) 28 | 29 | return FunctionFilter(this, on_next=on_next, name='from_json') 30 | 31 | -------------------------------------------------------------------------------- /thingflow/filters/map.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | Transform each event in the stream. thingflow.filters.select and 5 | thingflow.filters.map have the same functionality. Just import one - 6 | the @filtermethod decorator will create the other as an alias. 7 | """ 8 | from thingflow.base import OutputThing, FunctionFilter, filtermethod 9 | 10 | @filtermethod(OutputThing, alias="select") 11 | def map(this, mapfun): 12 | """Returns a stream whose elements are the result of 13 | invoking the transform function on each element of source. 14 | If the function returns None, no event is passed downstream. 15 | """ 16 | def on_next(self, x): 17 | y = mapfun(x) 18 | if y is not None: 19 | self._dispatch_next(y) 20 | return FunctionFilter(this, on_next, name='map') 21 | -------------------------------------------------------------------------------- /thingflow/filters/never.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | from thingflow.base import OutputThing, DirectOutputThingMixin 4 | 5 | class Never(OutputThing, DirectOutputThingMixin): 6 | """An OutputThing that never calls its connections: creates an empty stream that never goes away 7 | """ 8 | def __init__(self): 9 | super().__init__() 10 | 11 | def _observe(self): 12 | """Do nothing 13 | """ 14 | pass 15 | -------------------------------------------------------------------------------- /thingflow/filters/output.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | from sys import stdout 4 | import traceback as tb 5 | 6 | from thingflow.base import OutputThing, XformOrDropFilter, filtermethod 7 | 8 | class Output(XformOrDropFilter): 9 | def __init__(self, previous_in_chain, file=stdout): 10 | super().__init__(previous_in_chain) 11 | self.file = file 12 | 13 | def _filter(self, x): 14 | print(x, file=self.file) 15 | return x 16 | 17 | def on_error(self, e): 18 | if hasattr(e, '__traceback__'): 19 | tb.print_exception(type(e), e, e.__traceback__, file=self.file) 20 | else: 21 | print(e, file=self.file) 22 | self._dispatch_error(e) 23 | 24 | def __str__(self): 25 | if self.file==stdout: 26 | return 'output()' 27 | else: 28 | return 'output(%s)' % str(self.file) 29 | 30 | 31 | @filtermethod(OutputThing) 32 | def output(this, file=stdout): 33 | """Print each element of the sequence. Exceptions are printed 34 | as well. We don't call it print, because that will override the 35 | built-in print function. 36 | """ 37 | return Output(this, file=file) 38 | 39 | class OutputCount(XformOrDropFilter): 40 | def __init__(self, previous_in_chain, file=stdout): 41 | super().__init__(previous_in_chain) 42 | self.file = file 43 | self.count = 0 44 | 45 | def _filter(self, x): 46 | self.count += 1 47 | return x 48 | 49 | def on_completed(self): 50 | msg = "* %d events processed *" % self.count 51 | print('*'*len(msg), file=self.file) 52 | print(msg, file=self.file) 53 | print('*'*len(msg), file=self.file) 54 | self._dispatch_completed() 55 | 56 | @filtermethod(OutputThing) 57 | def output_count(this, file=stdout): 58 | """Just count the number of events and print out a banner with the 59 | total at the end. 60 | """ 61 | return OutputCount(this, file=file) 62 | -------------------------------------------------------------------------------- /thingflow/filters/scan.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | from thingflow.base import OutputThing, filtermethod 4 | 5 | @filtermethod(OutputThing, alias="aggregate") 6 | def reduce(self, accumulator, seed=None): 7 | """Applies an accumulator function over a sequence, 8 | returning the result of the aggregation as a single element in the 9 | result sequence. The specified seed value is used as the initial 10 | accumulator value. 11 | Example: 12 | 1 - res = source.reduce(lambda acc, x: acc + x) 13 | 2 - res = source.reduce(lambda acc, x: acc + x, 0) 14 | Keyword arguments: 15 | :param accumulator: An accumulator function to be 16 | invoked on each element. 17 | :param seed: Optional initial accumulator value. 18 | :returns: An observable sequence containing a single element with the 19 | final accumulator value. 20 | """ 21 | 22 | return self.scan(accumulator, seed=seed).last() 23 | 24 | @filtermethod(OutputThing) 25 | def scan(this, accumulator, seed=None): 26 | """Applies an accumulator function over an observable sequence and 27 | returns each intermediate result. The optional seed value is used as 28 | the initial accumulator value. 29 | For aggregation behavior with no intermediate results, see OutputThing.aggregate. 30 | 1 - scanned = source.scan(lambda acc, x: acc + x) 31 | 2 - scanned = source.scan(lambda acc, x: acc + x, 0) 32 | Keyword arguments: 33 | accumulator -- An accumulator function to be invoked on each element. 34 | seed -- [Optional] The initial accumulator value. 35 | Returns an observable sequence containing the accumulated values. 36 | """ 37 | 38 | has_seed = False 39 | if seed is not None: 40 | has_seed = True 41 | has_accumulation = [False] 42 | accumulation = [None] 43 | 44 | def calculate(x): 45 | if has_accumulation[0]: 46 | accumulation[0] = accumulator(accumulation[0], x) 47 | else: 48 | accumulation[0] = accumulator(seed, x) if has_seed else x 49 | has_accumulation[0] = True 50 | return accumulation[0] 51 | 52 | return this.map(calculate) 53 | -------------------------------------------------------------------------------- /thingflow/filters/select.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | Transform each event in the stream. thingflow.filters.select and 5 | thingflow.filters.map have the same functionality. Just import one - 6 | the @filtermethod decorator will create the other as an alias. 7 | """ 8 | from thingflow.base import OutputThing, FunctionFilter, filtermethod 9 | 10 | 11 | @filtermethod(OutputThing, alias="map") 12 | def select(this, mapfun): 13 | """Returns a stream whose elements are the result of 14 | invoking the transform function on each element of source. 15 | If the function returns None, no event is passed downstream. 16 | """ 17 | def on_next(self, x): 18 | y = mapfun(x) 19 | if y is not None: 20 | self._dispatch_next(y) 21 | return FunctionFilter(this, on_next, name='select') 22 | -------------------------------------------------------------------------------- /thingflow/filters/skip.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | from thingflow.base import OutputThing, FunctionFilter, filtermethod 4 | 5 | @filtermethod(OutputThing, alias="drop") 6 | def skip(this, count): 7 | """Bypasses a specified number of elements in an event sequence 8 | and then returns the remaining elements. 9 | Keyword arguments: 10 | count: The number of elements to skip before returning the remaining 11 | elements. 12 | Returns an event sequence that contains the elements that occur 13 | after the specified index in the input sequence. 14 | """ 15 | 16 | if count < 0: 17 | raise ArgumentOutOfRangeException() 18 | 19 | remaining = [count] 20 | def on_next(self, value): 21 | if remaining[0] <= 0: 22 | self._dispatch_next(value) 23 | else: 24 | remaining[0] -= 1 25 | 26 | return FunctionFilter(this, on_next=on_next, name="skip") 27 | 28 | -------------------------------------------------------------------------------- /thingflow/filters/some.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | from thingflow.base import OutputThing, FunctionFilter, filtermethod 4 | 5 | 6 | @filtermethod(OutputThing) 7 | def some(this, predicate=None): 8 | """Determines whether some element of an observable sequence satisfies a 9 | condition if present, else if some items are in the sequence. 10 | Example: 11 | result = source.some() 12 | result = source.some(lambda x: x > 3) 13 | Keyword arguments: 14 | predicate -- A function to test each element for a condition. 15 | Returns {OutputThing} an observable sequence containing a single element 16 | determining whether some elements in the source sequence pass the test 17 | in the specified predicate if given, else if some items are in the 18 | sequence. 19 | """ 20 | 21 | def on_next(self, x): 22 | self._dispatch_next(True) 23 | self._dispatch_completed() 24 | self.disconnect_from_upstream() 25 | 26 | def on_error(self, e): 27 | self._dispatch_next(False) 28 | self._dispatch_completed() 29 | self.disconnect_from_upstream() 30 | 31 | def on_completed(self): 32 | self._dispatch_next(False) 33 | self._dispatch_completed() 34 | self.disconnect_from_upstream() 35 | 36 | if predicate: 37 | return this.filter(predicate).some() 38 | else: 39 | return FunctionFilter(this, 40 | on_next=on_next, 41 | on_error=on_error, 42 | on_completed=on_completed, name="some") 43 | -------------------------------------------------------------------------------- /thingflow/filters/take.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | from thingflow.base import OutputThing, FunctionFilter, FatalError, filtermethod 4 | 5 | class ArgumentOutOfRangeException(FatalError): 6 | pass 7 | 8 | class SequenceContainsNoElementsError(FatalError): 9 | pass 10 | 11 | @filtermethod(OutputThing) 12 | def take_last(this, count): 13 | """Takes a specified number of contiguous elements from the end of an observable sequence. 14 | This operator accumulates a buffer with a length enough to store 15 | elements count elements. Upon completion of the source sequence, this 16 | buffer is drained on the result sequence. This causes the elements to be 17 | delayed. 18 | Keyword arguments: 19 | count: The number of elements to take from the end of the sequence 20 | """ 21 | q = [] 22 | def on_next(self, x): 23 | q.append(x) 24 | if len(q) > count: 25 | q.pop(0) 26 | 27 | def on_completed(self): 28 | while len(q): 29 | v = q.pop(0) 30 | self._dispatch_next(v) 31 | self._dispatch_completed() 32 | 33 | return FunctionFilter(this, on_next=on_next, on_completed=on_completed) 34 | 35 | 36 | @filtermethod(OutputThing) 37 | def last(this, default=None): 38 | value = [default] 39 | seen_value = [False] 40 | 41 | def on_next(self, x): 42 | value[0] = x 43 | seen_value[0] = True 44 | 45 | def on_completed(self): 46 | if not seen_value[0] and default is None: 47 | self._dispatch_error(SequenceContainsNoElementsError()) 48 | else: 49 | self._dispatch_next(value[0]) 50 | self._dispatch_completed() 51 | return FunctionFilter(this, on_next=on_next, on_completed=on_completed, 52 | name='last') 53 | 54 | @filtermethod(OutputThing) 55 | def take(this, count): 56 | """Takes a specified number of contiguous elements in an event sequence. 57 | Keyword arguments: 58 | count: The number of elements to send forward before skipping the remaining 59 | elements. 60 | """ 61 | 62 | if count < 0: 63 | raise ArgumentOutOfRangeException() 64 | 65 | remaining = [count] 66 | completed = [False] 67 | 68 | def on_next(self, value): 69 | if remaining[0] > 0: 70 | remaining[0] -= 1 71 | self._dispatch_next(value) 72 | if remaining[0]==0 and completed[0]==False: 73 | completed[0] = True 74 | self.disconnect_from_upstream() 75 | self._dispatch_completed() 76 | 77 | def on_completed(self): 78 | # We may have already given a completed notification if we hit count 79 | # elements. On the other hand, we might still need to provide a notification 80 | # if the actual sequence length is less than count. 81 | if completed[0]==False: 82 | completed[0] = True 83 | self._dispatch_completed() 84 | 85 | return FunctionFilter(this, on_next=on_next, on_completed=on_completed, 86 | name="take(%s)" % count) 87 | 88 | -------------------------------------------------------------------------------- /thingflow/filters/timeout.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Timeout-related output things and filters. 4 | """ 5 | from thingflow.base import OutputThing, DirectOutputThingMixin, FunctionFilter,\ 6 | FatalError, filtermethod 7 | 8 | class Timeout(OutputThing, DirectOutputThingMixin): 9 | """An output thing that can shedule timeouts for itself. When a 10 | timeout occurs, an event is sent on the default port. 11 | The timeout_thunk is called to get the actual event. 12 | """ 13 | def __init__(self, scheduler, timeout_thunk): 14 | super().__init__() 15 | self.scheduler = scheduler 16 | self.timeout_thunk = timeout_thunk 17 | self.cancel = None 18 | 19 | def start(self, interval): 20 | if self.cancel: 21 | self.cancel() 22 | self.cancel = self.scheduler.schedule_later_one_time(self, interval) 23 | 24 | def clear(self): 25 | if self.cancel: 26 | self.cancel() 27 | self.cancel = None 28 | 29 | def _observe(self): 30 | """If this gets called, we hit the timeout 31 | """ 32 | self.cancel = None 33 | self._dispatch_next(self.timeout_thunk()) 34 | 35 | class EventWatcher: 36 | """Watch the event stream and then produce an event for a timeout 37 | when asked. This can be subclassed to implement different 38 | policies. 39 | """ 40 | def on_next(self, x): 41 | pass # we get a regular event 42 | def produce_event_for_timeout(self): 43 | return None # return the timeout event 44 | def close(self): # called for on_completed or on_error 45 | pass 46 | 47 | 48 | class SupplyEventWhenTimeout(FunctionFilter): 49 | """This filter sits in a chain and passes incoming events through to 50 | its output. It also passes all events to the on_next() method of the 51 | event watcher. If no event arrives on the input after the interval has 52 | passed since the last event, event_watcher.produce_event_for_timeout() 53 | is called to get a dummy event, which is passed upstream. 54 | """ 55 | def __init__(self, previous_in_chain, event_watcher, scheduler, interval): 56 | self.event_watcher = event_watcher 57 | self.timeout_thing = \ 58 | Timeout(scheduler, self.event_watcher.produce_event_for_timeout) 59 | self.interval = interval 60 | def on_next(self, x): 61 | self.event_watcher.on_next(x) 62 | # reset the timer 63 | self.timeout_thing.start(self.interval) 64 | self._dispatch_next(x) 65 | def on_completed(self): 66 | self.event_watcher.close() 67 | self.timeout_thing.clear() 68 | self._dispatch_completed() 69 | def on_error(self, e): 70 | self.event_watcher.close() 71 | self.timeout_thing.clear() 72 | self._dispatch_error(e) 73 | super().__init__(previous_in_chain, on_next=on_next, 74 | on_completed=on_completed, on_error=on_error, 75 | name='supply_event_when_timeout') 76 | # pass the timeout_thing's timeout events to my on_timeout_next() 77 | # method< 78 | self.timeout_thing.connect(self, 79 | port_mapping=('default','timeout')) 80 | # We start the timeout now - if we don't get a first event from the 81 | # input within the timeout, we should supply a timeout event. This 82 | # timeout won't start counting down until we start the scheduler. 83 | self.timeout_thing.start(interval) 84 | 85 | def on_timeout_next(self, x): 86 | """This method is connected to the Timeout thing's output. If it 87 | gets called, the timeout has fired. We need to reschedule the timeout 88 | as well, so that we continue to produce events in the case of multiple 89 | consecutive timeouts. 90 | """ 91 | self.timeout_thing.start(self.interval) 92 | self._dispatch_next(x) 93 | 94 | def on_timeout_error(self, e): 95 | """This won't get called, as the Timeout thing does not republish any 96 | errors it receives. 97 | """ 98 | raise FatalError("%s.on_timeout_error should not be called" % self) 99 | 100 | def on_timeout_completed(self): 101 | """This won't get called, as the timeout thing does not propate 102 | any completions. We just use the primary event stream to figure out when 103 | things are done and clear any pending timeouts at that time. 104 | """ 105 | raise FatalError("%s.on_timeout_completed should not be called" % self) 106 | 107 | @filtermethod(OutputThing) 108 | def supply_event_when_timeout(this, event_watcher, scheduler, interval): 109 | return SupplyEventWhenTimeout(this, event_watcher, scheduler, interval) 110 | -------------------------------------------------------------------------------- /thingflow/filters/where.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | from thingflow.base import OutputThing, FunctionFilter, filtermethod 4 | 5 | @filtermethod(OutputThing, alias="filter") 6 | def where(this, predicate): 7 | """Filter a stream based on the specified predicate function. 8 | """ 9 | def on_next(self, x): 10 | if predicate(x): 11 | self._dispatch_next(x) 12 | return FunctionFilter(this, on_next, name="where") 13 | 14 | -------------------------------------------------------------------------------- /thingflow/internal/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Microsoft Open Technologies, Inc. This file was adapted from RxPY 2 | # (https://github.com/ReactiveX/RxPY). Licensed under the Apache 2.0 License. 3 | from .basic import noop, default_error, default_comparer 4 | 5 | -------------------------------------------------------------------------------- /thingflow/internal/basic.py: -------------------------------------------------------------------------------- 1 | # Copyright Microsoft Open Technologies, Inc. This file was adapted from RxPY 2 | # (https://github.com/ReactiveX/RxPY). Licensed under the Apache 2.0 License. 3 | from datetime import datetime 4 | 5 | 6 | # Defaults 7 | def noop(*args, **kw): 8 | """No operation. Returns nothing""" 9 | pass 10 | 11 | 12 | def identity(x): 13 | """Returns argument x""" 14 | return x 15 | 16 | 17 | def default_now(): 18 | return datetime.utcnow() 19 | 20 | 21 | def default_comparer(x, y): 22 | return x == y 23 | 24 | 25 | def default_sub_comparer(x, y): 26 | return x - y 27 | 28 | 29 | def default_key_serializer(x): 30 | return str(x) 31 | 32 | 33 | def default_error(err): 34 | if isinstance(err, BaseException): 35 | raise err 36 | else: 37 | raise Exception(err) 38 | -------------------------------------------------------------------------------- /thingflow/sensors/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | -------------------------------------------------------------------------------- /thingflow/sensors/rpi/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | -------------------------------------------------------------------------------- /thingflow/sensors/rpi/adxl345_py3.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Sensor for the ADXL345 accelerometer. 4 | Code originally from https://github.com/pimoroni/adxl345-python, 5 | with edits for ThingFlow""" 6 | 7 | # 8 | # ADXL345 Python library for Raspberry Pi 9 | # 10 | # author: Jonathan Williamson 11 | # license: BSD, see LICENSE.txt included in the original version. 12 | # 13 | # This is a Raspberry Pi Python implementation to help you get started with 14 | # the Adafruit Triple Axis ADXL345 breakout board: 15 | # http://shop.pimoroni.com/products/adafruit-triple-axis-accelerometer 16 | # 17 | # Changes for ThingFlow: 18 | # Minor edit to print statement for Python 3 and ThingFlow API changes (need sensor_id) 19 | 20 | import smbus 21 | 22 | # select the correct i2c bus for this revision of Raspberry Pi 23 | revision = ([l[12:-1] for l in open('/proc/cpuinfo','r').readlines() if l[:8]=="Revision"]+['0000'])[0] 24 | bus = smbus.SMBus(1 if int(revision, 16) >= 4 else 0) 25 | 26 | # ADXL345 constants 27 | EARTH_GRAVITY_MS2 = 9.80665 28 | SCALE_MULTIPLIER = 0.004 29 | 30 | DATA_FORMAT = 0x31 31 | BW_RATE = 0x2C 32 | POWER_CTL = 0x2D 33 | 34 | BW_RATE_1600HZ = 0x0F 35 | BW_RATE_800HZ = 0x0E 36 | BW_RATE_400HZ = 0x0D 37 | BW_RATE_200HZ = 0x0C 38 | BW_RATE_100HZ = 0x0B 39 | BW_RATE_50HZ = 0x0A 40 | BW_RATE_25HZ = 0x09 41 | 42 | RANGE_2G = 0x00 43 | RANGE_4G = 0x01 44 | RANGE_8G = 0x02 45 | RANGE_16G = 0x03 46 | 47 | MEASURE = 0x08 48 | AXES_DATA = 0x32 49 | 50 | class ADXL345: 51 | 52 | address = None 53 | 54 | def __init__(self, sensor_id, address = 0x53): 55 | self.sensor_id = sensor_id 56 | self.address = address 57 | self.setBandwidthRate(BW_RATE_100HZ) 58 | self.setRange(RANGE_2G) 59 | self.enableMeasurement() 60 | 61 | def enableMeasurement(self): 62 | bus.write_byte_data(self.address, POWER_CTL, MEASURE) 63 | 64 | def setBandwidthRate(self, rate_flag): 65 | bus.write_byte_data(self.address, BW_RATE, rate_flag) 66 | 67 | # set the measurement range for 10-bit readings 68 | def setRange(self, range_flag): 69 | value = bus.read_byte_data(self.address, DATA_FORMAT) 70 | 71 | value &= ~0x0F; 72 | value |= range_flag; 73 | value |= 0x08; 74 | 75 | bus.write_byte_data(self.address, DATA_FORMAT, value) 76 | 77 | # returns the current reading from the sensor for each axis 78 | # 79 | # parameter gforce: 80 | # False (default): result is returned in m/s^2 81 | # True : result is returned in gs 82 | def sample(self, gforce = False): 83 | bytes = bus.read_i2c_block_data(self.address, AXES_DATA, 6) 84 | 85 | x = bytes[0] | (bytes[1] << 8) 86 | if(x & (1 << 16 - 1)): 87 | x = x - (1<<16) 88 | 89 | y = bytes[2] | (bytes[3] << 8) 90 | if(y & (1 << 16 - 1)): 91 | y = y - (1<<16) 92 | 93 | z = bytes[4] | (bytes[5] << 8) 94 | if(z & (1 << 16 - 1)): 95 | z = z - (1<<16) 96 | 97 | x = x * SCALE_MULTIPLIER 98 | y = y * SCALE_MULTIPLIER 99 | z = z * SCALE_MULTIPLIER 100 | 101 | if gforce == False: 102 | x = x * EARTH_GRAVITY_MS2 103 | y = y * EARTH_GRAVITY_MS2 104 | z = z * EARTH_GRAVITY_MS2 105 | 106 | x = round(x, 4) 107 | y = round(y, 4) 108 | z = round(z, 4) 109 | 110 | return {"x": x, "y": y, "z": z} 111 | 112 | if __name__ == "__main__": 113 | # if run directly we'll just create an instance of the class and output 114 | # the current readings 115 | adxl345 = ADXL345() 116 | 117 | axes = adxl345.getAxes(True) 118 | print("ADXL345 on address 0x%x:" % (adxl345.address)) 119 | print(" x = %.3fG" % ( axes['x'] )) 120 | print(" y = %.3fG" % ( axes['y'] )) 121 | print(" z = %.3fG" % ( axes['z'] )) 122 | -------------------------------------------------------------------------------- /thingflow/sensors/rpi/arduino.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Sensors for AntEvents 4 | Uses the nanpy library (https://github.com/nanpy/nanpy), which controls 5 | a slave Arduino processor. The sensors are connected to the Arduino. 6 | 7 | Both digital (1/0 output) and analogue (0-1023 ouput) sensors may be 8 | be connected to the Arduino. To use this, Nanpy firmware needs to be 9 | flashed onto the Arduino to allow Python to be used. 10 | 11 | Note -This sensor class can only be used with sensors which send their output 12 | straight to the Arduino pins. For sensors which use I2C or SPI, with their 13 | own registers, a library to use them has to be written separately. 14 | """ 15 | 16 | from thingflow.base import OutputThing, IndirectOutputThingMixin 17 | 18 | 19 | from nanpy import ArduinoApi,SerialManager 20 | ardApi = ArduinoApi(connection=SerialManager(device = '/dev/ttyACM0')) 21 | 22 | class ArduinoSensor(OutputThing, IndirectOutputThingMixin): 23 | """Sensor connected to Arduino. Output is analogue(1/0) or digital output(0 - 1023). Nanpy firmware needs to be flashed onto Arduino. 24 | """ 25 | def __init__(self,sensor_id,AD): 26 | """sensor_id is port number, AD is True/False for Analogue/Digital 27 | """ 28 | super().__init__() 29 | self.sensor_id = sensor_id 30 | self.AD = AD 31 | ardApi.pinMode(sensor_id,ardApi.INPUT) 32 | 33 | def sample(self): 34 | if self.AD: 35 | val = ardApi.digitalRead(self.sensor_id) 36 | else: 37 | val = ardApi.analogRead(self.sensor_id) 38 | return val 39 | 40 | def __str__(self): 41 | return 'Arduino Sensor (port=%s, AD=%s)'% (self.sensor_id, self.AD) 42 | -------------------------------------------------------------------------------- /thingflow/sensors/rpi/gpio.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Raspberry Pi GPIO Sensor for AntEvents. 4 | Allows digital (1/0 output) sensors to be connected straight to the 5 | Raspberry Pi (ADC needed for the Pi to take analogue output). 6 | 7 | This sensor class can only be used with sensors which send their output straight 8 | to the Raspberry Pi GPIO pins. For sensors which use I2C or SPI, with their 9 | own registers, a library to use them has to be written separately. 10 | """ 11 | 12 | from thingflow.base import OutputThing, IndirectOutputThingMixin 13 | 14 | import RPi.GPIO as GPIO 15 | class RPISensor(OutputThing, IndirectOutputThingMixin): 16 | """Sensor connected to Raspberry Pi. Output of sensor is digital 17 | (RPi does not come with an ADC unlike the Arduino) 18 | """ 19 | def __init__(self,sensor_id): 20 | """sensor_id is port number 21 | """ 22 | super().__init__() 23 | self.sensor_id = sensor_id 24 | GPIO.setmode(GPIO.BOARD) 25 | GPIO.setup(sensor_id,GPIO.IN) 26 | 27 | def sample(self): 28 | val = GPIO.input(self.sensor_id) 29 | return val 30 | 31 | def __str__(self): 32 | return 'Raspberry Pi Sensor (port=%s)'% self.sensor_id 33 | 34 | -------------------------------------------------------------------------------- /thingflow/sensors/rpi/lux_sensor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """ 4 | This is a sensor for the tsl2591 lux (light level) sensor breakout board 5 | from Adafruit. It is a thin layer on top of python-tsl2591. 6 | To install the tsl2591 library: 7 | sudo apt-get install build-essential libi2c-dev i2c-tools python-dev libffi-dev 8 | sudo /usr/bin/pip install cffi 9 | git clone https://github.com/maxlklaxl/python-tsl2591.git 10 | """ 11 | 12 | import tsl2591 13 | 14 | 15 | class LuxSensor: 16 | tsl = None 17 | def __init__(self, sensor_id=1): 18 | self.sensor_id = sensor_id 19 | if LuxSensor.tsl is None: 20 | LuxSensor.tsl = tsl2591.Tsl2591() # initialize the sensor context 21 | 22 | def sample(self): 23 | """Read the tsl2591 lux sensor and dispatch the luminosity value. 24 | """ 25 | full, ir = LuxSensor.tsl.get_full_luminosity() 26 | lux = LuxSensor.tsl.calculate_lux(full, ir) 27 | # we convert the lux value to an integer - fractions are not 28 | # significant. 29 | return int(round(lux, 0)) 30 | 31 | def __str__(self): 32 | return 'LuxSensor(%s)' % self.sensor_id 33 | -------------------------------------------------------------------------------- /thingflow/sensors/rpi/sensor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 by MPI-SWS and Data-Ken Research. 2 | # Licensed under the Apache 2.0 License. 3 | """Sensors for ThingFlow 4 | Updated to suit the API changes Jeff mentioned, so that the following can be used as follows: 5 | sensor = SensorAsOutputThing(RPISensor()) 6 | The following classes allow digital/analogue sensors (which are not connected using I2C) to be connected to a Raspberry Pi/Arduino and used with ThingFlow 7 | """ 8 | 9 | from thingflow.base import OutputThing, IndirectOutputThingMixin 10 | 11 | import RPi.GPIO as GPIO 12 | class RPISensor(OutputThing, IndirectOutputThingMixin): 13 | """Sensor connected to Raspberry Pi. Output of sensor is digital (RPi does not come with an ADC unlike the Arduino) 14 | """ 15 | def __init__(self,sensor_id): 16 | """sensor_id is port number 17 | """ 18 | super().__init__() 19 | self.sensor_id = sensor_id 20 | GPIO.setmode(GPIO.BOARD) 21 | GPIO.setup(sensor_id,GPIO.IN) 22 | 23 | def sample(self): 24 | val = GPIO.input(self.sensor_id) 25 | return val 26 | 27 | def __str__(self): 28 | return 'Raspberry Pi Sensor (port=%s)'% self.sensor_id 29 | 30 | 31 | from nanpy import ArduinoApi,SerialManager 32 | ardApi = ArduinoApi(connection=SerialManager(device = '/dev/ttyACM0')) 33 | 34 | class ArduinoSensor(OutputThing, IndirectOutputThingMixin): 35 | """Sensor connected to Arduino. Output is analogue(1/0) or digital output(0 - 1023). Nanpy firmware needs to be flashed onto Arduino. 36 | """ 37 | def __init__(self,sensor_id,AD): 38 | """sensor_id is port number, AD is True/False for Analogue/Digital 39 | """ 40 | super().__init__() 41 | self.sensor_id = sensor_id 42 | self.AD = AD 43 | ardApi.pinMode(sensor_id,ardApi.INPUT) 44 | 45 | def sample(self): 46 | if self.AD: 47 | val = ardApi.digitalRead(self.sensor_id) 48 | else: 49 | val = ardApi.analogRead(self.sensor_id) 50 | return val 51 | 52 | def __str__(self): 53 | return 'Arduino Sensor (port=%s, AD=%s)'% (self.sensor_id, self.AD) 54 | --------------------------------------------------------------------------------