├── .dockerignore ├── .flake8 ├── .gitignore ├── .style.yapf ├── .travis.yml ├── AUTHORS ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile.release ├── LICENSE.txt ├── README.md ├── docs ├── connectors.md └── setup.md ├── examples ├── README.md ├── aapl.py ├── actor.py ├── batching │ ├── batching.py │ ├── batching_source.py │ └── batching_with_multitask.py ├── binance │ └── binance_source.py ├── blogs │ └── binance_to_slack.py ├── cloud_object_storage │ ├── cos_sink.py │ ├── cos_sink_from_directory.py │ ├── cos_sink_from_file.py │ ├── cos_sink_multi_part.py │ ├── cos_sink_multitask.py │ ├── cos_source.py │ ├── cos_source_meta.py │ ├── cos_source_move_after_read.py │ └── test_files │ │ └── test.txt ├── datasets │ ├── file_source_to_dataset.py │ ├── file_source_to_dataset_actor.py │ └── test_files │ │ └── test.txt ├── file │ ├── directory_source_and_sink.ipynb │ ├── file_source.py │ ├── file_watch_source.py │ └── test_files │ │ └── test.txt ├── function.py ├── generic │ ├── generic_sink.py │ ├── generic_source.py │ └── periodic_generic_source.py ├── http │ └── http_source.py ├── kafka │ ├── kafka_event_streaming_sink.py │ ├── kafka_event_streaming_source.py │ ├── kafka_partitioned_source_sink.py │ └── kafka_source_sink.py ├── multitask │ └── multitask.py ├── slack │ ├── slack.py │ └── slack_operator_mode.py ├── source.py ├── stream.py ├── task.py └── telegram │ └── telegram_source_sink.py ├── rayvens ├── __init__.py ├── api.py ├── cli │ ├── README.md │ ├── __init__.py │ ├── build.py │ ├── delete.py │ ├── docker.py │ ├── examples │ │ ├── build-base-image.sh │ │ ├── build-slack-sink-image.sh │ │ ├── delete-slack-sink-local.sh │ │ └── run-slack-sink-local.sh │ ├── file.py │ ├── java.py │ ├── kamel.py │ ├── kubernetes.py │ ├── rayvens_print.py │ ├── rayvens_setup.py │ ├── run.py │ ├── utils.py │ └── version.py ├── core │ ├── FileQueue.java │ ├── FileQueueJson.java │ ├── FileQueueName.java │ ├── FileWatchQueue.java │ ├── MetaEventQueue.java │ ├── ProcessFile.java │ ├── ProcessPath.java │ ├── Queue.java │ ├── __init__.py │ ├── catalog.py │ ├── catalog_utils.py │ ├── common.py │ ├── harness.py │ ├── integration.py │ ├── invocation.py │ ├── kafka.py │ ├── kamel.py │ ├── kamel_backend.py │ ├── kamel_utils.py │ ├── kubernetes.py │ ├── kubernetes_utils.py │ ├── local.py │ ├── mode.py │ ├── name.py │ ├── operator.py │ ├── operator_kafka.py │ ├── ray_serve.py │ ├── utils.py │ └── verify.py ├── rayvens └── rayvens_cli.py ├── resources ├── logo.png └── logo.pptx ├── scripts ├── Preloader.java ├── camel-test-source.yaml ├── cluster.yaml ├── docker-compose.yaml ├── kafka.yaml ├── kind.yaml ├── rayvens-setup.sh ├── start-kind.sh ├── travis.sh └── update-rayvens.sh ├── setup.py └── tests ├── generic_sink.py ├── generic_source.py ├── kafka.py ├── kafka_scaling_transport.py ├── kafka_transport.py ├── sink.py ├── source.py ├── source_operator.py └── stream.py /.dockerignore: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | **/__pycache__ 18 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | [flake8] 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | __pycache__/ 18 | *.egg-info/ 19 | /build/ 20 | /dist/ 21 | .python-version 22 | *.ipynb_checkpoints* 23 | -------------------------------------------------------------------------------- /.style.yapf: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | [style] 18 | based_on_style=pep8 19 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | dist: bionic 18 | language: java 19 | services: 20 | - docker 21 | 22 | before_install: 23 | - ./scripts/travis.sh 24 | 25 | install: 26 | - pyenv global 3.8.1 27 | - pip install --upgrade pip 28 | - pip install . 29 | 30 | script: 31 | - ./scripts/start-kind.sh 32 | 33 | # Test local container run mode 34 | # - RAYVENS_TEST_MODE=local python ./tests/stream.py 35 | # - RAYVENS_TEST_MODE=local python ./tests/source.py 36 | - RAYVENS_TEST_MODE=local RAYVENS_TRANSPORT=kafka python ./tests/source.py 37 | # - ray submit ./scripts/cluster.yaml ./tests/stream.py 38 | - ray submit ./scripts/cluster.yaml ./tests/source.py 39 | - RAYVENS_TEST_MODE=local python ./tests/kafka_transport.py 40 | - ray submit ./scripts/cluster.yaml ./tests/kafka_transport.py local 41 | - RAYVENS_TEST_MODE=local python ./tests/kafka_scaling_transport.py 42 | - ray submit ./scripts/cluster.yaml ./tests/kafka_scaling_transport.py local 43 | 44 | # Test operator mode 45 | # - ray submit ./scripts/cluster.yaml ./tests/source_operator.py 46 | - RAYVENS_TEST_MODE=local python ./tests/sink.py 47 | - ray submit ./scripts/cluster.yaml ./tests/sink.py 48 | # - RAYVENS_TEST_MODE=local python ./tests/kafka.py 49 | # - ray submit ./scripts/cluster.yaml ./tests/kafka.py 50 | # - RAYVENS_TEST_MODE=local python ./tests/generic_sink.py 51 | # - ray submit ./scripts/cluster.yaml ./tests/generic_sink.py 52 | # - RAYVENS_TEST_MODE=local python ./tests/generic_source.py 53 | # - ray submit ./scripts/cluster.yaml ./tests/generic_source.py 54 | - ray submit ./scripts/cluster.yaml ./tests/kafka_transport.py operator 55 | - ray submit ./scripts/cluster.yaml ./tests/kafka_scaling_transport.py operator 56 | 57 | # Test mixed mode 58 | - RAYVENS_TEST_MODE=mixed python ./tests/sink.py 59 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | Rayvens has been contributed to by the following: 18 | 19 | Institutions 20 | ------------ 21 | 22 | (*) IBM TJ Watson Research Center, Yorktown Heights, NY 23 | 24 | Individuals (chronological) 25 | ----------- 26 | 27 | Gheorghe-Teodor Bercea (*) 28 | Olivier Tardieu (*) 29 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Contributing 18 | 19 | This page contains information about reporting issues. 20 | 21 | ## Table of Contents 22 | 23 | * [Reporting an Issue](#reporting-an-issue) 24 | 25 | ## Reporting an Issue 26 | 27 | To report an issue, or to suggest an idea for a change, open an 28 | [issue](../../issues/new). It is best to check 29 | our existing [issues](../../issues) first 30 | to see if a similar one has already been opened and discussed. 31 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | ARG base_image=rayproject/ray:1.13.0-py38 18 | FROM ${base_image} 19 | 20 | COPY --from=docker.io/apache/camel-k:1.5.1 /usr/local/bin/kamel /usr/local/bin/ 21 | 22 | RUN sudo apt-get update \ 23 | && sudo apt-get install -y --no-install-recommends openjdk-11-jdk maven \ 24 | && sudo rm -rf /var/lib/apt/lists/* \ 25 | && sudo apt-get clean 26 | 27 | COPY --chown=ray:users scripts/Preloader.java . 28 | RUN kamel local run Preloader.java --dependency mvn:org.apache.camel.quarkus:camel-quarkus-java-joor-dsl; rm Preloader.java 29 | 30 | COPY --chown=ray:users setup.py rayvens/ 31 | COPY --chown=ray:users rayvens rayvens/rayvens/ 32 | COPY --chown=ray:users scripts/rayvens-setup.sh rayvens/scripts/ 33 | 34 | RUN pip install -e ./rayvens 35 | -------------------------------------------------------------------------------- /Dockerfile.release: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | ARG base_image=rayproject/ray:1.13.0-py38 18 | FROM ${base_image} 19 | 20 | COPY --from=docker.io/apache/camel-k:1.5.1 /usr/local/bin/kamel /usr/local/bin/ 21 | 22 | RUN sudo apt-get update \ 23 | && sudo apt-get install -y --no-install-recommends openjdk-11-jdk maven \ 24 | && sudo rm -rf /var/lib/apt/lists/* \ 25 | && sudo apt-get clean 26 | 27 | ARG rayvens_version 28 | RUN pip install rayvens==${rayvens_version} 29 | RUN rayvens-setup.sh --preload 30 | -------------------------------------------------------------------------------- /examples/aapl.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import json 18 | import ray 19 | import rayvens 20 | import sys 21 | import time 22 | 23 | # This example demonstrates how to use Camel 24 | # to receive and emit external events. 25 | # 26 | # fetch AAPL quotes every 3 seconds 27 | # analyze trend (up/down/stable) 28 | # publish trend to slack 29 | # 30 | # http source -> comparator actor -> slack sink 31 | 32 | # process command line arguments 33 | if len(sys.argv) != 3 and len(sys.argv) != 1: 34 | print(f'usage: {sys.argv[0]} [ ]') 35 | sys.exit(1) 36 | slack_channel = sys.argv[1] if len(sys.argv) == 3 else '' 37 | slack_webhook = sys.argv[2] if len(sys.argv) == 3 else '' 38 | 39 | # initialize ray 40 | ray.init() 41 | 42 | # initialize rayvens 43 | rayvens.init() 44 | 45 | # create a source stream 46 | source_config = dict( 47 | kind='http-source', 48 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 49 | period=3000) 50 | source = rayvens.Stream('http', source_config=source_config) 51 | 52 | # log incoming events 53 | source >> (lambda event: print('LOG:', event)) 54 | 55 | if slack_channel != '': 56 | # create a sink stream 57 | sink_config = dict(kind='slack-sink', 58 | channel=slack_channel, 59 | webhook_url=slack_webhook) 60 | sink = rayvens.Stream('slack', sink_config=sink_config) 61 | 62 | # actor to compare APPL quote with last quote 63 | @ray.remote 64 | class Comparator: 65 | def __init__(self): 66 | self.last_quote = None 67 | 68 | def append(self, event): 69 | payload = json.loads(event) # parse event payload to json 70 | quote = payload['quoteResponse']['result'][0][ 71 | 'regularMarketPrice'] # extract AAPL quote 72 | if self.last_quote: 73 | if quote > self.last_quote: 74 | sink.append('AAPL is up') 75 | elif quote < self.last_quote: 76 | sink.append('AAPL is down') 77 | else: 78 | sink.append('AAPL is stable') 79 | self.last_quote = quote 80 | 81 | comparator = Comparator.remote() # instantiate comparator actor 82 | source.send_to(comparator) # subscribe comparator to source 83 | 84 | # run for a while 85 | time.sleep(120) 86 | -------------------------------------------------------------------------------- /examples/actor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | 20 | # This example demonstrates the use of Ray actors to handle events. 21 | # The events are delivered to each actor in order. The processing order 22 | # is therefore deterministic for each actor. But the interleaving of events 23 | # across actors is arbitrary. 24 | 25 | # initialize ray 26 | ray.init() 27 | 28 | # initialize rayvens 29 | rayvens.init() 30 | 31 | # create a stream 32 | stream = rayvens.Stream('example') 33 | 34 | 35 | # Ray actor to handle events 36 | @ray.remote 37 | class Accumulator: 38 | def __init__(self, name): 39 | self.name = name 40 | self.value = 0 41 | 42 | def append(self, delta): 43 | self.value += delta 44 | print(self.name, '=', self.value) 45 | 46 | 47 | # create two actor instances 48 | acc1 = Accumulator.remote('actor1') 49 | acc2 = Accumulator.remote('actor2') 50 | 51 | # subscribe actors to stream 52 | stream >> acc1.append 53 | stream >> acc2 # .append is implicit if no method name is provided 54 | 55 | # publish a few events 56 | for i in range(10): 57 | stream << i 58 | -------------------------------------------------------------------------------- /examples/batching/batching.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | 21 | # Send message to Slack sink using a stream which batches events. 22 | 23 | # Command line arguments and validation: 24 | if len(sys.argv) < 4: 25 | print(f'usage: {sys.argv[0]} ') 26 | sys.exit(1) 27 | slack_channel = sys.argv[1] 28 | slack_webhook = sys.argv[2] 29 | run_mode = sys.argv[3] 30 | if run_mode not in ['local', 'mixed', 'operator']: 31 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 32 | 33 | # Initialize ray either on the cluster or locally otherwise. 34 | if run_mode == 'operator': 35 | ray.init(address='auto') 36 | else: 37 | ray.init() 38 | 39 | # Start rayvens in operator mode. 40 | rayvens.init(mode=run_mode) 41 | 42 | # Create stream. 43 | stream = rayvens.Stream('slack', batch_size=2) 44 | 45 | 46 | # Operator task: 47 | @ray.remote 48 | def batching_operator(incoming_events): 49 | print(incoming_events) 50 | return " ".join(incoming_events) 51 | 52 | 53 | # Event sink config. 54 | sink_config = dict(kind='slack-sink', 55 | route='/toslack', 56 | channel=slack_channel, 57 | webhook_url=slack_webhook) 58 | 59 | # Add sink to stream. 60 | sink = stream.add_sink(sink_config) 61 | 62 | # Add multi-task operator to stream. 63 | stream.add_operator(batching_operator) 64 | 65 | # Sends messages to all sinks attached to this stream. 66 | stream << "Hello" 67 | stream << "World" 68 | stream << "Hello" 69 | stream << "Mars" 70 | stream << "Hello" 71 | stream << "Jupiter" 72 | 73 | # Disconnect any sources or sinks attached to the stream 2 seconds after 74 | # the stream is idle (i.e. no events were propagated by the stream). 75 | stream.disconnect_all(after_idle_for=2) 76 | -------------------------------------------------------------------------------- /examples/batching/batching_source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import json 21 | 22 | # Send message to Slack sink using a stream which batches events received 23 | # from an HTTP source. 24 | 25 | # Command line arguments and validation: 26 | if len(sys.argv) < 4: 27 | print(f'usage: {sys.argv[0]} ') 28 | sys.exit(1) 29 | slack_channel = sys.argv[1] 30 | slack_webhook = sys.argv[2] 31 | run_mode = sys.argv[3] 32 | if run_mode not in ['local', 'mixed', 'operator']: 33 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 34 | 35 | # Initialize ray either on the cluster or locally otherwise. 36 | if run_mode == 'operator': 37 | ray.init(address='auto') 38 | else: 39 | ray.init() 40 | 41 | # Start rayvens in operator mode. 42 | rayvens.init(mode=run_mode) 43 | 44 | # Create stream. 45 | stream = rayvens.Stream('slack', batch_size=2) 46 | 47 | 48 | # Operator task: 49 | @ray.remote 50 | def batching_operator(incoming_events): 51 | quotes = [] 52 | for event in incoming_events: 53 | payload = json.loads(event) # parse event string to json 54 | quotes.append( 55 | str(payload['quoteResponse']['result'][0]['regularMarketPrice'])) 56 | return " ".join(quotes) 57 | 58 | 59 | # Event sink config. 60 | sink_config = dict(kind='slack-sink', 61 | route='/toslack', 62 | channel=slack_channel, 63 | webhook_url=slack_webhook) 64 | 65 | # Add sink to stream. 66 | sink = stream.add_sink(sink_config) 67 | 68 | # Add multi-task operator to stream. 69 | stream.add_operator(batching_operator) 70 | 71 | # Event source config. 72 | source_config = dict( 73 | kind='http-source', 74 | name='source-1', 75 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 76 | route='/from-http', 77 | period=3000) 78 | 79 | # Attach source to stream. 80 | source = stream.add_source(source_config) 81 | 82 | # Disconnect any sources or sinks attached to the stream 2 seconds after 83 | # the stream is idle (i.e. no events were propagated by the stream). 84 | stream.disconnect_all(after=10) 85 | -------------------------------------------------------------------------------- /examples/batching/batching_with_multitask.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | 21 | # Send message to Slack sink using a stream which batches events and a 22 | # multi-task operator. 23 | 24 | # Command line arguments and validation: 25 | if len(sys.argv) < 4: 26 | print(f'usage: {sys.argv[0]} ') 27 | sys.exit(1) 28 | slack_channel = sys.argv[1] 29 | slack_webhook = sys.argv[2] 30 | run_mode = sys.argv[3] 31 | if run_mode not in ['local', 'mixed', 'operator']: 32 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 33 | 34 | # Initialize ray either on the cluster or locally otherwise. 35 | if run_mode == 'operator': 36 | ray.init(address='auto') 37 | else: 38 | ray.init() 39 | 40 | # Start rayvens in operator mode. 41 | rayvens.init(mode=run_mode) 42 | 43 | # Create stream. 44 | stream = rayvens.Stream('slack', batch_size=6) 45 | 46 | 47 | # Operator sub-task: 48 | @ray.remote 49 | def sub_task(context, sub_event): 50 | context.publish(sub_event) 51 | 52 | 53 | # Operator task: 54 | @ray.remote 55 | def batching_multi_task_operator(context, incoming_events): 56 | for sub_event in incoming_events: 57 | sub_task.remote(context, sub_event) 58 | 59 | 60 | # Event sink config. 61 | sink_config = dict(kind='slack-sink', 62 | route='/toslack', 63 | channel=slack_channel, 64 | webhook_url=slack_webhook) 65 | 66 | # Add sink to stream. 67 | sink = stream.add_sink(sink_config) 68 | 69 | # Add multi-task operator to stream. 70 | stream.add_multitask_operator(batching_multi_task_operator) 71 | 72 | # Sends messages to all sinks attached to this stream. 73 | stream << "Hello" 74 | stream << "World" 75 | stream << "Hello" 76 | stream << "Mars" 77 | stream << "Hello" 78 | stream << "Jupiter" 79 | 80 | # Disconnect any sources or sinks attached to the stream 2 seconds after 81 | # the stream is idle (i.e. no events were propagated by the stream). 82 | stream.disconnect_all(after_idle_for=2) 83 | -------------------------------------------------------------------------------- /examples/binance/binance_source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import time 20 | import sys 21 | import json 22 | 23 | # Source for coin prices. 24 | 25 | # Initialize ray. 26 | if len(sys.argv) < 2: 27 | print(f'usage: {sys.argv[0]} ') 28 | sys.exit(1) 29 | run_mode = sys.argv[1] 30 | 31 | if run_mode not in ['local', 'operator']: 32 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 33 | 34 | # Initialize ray either on the cluster or locally otherwise. 35 | if run_mode == 'operator': 36 | ray.init(address='auto') 37 | else: 38 | ray.init() 39 | 40 | # Start rayvens in the desired mode. 41 | rayvens.init(mode=run_mode) 42 | 43 | # Establish an existing portfolio. 44 | coins = ["BTC", "ETH"] 45 | 46 | # Create stream. 47 | stream = rayvens.Stream('crypto') 48 | 49 | # Event source config. 50 | source_config = dict(kind='binance-source', coin=coins, period='3000') 51 | 52 | # Attach source to stream. 53 | source = stream.add_source(source_config) 54 | 55 | 56 | def process_message(event): 57 | # Parse event: 58 | parsed_event = json.loads(event) 59 | 60 | # Extract currency name: 61 | currency = parsed_event['instrument'].split("/")[0] 62 | 63 | # Extract price: 64 | price = parsed_event['last'] 65 | 66 | # Output latest currency price: 67 | print(f"{currency} : {price}") 68 | 69 | 70 | # Send message to processor. 71 | stream >> process_message 72 | 73 | # Wait before ending program. 74 | time.sleep(20) 75 | 76 | stream.disconnect_all() 77 | -------------------------------------------------------------------------------- /examples/blogs/binance_to_slack.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import json 21 | import time 22 | 23 | # Scaling event streaming from a Binance event source to Slack. 24 | 25 | # Command line arguments and validation: 26 | if len(sys.argv) < 3: 27 | print(f'usage: {sys.argv[0]} ' '') 28 | sys.exit(1) 29 | 30 | # Slack credentials: 31 | slack_channel = sys.argv[1] 32 | slack_webhook = sys.argv[2] 33 | 34 | # Initialize ray either on the cluster or locally otherwise. 35 | ray.init() 36 | 37 | # Start rayvens in operator mode." 38 | rayvens.init(transport="kafka") 39 | 40 | 41 | @ray.remote 42 | def process_currency_price(event): 43 | # Parse event: 44 | parsed_event = json.loads(event) 45 | 46 | # Extract currency name: 47 | currency = parsed_event['instrument'].split("/")[0] 48 | 49 | # Extract price: 50 | price = parsed_event['last'] 51 | 52 | # Log latest currency price: 53 | print(f"{currency} : {price}") 54 | 55 | # Simulate load intensity: 56 | time.sleep(1) 57 | 58 | return f"{currency} : {price}" 59 | 60 | 61 | # Create stream. 62 | stream = rayvens.Stream('kafka-eventing', operator=process_currency_price) 63 | 64 | # Event sink config. 65 | sink_config = dict(kind='slack-sink', 66 | channel=slack_channel, 67 | webhook_url=slack_webhook) 68 | 69 | # Add sink to stream. 70 | sink = stream.add_sink(sink_config) 71 | 72 | # Event source config. 73 | coins = ["BTC", "ETH", "ADA"] 74 | source_config = dict(kind='binance-source', 75 | coin=coins, 76 | period='500', 77 | kafka_transport_partitions=3) 78 | 79 | # Attach source to stream. 80 | source = stream.add_source(source_config) 81 | 82 | # Disconnect source after 8 seconds. 83 | stream.disconnect_all(after=8) 84 | -------------------------------------------------------------------------------- /examples/cloud_object_storage/cos_sink.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import time 21 | import json 22 | 23 | # This example demonstrates how to send objects to the AWS S3 or 24 | # IBM Cloud Object Storage. 25 | 26 | # Parse command-line arguments 27 | if len(sys.argv) < 5: 28 | print(f'usage: {sys.argv[0]} ' 29 | ' []') 30 | sys.exit(1) 31 | bucket = sys.argv[1] 32 | access_key_id = sys.argv[2] 33 | secret_access_key = sys.argv[3] 34 | endpoint = sys.argv[4] 35 | region = None 36 | if len(sys.argv) == 6: 37 | region = sys.argv[5] 38 | 39 | # Initialize Ray and Rayvens 40 | ray.init() 41 | 42 | # TODO: make header setting work for Kafka transport. Currently the 43 | # Camel-K component for Kafka does not propagate message headers. This 44 | # will be fixed by Camel-K 1.8.0 release. 45 | # rayvens.init(transport="kafka") 46 | rayvens.init() 47 | 48 | # Create an object stream 49 | stream = rayvens.Stream('upload-file') 50 | 51 | # Configure the sink 52 | sink_config = dict(kind='cloud-object-storage-sink', 53 | bucket_name=bucket, 54 | access_key_id=access_key_id, 55 | secret_access_key=secret_access_key, 56 | endpoint=endpoint) 57 | 58 | if region is not None: 59 | sink_config['region'] = region 60 | 61 | # Run the sink 62 | sink = stream.add_sink(sink_config) 63 | 64 | # Send file contents to Cloud Object Storage: 65 | json_content = ['test', {'json': ('content', None, 1.0, 2)}] 66 | event = rayvens.OutputEvent(json.dumps(json_content), 67 | {"CamelAwsS3Key": "custom_file.json"}) 68 | stream << event 69 | 70 | # Run for a while 71 | time.sleep(10) 72 | -------------------------------------------------------------------------------- /examples/cloud_object_storage/cos_sink_from_directory.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | 21 | # This example demonstrates how to send objects to the AWS S3 or 22 | # IBM Cloud Object Storage using multi-part uploads. 23 | 24 | # Parse command-line arguments 25 | if len(sys.argv) < 7: 26 | print(f'usage: {sys.argv[0]} ' 27 | ' ') 28 | sys.exit(1) 29 | bucket = sys.argv[1] 30 | access_key_id = sys.argv[2] 31 | secret_access_key = sys.argv[3] 32 | endpoint = sys.argv[4] 33 | region = sys.argv[5] 34 | run_mode = sys.argv[6] 35 | 36 | # Initialize Ray and Rayvens 37 | if run_mode not in ['local', 'operator']: 38 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 39 | 40 | # Initialize ray either on the cluster or locally otherwise. 41 | if run_mode == 'operator': 42 | ray.init(address='auto') 43 | else: 44 | ray.init() 45 | 46 | # Start rayvens in the desired mode. 47 | rayvens.init(mode=run_mode) 48 | 49 | # Create an object stream 50 | stream = rayvens.Stream('upload-file') 51 | 52 | # Configure the sink to upload a message from a file system directory whenever 53 | # a new file is dumped inside the directory. The upload will consume (i.e. 54 | # delete) the file unless the `keep_file` is set to True. The upload event is 55 | # triggered automatically without any user intervention. 56 | dir_to_sink_config = dict(kind='cloud-object-storage-sink', 57 | name='sink', 58 | bucket_name=bucket, 59 | access_key_id=access_key_id, 60 | secret_access_key=secret_access_key, 61 | endpoint=endpoint, 62 | from_directory="test_files_copy/") 63 | 64 | if region is not None: 65 | dir_to_sink_config['region'] = region 66 | 67 | # Run the sink which gets automatically triggered whenever a file is dumped in 68 | # the monitored file system directory. 69 | dir_to_sink = stream.add_sink(dir_to_sink_config) 70 | 71 | # Run for a while to give a chance for files to be dropped inside the directory 72 | stream.disconnect_all(after_idle_for=20) 73 | -------------------------------------------------------------------------------- /examples/cloud_object_storage/cos_sink_from_file.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | from pathlib import Path 21 | 22 | # This example demonstrates how to send objects to the AWS S3 or 23 | # IBM Cloud Object Storage using multi-part uploads. 24 | 25 | # Parse command-line arguments 26 | if len(sys.argv) < 7: 27 | print(f'usage: {sys.argv[0]} ' 28 | ' ') 29 | sys.exit(1) 30 | bucket = sys.argv[1] 31 | access_key_id = sys.argv[2] 32 | secret_access_key = sys.argv[3] 33 | endpoint = sys.argv[4] 34 | region = sys.argv[5] 35 | run_mode = sys.argv[6] 36 | 37 | # Initialize Ray and Rayvens 38 | if run_mode not in ['local', 'operator']: 39 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 40 | 41 | # Initialize ray either on the cluster or locally otherwise. 42 | if run_mode == 'operator': 43 | ray.init(address='auto') 44 | else: 45 | ray.init() 46 | 47 | # Start rayvens in the desired mode. 48 | rayvens.init(mode=run_mode) 49 | 50 | # Create an object stream 51 | stream = rayvens.Stream('upload-file') 52 | 53 | # Configure the sink to upload a message from a file system file whenever 54 | # the file is updated. The upload will consume (i.e. delete) the file unless 55 | # the `keep_file` is set to True. The upload event is triggered 56 | # automatically without any user intervention if the file is present initially 57 | # or created/written after the sink is started. The file is uploaded in parts 58 | # but since no custom part size has been specified, the default 25 MB part 59 | # size will be used. 60 | file_to_sink_config = dict(kind='cloud-object-storage-sink', 61 | name='sink-1', 62 | bucket_name=bucket, 63 | access_key_id=access_key_id, 64 | secret_access_key=secret_access_key, 65 | endpoint=endpoint, 66 | from_file="test_files/test2.txt") 67 | 68 | # Configure the sink to upload a message from a file system file 69 | # in chunks of 2 MB whenever the file is updated. This sink can also 70 | # upload, on demand, any other file system file which is provided by 71 | # the user. 72 | advanced_sink_config = dict(kind='cloud-object-storage-sink', 73 | name='sink-2', 74 | bucket_name=bucket, 75 | access_key_id=access_key_id, 76 | secret_access_key=secret_access_key, 77 | endpoint=endpoint, 78 | upload_type="multi-part", 79 | part_size=2 * 1024 * 1024, 80 | from_file="test_files/test3.txt") 81 | 82 | if region is not None: 83 | advanced_sink_config['region'] = region 84 | file_to_sink_config['region'] = region 85 | 86 | # Run the sink which gets automatically triggered whenever the `from_file` 87 | # becomes available. 88 | file_to_sink = stream.add_sink(file_to_sink_config) 89 | 90 | # Run the more advanced sink which can do both the on demand uploads and the 91 | # automatic uploads. 92 | advanced_sink = stream.add_sink(advanced_sink_config) 93 | 94 | # The on demand file upload will only reach the advanced_sink. The message 95 | # for the other sink will be dropped as the automatically triggered sink 96 | # cannot receive events directly from Rayvens. 97 | if run_mode == "local": 98 | stream << Path("test_files/test.txt") 99 | 100 | # This message type is not accepted by advanced_sink so it will not be sent 101 | # to the advanced_sink. 102 | stream << "Some other input which is invalid." 103 | 104 | # Run for a while 105 | stream.disconnect_all(after_idle_for=5) 106 | -------------------------------------------------------------------------------- /examples/cloud_object_storage/cos_sink_multi_part.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | from pathlib import Path 21 | 22 | # This example demonstrates how to send objects to the AWS S3 or 23 | # IBM Cloud Object Storage using multi-part uploads. 24 | 25 | # Parse command-line arguments 26 | if len(sys.argv) < 7: 27 | print(f'usage: {sys.argv[0]} ' 28 | ' ') 29 | sys.exit(1) 30 | bucket = sys.argv[1] 31 | access_key_id = sys.argv[2] 32 | secret_access_key = sys.argv[3] 33 | endpoint = sys.argv[4] 34 | region = sys.argv[5] 35 | run_mode = sys.argv[6] 36 | 37 | # Initialize Ray and Rayvens 38 | if run_mode not in ['local', 'operator']: 39 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 40 | 41 | # Initialize ray either on the cluster or locally otherwise. 42 | if run_mode == 'operator': 43 | ray.init(address='auto') 44 | else: 45 | ray.init() 46 | 47 | # Start rayvens in the desired mode. 48 | rayvens.init(mode=run_mode) 49 | 50 | # Create an object stream 51 | stream = rayvens.Stream('upload-file') 52 | 53 | # Configure the sink to upload a file in 2 MB chunks on demand i.e. when 54 | # the user passes the file as `Path("test_files/test.txt")` to the stream. 55 | # The file is not deleted after upload. 56 | sink_config = dict(kind='cloud-object-storage-sink', 57 | bucket_name=bucket, 58 | access_key_id=access_key_id, 59 | secret_access_key=secret_access_key, 60 | endpoint=endpoint, 61 | upload_type="multi-part", 62 | part_size=2 * 1024 * 1024) 63 | 64 | if region is not None: 65 | sink_config['region'] = region 66 | 67 | # Run the sink 68 | sink = stream.add_sink(sink_config) 69 | 70 | # Upload a local file to the COS. 71 | if run_mode == "local": 72 | stream << Path("test_files/test.txt") 73 | 74 | # Run for a while 75 | stream.disconnect_all(after_idle_for=5) 76 | -------------------------------------------------------------------------------- /examples/cloud_object_storage/cos_sink_multitask.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import time 21 | 22 | # This example demonstrates how to send objects to the AWS S3 or 23 | # IBM Cloud Object Storage. 24 | 25 | # Parse command-line arguments 26 | if len(sys.argv) < 5: 27 | print(f'usage: {sys.argv[0]} ' 28 | ' []') 29 | sys.exit(1) 30 | bucket = sys.argv[1] 31 | access_key_id = sys.argv[2] 32 | secret_access_key = sys.argv[3] 33 | endpoint = sys.argv[4] 34 | region = None 35 | if len(sys.argv) == 6: 36 | region = sys.argv[5] 37 | 38 | # Initialize Ray and Rayvens 39 | ray.init() 40 | 41 | # TODO: make header setting work for Kafka transport. Currently the 42 | # Camel-K component for Kafka does not propagate message headers. This 43 | # will be fixed by Camel-K 1.8.0 release. 44 | # rayvens.init(transport="kafka") 45 | rayvens.init() 46 | 47 | # Create an object stream 48 | stream = rayvens.Stream('upload-file') 49 | 50 | # Configure the sink 51 | sink_config = dict(kind='cloud-object-storage-sink', 52 | bucket_name=bucket, 53 | access_key_id=access_key_id, 54 | secret_access_key=secret_access_key, 55 | endpoint=endpoint) 56 | 57 | if region is not None: 58 | sink_config['region'] = region 59 | 60 | # Run the sink 61 | sink = stream.add_sink(sink_config) 62 | 63 | 64 | # Operator sub-task: 65 | @ray.remote 66 | def sub_task(context, intermediate_data): 67 | contents = "sub-task " + intermediate_data 68 | print(contents) 69 | sub_task_outgoing_event = rayvens.OutputEvent( 70 | contents, 71 | {"CamelAwsS3Key": "custom_file_" + intermediate_data + ".json"}) 72 | context.publish(sub_task_outgoing_event) 73 | 74 | 75 | # Operator task: 76 | @ray.remote 77 | def multi_part_task(context, incoming_event): 78 | print("multi-part-task:", incoming_event) 79 | for i in range(3): 80 | sub_task.remote(context, str(i)) 81 | 82 | 83 | # Add multi-task operator to stream. 84 | stream.add_multitask_operator(multi_part_task) 85 | 86 | # Send file contents to Cloud Object Storage: 87 | stream << "Random Event" 88 | 89 | # Run for a while 90 | time.sleep(10) 91 | -------------------------------------------------------------------------------- /examples/cloud_object_storage/cos_source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import json 21 | 22 | # This example demonstrates how to receive objects from AWS S3 or 23 | # IBM Cloud Object Storage. It requires a bucket name, HMAC credentials, 24 | # and the endpoint url. It simply pulls objects from the bucket and 25 | # output the size of each object. 26 | # 27 | # WARNING: This example program deletes the objects from the bucket. 28 | 29 | # Parse command-line arguments 30 | if len(sys.argv) < 5: 31 | print(f'usage: {sys.argv[0]} ' 32 | ' []') 33 | sys.exit(1) 34 | bucket = sys.argv[1] 35 | access_key_id = sys.argv[2] 36 | secret_access_key = sys.argv[3] 37 | endpoint = sys.argv[4] 38 | region = None 39 | if len(sys.argv) == 6: 40 | region = sys.argv[5] 41 | 42 | # Initialize Ray and Rayvens 43 | ray.init() 44 | rayvens.init() 45 | 46 | # Create an object stream 47 | stream = rayvens.Stream('bucket') 48 | 49 | # Configure the source 50 | source_config = dict(kind='cloud-object-storage-source', 51 | bucket_name=bucket, 52 | access_key_id=access_key_id, 53 | secret_access_key=secret_access_key, 54 | endpoint=endpoint) 55 | 56 | if region is not None: 57 | source_config['region'] = region 58 | 59 | # Run the source 60 | source = stream.add_source(source_config) 61 | 62 | 63 | def process_file(event): 64 | print(f'received {len(event)} bytes') 65 | json_event = json.loads(event) 66 | print("Contents:") 67 | print(json_event['filename']) 68 | print(json_event['body']) 69 | 70 | 71 | # Log object sizes to the console 72 | stream >> process_file 73 | 74 | # Run for a while 75 | stream.disconnect_all(after_idle_for=2) 76 | -------------------------------------------------------------------------------- /examples/cloud_object_storage/cos_source_meta.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import json 21 | 22 | # This example demonstrates how to receive events about data becoming available 23 | # in Cloud Object Storage. 24 | 25 | # No data will be transferred 26 | 27 | # Parse command-line arguments 28 | if len(sys.argv) < 5: 29 | print(f'usage: {sys.argv[0]} ' 30 | ' []') 31 | sys.exit(1) 32 | bucket = sys.argv[1] 33 | access_key_id = sys.argv[2] 34 | secret_access_key = sys.argv[3] 35 | endpoint = sys.argv[4] 36 | region = None 37 | if len(sys.argv) == 6: 38 | region = sys.argv[5] 39 | 40 | # Initialize Ray and Rayvens 41 | ray.init() 42 | rayvens.init() 43 | 44 | # Create an object stream 45 | stream = rayvens.Stream('bucket') 46 | 47 | # Configure the source 48 | source_config = dict(kind='cloud-object-storage-source', 49 | bucket_name=bucket, 50 | access_key_id=access_key_id, 51 | secret_access_key=secret_access_key, 52 | endpoint=endpoint, 53 | meta_event_only=True) 54 | 55 | if region is not None: 56 | source_config['region'] = region 57 | 58 | # Run the source 59 | source = stream.add_source(source_config) 60 | 61 | 62 | def process_file(event): 63 | print(f'received {len(event)} bytes') 64 | json_event = json.loads(event) 65 | print("Contents:") 66 | print(json_event['filename']) 67 | 68 | 69 | # Log object sizes to the console 70 | stream >> process_file 71 | 72 | # Run for a while 73 | stream.disconnect_all(after_idle_for=2) 74 | -------------------------------------------------------------------------------- /examples/cloud_object_storage/cos_source_move_after_read.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import json 21 | 22 | # This example demonstrates how to receive objects from AWS S3 or 23 | # IBM Cloud Object Storage. It requires a bucket name, HMAC credentials, 24 | # and the endpoint url. It simply pulls objects from the bucket and 25 | # output the size of each object. 26 | # 27 | # WARNING: This example program deletes the objects from the bucket. 28 | 29 | # Parse command-line arguments 30 | if len(sys.argv) < 5: 31 | print(f'usage: {sys.argv[0]} ' 32 | ' []') 33 | sys.exit(1) 34 | bucket = sys.argv[1] 35 | access_key_id = sys.argv[2] 36 | secret_access_key = sys.argv[3] 37 | endpoint = sys.argv[4] 38 | region = None 39 | if len(sys.argv) == 6: 40 | region = sys.argv[5] 41 | 42 | # Initialize Ray and Rayvens 43 | ray.init() 44 | rayvens.init() 45 | 46 | # Create an object stream 47 | stream = rayvens.Stream('bucket') 48 | 49 | # Configure the source 50 | source_config = dict(kind='cloud-object-storage-source', 51 | bucket_name=bucket, 52 | access_key_id=access_key_id, 53 | secret_access_key=secret_access_key, 54 | endpoint=endpoint, 55 | move_after_read="new-bucket-name") 56 | 57 | if region is not None: 58 | source_config['region'] = region 59 | 60 | # Run the source 61 | source = stream.add_source(source_config) 62 | 63 | 64 | def process_file(event): 65 | print(f'received {len(event)} bytes') 66 | json_event = json.loads(event) 67 | print("Contents:") 68 | print(json_event['filename']) 69 | print(json_event['body']) 70 | 71 | 72 | # Log object sizes to the console 73 | stream >> process_file 74 | 75 | # Run for a while 76 | stream.disconnect_all(after_idle_for=2) 77 | -------------------------------------------------------------------------------- /examples/cloud_object_storage/test_files/test.txt: -------------------------------------------------------------------------------- 1 | This is just a test file! 2 | -------------------------------------------------------------------------------- /examples/datasets/file_source_to_dataset.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import json 21 | import time 22 | 23 | # This example demonstrates how to receive events. 24 | 25 | # Parse command-line arguments 26 | if len(sys.argv) < 2: 27 | print(f'usage: {sys.argv[0]} ') 28 | sys.exit(1) 29 | 30 | # Check run mode: 31 | run_mode = sys.argv[1] 32 | if run_mode not in ['local', 'operator']: 33 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 34 | 35 | # Initialize ray either on the cluster or locally otherwise. 36 | if run_mode == 'operator': 37 | ray.init(address='auto') 38 | else: 39 | ray.init() 40 | 41 | # Start rayvens in the desired mode. 42 | rayvens.init(mode=run_mode) 43 | 44 | # Create an object stream: 45 | stream = rayvens.Stream('bucket') 46 | 47 | # Configure the source: 48 | source_config = dict(kind='file-watch-source', 49 | path='test_files', 50 | events='CREATE') 51 | 52 | # Run the source: 53 | source = stream.add_source(source_config) 54 | 55 | 56 | @ray.remote 57 | class Filename: 58 | def __init__(self): 59 | self.filename = None 60 | 61 | def set_filename(self, filename): 62 | self.filename = filename 63 | 64 | def get_filename(self): 65 | return self.filename 66 | 67 | 68 | filename_obj = Filename.remote() 69 | 70 | 71 | def process_file(event, filename_obj): 72 | print(f'received {len(event)} bytes') 73 | json_event = json.loads(event) 74 | print("Contents:") 75 | print("Filename:", json_event['filename']) 76 | print("Event type:", json_event['event_type']) 77 | filename_obj.set_filename.remote(json_event['filename']) 78 | 79 | # filename = json_event['filename'] 80 | # WARNING: Cannot pickle Ray itself so we cannot read a 81 | # file using Datasets API in response to an event. 82 | # ds = ray.experimental.data.read_json([filename]) 83 | # print(ds) 84 | 85 | 86 | # Process incoming file name. 87 | stream >> (lambda event: process_file(event, filename_obj)) 88 | 89 | # Create a data set and write the csv file using datasets. 90 | # TODO: Ray 1.5.1 uses pandas to write a CSV file so we avoid 91 | # using this method for writing CSV files. 92 | # test_ds = ray.experimental.data.range(100) 93 | # test_ds.write_csv("test_files/test.csv") 94 | 95 | # Read JSON file to Ray dataset: 96 | timeout_counter = 100 97 | filename = ray.get(filename_obj.get_filename.remote()) 98 | while filename is None and timeout_counter > 0: 99 | filename = ray.get(filename_obj.get_filename.remote()) 100 | timeout_counter -= 1 101 | time.sleep(1) 102 | if filename is not None: 103 | ds = ray.experimental.data.read_json([filename]) 104 | print(ds) 105 | print("Dataset constructed correctly") 106 | else: 107 | print("No file was received") 108 | 109 | # Run while events are still being received then stop if not. 110 | stream.disconnect_all(after_idle_for=2) 111 | -------------------------------------------------------------------------------- /examples/datasets/file_source_to_dataset_actor.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import json 21 | import time 22 | 23 | # This example demonstrates how to receive events. 24 | 25 | # Parse command-line arguments 26 | if len(sys.argv) < 2: 27 | print(f'usage: {sys.argv[0]} ') 28 | sys.exit(1) 29 | 30 | # Check run mode: 31 | run_mode = sys.argv[1] 32 | if run_mode not in ['local', 'operator']: 33 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 34 | 35 | # Initialize ray either on the cluster or locally otherwise. 36 | if run_mode == 'operator': 37 | ray.init(address='auto') 38 | else: 39 | ray.init() 40 | 41 | # Start rayvens in the desired mode. 42 | rayvens.init(mode=run_mode) 43 | 44 | # Create an object stream: 45 | stream = rayvens.Stream('bucket') 46 | 47 | # Configure the source: 48 | source_config = dict(kind='file-watch-source', 49 | path='test_files', 50 | events='CREATE') 51 | 52 | # Run the source: 53 | source = stream.add_source(source_config) 54 | 55 | 56 | @ray.remote 57 | class Filename: 58 | def __init__(self): 59 | self.filename = None 60 | 61 | def set_filename(self, event): 62 | print(f'received {len(event)} bytes') 63 | json_event = json.loads(event) 64 | print("Contents:") 65 | print("Filename:", json_event['filename']) 66 | print("Event type:", json_event['event_type']) 67 | self.filename = json_event['filename'] 68 | 69 | def get_filename(self): 70 | return self.filename 71 | 72 | 73 | # Instantiate actor. 74 | filename_actor = Filename.remote() 75 | 76 | # Process incoming file name. 77 | stream >> filename_actor.set_filename 78 | 79 | # Read JSON file to Ray dataset: 80 | timeout_counter = 100 81 | filename = None 82 | while filename is None and timeout_counter > 0: 83 | filename = ray.get(filename_actor.get_filename.remote()) 84 | timeout_counter -= 1 85 | time.sleep(1) 86 | if filename is not None: 87 | ds = ray.experimental.data.read_json([filename]) 88 | print(ds) 89 | print("Dataset constructed correctly") 90 | else: 91 | print("No file was received") 92 | 93 | # Run while events are still being received then stop if not. 94 | stream.disconnect_all(after_idle_for=2) 95 | -------------------------------------------------------------------------------- /examples/datasets/test_files/test.txt: -------------------------------------------------------------------------------- 1 | {"greeting": "Hello!"} -------------------------------------------------------------------------------- /examples/file/file_source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | 21 | # This example shows how a file is used to trigger an event to process 22 | # the file. 23 | 24 | # Parse command-line arguments 25 | if len(sys.argv) < 2: 26 | print(f'usage: {sys.argv[0]} ') 27 | sys.exit(1) 28 | run_mode = sys.argv[1] 29 | 30 | # Initialize Ray and Rayvens: 31 | if run_mode == 'operator': 32 | ray.init(address='auto') 33 | else: 34 | ray.init() 35 | rayvens.init(mode=run_mode, release=True) 36 | 37 | # Create an object stream: 38 | stream = rayvens.Stream('file-process') 39 | 40 | # Configure the source: 41 | source_config = dict(kind='file-source-raw', 42 | path='test_files/test.txt', 43 | keep_file=True) 44 | 45 | # Run the source: 46 | source = stream.add_source(source_config) 47 | 48 | def process_file(event): 49 | print(f'Received bytes: {len(event)}.') 50 | print(f"Contents: {event}") 51 | 52 | # Log object sizes to the console: 53 | stream >> process_file 54 | 55 | # Run for a while. 56 | stream.disconnect_all(after_idle_for=2) 57 | 58 | ray.shutdown() 59 | -------------------------------------------------------------------------------- /examples/file/file_watch_source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import json 21 | 22 | # This example demonstrates how to receive events. 23 | 24 | # Parse command-line arguments 25 | if len(sys.argv) < 2: 26 | print(f'usage: {sys.argv[0]} ') 27 | sys.exit(1) 28 | 29 | # Check run mode: 30 | run_mode = sys.argv[1] 31 | if run_mode not in ['local', 'operator']: 32 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 33 | 34 | # Initialize ray either on the cluster or locally otherwise. 35 | if run_mode == 'operator': 36 | ray.init(address='auto') 37 | else: 38 | ray.init() 39 | 40 | # Start rayvens in the desired mode. 41 | rayvens.init(mode=run_mode) 42 | 43 | # Create an object stream: 44 | stream = rayvens.Stream('bucket') 45 | 46 | # Configure the source: 47 | source_config = dict(kind='file-watch-source', 48 | path='test_files', 49 | events='DELETE,CREATE') 50 | 51 | # Run the source: 52 | source = stream.add_source(source_config) 53 | 54 | 55 | def process_file(event): 56 | print(f'received {len(event)} bytes') 57 | json_event = json.loads(event) 58 | print("Contents:") 59 | print(json_event['filename']) 60 | print(json_event['event_type']) 61 | 62 | 63 | # Log object sizes to the console 64 | stream >> process_file 65 | 66 | # Run for a while 67 | stream.disconnect_all(after_idle_for=2) 68 | -------------------------------------------------------------------------------- /examples/file/test_files/test.txt: -------------------------------------------------------------------------------- 1 | This is just a test file! -------------------------------------------------------------------------------- /examples/function.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | 20 | # This example demonstrates the use of Python functions to handle events. 21 | # These functions are invoked synchronously in order. The processing order 22 | # is therefore deterministic (within and across the two subscribers). 23 | 24 | # initialize ray 25 | ray.init() 26 | 27 | # initialize rayvens 28 | rayvens.init() 29 | 30 | # create a stream 31 | stream = rayvens.Stream('example') 32 | 33 | 34 | # define a first event handler 35 | def handler1(event): 36 | print('handler1 received', event) 37 | 38 | 39 | # define a second event handler 40 | def handler2(event): 41 | print('handler2 received', event) 42 | 43 | 44 | # subscribe handlers to stream 45 | stream >> handler1 46 | stream >> handler2 47 | 48 | # publish a few events 49 | for i in range(10): 50 | stream << f'event {i}' 51 | -------------------------------------------------------------------------------- /examples/generic/generic_sink.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import time 20 | import sys 21 | 22 | # Send message to a Slack channel. 23 | 24 | # Initialize ray. 25 | if len(sys.argv) < 4: 26 | print(f'usage: {sys.argv[0]} ') 27 | sys.exit(1) 28 | slack_channel = sys.argv[1] 29 | slack_webhook = sys.argv[2] 30 | run_mode = sys.argv[3] 31 | 32 | if run_mode not in ['local', 'operator']: 33 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 34 | 35 | # Initialize ray either on the cluster or locally otherwise. 36 | if run_mode == 'operator': 37 | ray.init(address='auto') 38 | else: 39 | ray.init() 40 | 41 | # Start rayvens in the desired mode. 42 | rayvens.init(mode=run_mode) 43 | 44 | # Create stream. 45 | stream = rayvens.Stream('slack') 46 | 47 | # Event sink config. 48 | sink_config = dict(kind='generic-sink', 49 | uri=f"slack:{slack_channel}?webhookUrl={slack_webhook}") 50 | 51 | # Attach source to stream. 52 | sink = stream.add_sink(sink_config) 53 | 54 | # Sends message to all sinks attached to this stream. 55 | stream << f'Sending message to Slack generic sink in run mode {run_mode}.' 56 | 57 | # Wait before ending program. 58 | time.sleep(20) 59 | 60 | stream.disconnect_all() 61 | -------------------------------------------------------------------------------- /examples/generic/generic_source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import time 20 | import sys 21 | 22 | # Receive message from stock price source and print it to console. 23 | 24 | # Initialize ray. 25 | if len(sys.argv) < 2: 26 | print(f'usage: {sys.argv[0]} ') 27 | sys.exit(1) 28 | run_mode = sys.argv[1] 29 | if run_mode not in ['local', 'operator']: 30 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 31 | 32 | # Initialize ray either on the cluster or locally otherwise. 33 | if run_mode == 'operator': 34 | ray.init(address='auto') 35 | else: 36 | ray.init() 37 | 38 | # Start rayvens in the desired mode. 39 | rayvens.init(mode=run_mode) 40 | 41 | # Create stream. 42 | stream = rayvens.Stream('http') 43 | 44 | # Event source config. 45 | source_config = dict(kind='generic-source', 46 | spec=""" 47 | - from: 48 | uri: timer:tick?period=3000 49 | steps: 50 | - to: https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL 51 | """) 52 | 53 | # Attach source to stream. 54 | source = stream.add_source(source_config) 55 | 56 | # Log all events from stream-attached sources. 57 | stream >> (lambda event: print('LOG:', event)) 58 | 59 | # Wait before ending program. 60 | time.sleep(20) 61 | 62 | stream.disconnect_all() 63 | -------------------------------------------------------------------------------- /examples/generic/periodic_generic_source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import time 20 | import sys 21 | 22 | # Receive message from stock price source and print it to console. 23 | 24 | # Initialize ray. 25 | if len(sys.argv) < 2: 26 | print(f'usage: {sys.argv[0]} ') 27 | sys.exit(1) 28 | run_mode = sys.argv[1] 29 | if run_mode not in ['local', 'operator']: 30 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 31 | 32 | # Initialize ray either on the cluster or locally otherwise. 33 | if run_mode == 'operator': 34 | ray.init(address='auto') 35 | else: 36 | ray.init() 37 | 38 | # Start rayvens in the desired mode. 39 | rayvens.init(mode=run_mode) 40 | 41 | # Create stream. 42 | stream = rayvens.Stream('http') 43 | 44 | # Event source config. 45 | source_config = dict( 46 | kind='generic-periodic-source', 47 | uri="https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL") 48 | 49 | # Attach source to stream. 50 | source = stream.add_source(source_config) 51 | 52 | # Log all events from stream-attached sources. 53 | stream >> (lambda event: print('LOG:', event)) 54 | 55 | # Wait before ending program. 56 | time.sleep(20) 57 | 58 | stream.disconnect_all() 59 | -------------------------------------------------------------------------------- /examples/http/http_source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | 21 | # Receive message from stock price source and print it to console using the 22 | # operator implementation. 23 | 24 | # Initialize ray. 25 | if len(sys.argv) < 2: 26 | print(f'usage: {sys.argv[0]} ') 27 | sys.exit(1) 28 | run_mode = sys.argv[1] 29 | if run_mode not in ['local', 'operator']: 30 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 31 | 32 | # Initialize ray either on the cluster or locally otherwise. 33 | if run_mode == 'operator': 34 | ray.init(address='auto') 35 | else: 36 | ray.init() 37 | 38 | # Start rayvens in the desired mode. 39 | rayvens.init(mode=run_mode) 40 | 41 | # Create stream. 42 | stream = rayvens.Stream('http') 43 | 44 | # Event source config. 45 | source_config = dict( 46 | kind='http-source', 47 | name='source-1', 48 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 49 | route='/from-http', 50 | period=3000) 51 | 52 | # Attach source to stream. 53 | source = stream.add_source(source_config) 54 | 55 | # Event source config. 56 | another_source_config = dict( 57 | kind='http-source', 58 | name='source-2', 59 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 60 | route='/from-another-http', 61 | period=5000) 62 | 63 | # Attach source to stream. 64 | another_source = stream.add_source(another_source_config) 65 | 66 | # Log all events from stream-attached sources. 67 | stream >> (lambda event: print('LOG:', event)) 68 | 69 | # Disconnect all sources after 20 seconds. 70 | stream.disconnect_all(after=20) 71 | -------------------------------------------------------------------------------- /examples/kafka/kafka_event_streaming_sink.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | 21 | # Send message to Slack sink using the kafka transport. 22 | 23 | # Command line arguments and validation: 24 | if len(sys.argv) < 4: 25 | print(f'usage: {sys.argv[0]} ' 26 | ' OR' 27 | f' {sys.argv[0]} ') 28 | sys.exit(1) 29 | 30 | # Brokers and run mode: 31 | brokers = None 32 | password = None 33 | slack_channel = sys.argv[1] 34 | slack_webhook = sys.argv[2] 35 | run_mode = sys.argv[3] 36 | if len(sys.argv) == 6: 37 | brokers = sys.argv[1] 38 | password = sys.argv[2] 39 | slack_channel = sys.argv[3] 40 | slack_webhook = sys.argv[4] 41 | run_mode = sys.argv[5] 42 | 43 | if run_mode not in ['local', 'mixed', 'operator']: 44 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 45 | 46 | # The Kafka topic used for communication. 47 | topic = "externalTopicSink" 48 | 49 | # Initialize ray either on the cluster or locally otherwise. 50 | if run_mode == 'operator': 51 | ray.init(address='auto') 52 | else: 53 | ray.init() 54 | 55 | # Start rayvens in operator mode." 56 | rayvens.init(mode=run_mode, transport="kafka") 57 | 58 | # Create stream. 59 | stream = rayvens.Stream('slack') 60 | 61 | # Event sink config. 62 | sink_config = dict(kind='slack-sink', 63 | channel=slack_channel, 64 | webhook_url=slack_webhook, 65 | kafka_transport_topic=topic, 66 | kafka_transport_partitions=3) 67 | 68 | # Add sink to stream. 69 | sink = stream.add_sink(sink_config) 70 | 71 | # Sends message to all sinks attached to this stream. 72 | stream << f'Message to Slack sink in run mode {run_mode} and Kafka transport.' 73 | 74 | # Disconnect any sources or sinks attached to the stream 2 seconds after 75 | # the stream is idle (i.e. no events were propagated by the stream). 76 | stream.disconnect_all(after_idle_for=2) 77 | -------------------------------------------------------------------------------- /examples/kafka/kafka_event_streaming_source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | 21 | # Event streaming from a third-party external source using Kafka. 22 | 23 | # Command line arguments and validation: 24 | if len(sys.argv) < 2: 25 | print(f'usage: {sys.argv[0]} OR' 26 | f' {sys.argv[0]} ') 27 | sys.exit(1) 28 | 29 | # Brokers and run mode: 30 | brokers = None 31 | password = None 32 | run_mode = sys.argv[1] 33 | if len(sys.argv) == 4: 34 | brokers = sys.argv[1] 35 | password = sys.argv[2] 36 | run_mode = sys.argv[3] 37 | 38 | if run_mode not in ['local', 'mixed', 'operator']: 39 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 40 | 41 | # The Kafka topic used for communication. 42 | topic = "externalTopicSource" 43 | 44 | # Initialize ray either on the cluster or locally otherwise. 45 | if run_mode == 'operator': 46 | ray.init(address='auto') 47 | else: 48 | ray.init() 49 | 50 | # Start rayvens in operator mode." 51 | rayvens.init(mode=run_mode, transport="kafka") 52 | 53 | # Create stream. 54 | stream = rayvens.Stream('http') 55 | 56 | # Log all events from stream-attached sources. 57 | stream >> (lambda event: print('LOG:', event)) 58 | 59 | # Event source config. 60 | source_config = dict( 61 | kind='http-source', 62 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 63 | route='/from-http', 64 | period=1000, 65 | kafka_transport_topic=topic, 66 | kafka_transport_partitions=3) 67 | 68 | # Attach source to stream. 69 | source = stream.add_source(source_config) 70 | 71 | # Disconnect source after 10 seconds. 72 | stream.disconnect_all(after=10) 73 | -------------------------------------------------------------------------------- /examples/kafka/kafka_partitioned_source_sink.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import time 21 | 22 | # An artificial example of using Kafka sources and sink. 23 | # Typically the user application will interact with an external Kafka 24 | # service to either subscribe or publish data to other services: 25 | # 26 | # EXT. SERVICE => KAFKA => RAYVENS KAFKA SOURCE 27 | # or 28 | # RAYVENS KAFKA SINK => KAFKA => EXT. SERVICE 29 | # 30 | # In this example we put together an artificial example where, to 31 | # demonstrate both Kafka sources and sinks at the same time we 32 | # set a Kafka sink to publish to a test topic then have a Kafka 33 | # source read from that test topic: 34 | # 35 | # RAYVENS KAFKA SINK => KAFKA => RAYVENS KAFKA SOURCE 36 | # 37 | 38 | # Command line arguments and validation: 39 | if len(sys.argv) < 2: 40 | print(f'usage: {sys.argv[0]} OR' 41 | f' {sys.argv[0]} ') 42 | sys.exit(1) 43 | 44 | # Brokers and run mode: 45 | brokers = None 46 | password = None 47 | run_mode = sys.argv[1] 48 | if len(sys.argv) == 4: 49 | brokers = sys.argv[1] 50 | password = sys.argv[2] 51 | run_mode = sys.argv[3] 52 | 53 | if run_mode not in ['local', 'mixed', 'operator']: 54 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 55 | 56 | # The Kafka topic used for communication. 57 | topic = "externalTopic" 58 | 59 | # If using the Kafka broker started by Rayvens the following brokers 60 | # are possible: 61 | # - from inside the cluster: kafka:9092 62 | # - from outside the cluster: localhost:31093 63 | # If using a different Kafka service please provide the brokers in the 64 | # form of host:port,host1:port1, ... . 65 | if brokers is None: 66 | brokers = 'localhost:31093' 67 | if run_mode == 'operator': 68 | brokers = "kafka:9092" 69 | 70 | # Initialize ray either on the cluster or locally otherwise. 71 | if run_mode == 'operator': 72 | ray.init(address='auto') 73 | else: 74 | ray.init() 75 | 76 | # Start rayvens in operator mode. 77 | rayvens.init(mode=run_mode) 78 | 79 | # Create source stream and configuration. 80 | source_stream = rayvens.Stream('kafka-source-stream') 81 | 82 | source_config = dict(kind='kafka-source', 83 | route='/fromkafka', 84 | topic=topic, 85 | brokers=brokers, 86 | partitions=3) 87 | if password is not None: 88 | source_config['SASL_password'] = password 89 | source = source_stream.add_source(source_config) 90 | # Log all events from stream-attached sources. 91 | source_stream >> (lambda event: print('KAFKA SOURCE:', event)) 92 | 93 | # Create sink stream and configuration. 94 | sink_stream = rayvens.Stream('kafka-sink-stream') 95 | sink_config = dict(kind='kafka-sink', 96 | route='/tokafka', 97 | topic=topic, 98 | brokers=brokers, 99 | partitions=3) 100 | if password is not None: 101 | sink_config['SASL_password'] = password 102 | sink = sink_stream.add_sink(sink_config) 103 | 104 | time.sleep(10) 105 | 106 | # Sends message to all sinks attached to this stream. 107 | sink_stream << f'Sending message to Kafka sink in run mode {run_mode}.' 108 | 109 | # Give a grace period to the message to propagate then disconnect source 110 | # and sink. 111 | time.sleep(30) 112 | source_stream.disconnect_all() 113 | sink_stream.disconnect_all() 114 | -------------------------------------------------------------------------------- /examples/kafka/kafka_source_sink.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | import time 21 | 22 | # An artificial example of using Kafka sources and sink. 23 | # Typically the user application will interact with an external Kafka 24 | # service to either subscribe or publish data to other services: 25 | # 26 | # EXT. SERVICE => KAFKA => RAYVENS KAFKA SOURCE 27 | # or 28 | # RAYVENS KAFKA SINK => KAFKA => EXT. SERVICE 29 | # 30 | # In this example we put together an artificial example where, to 31 | # demonstrate both Kafka sources and sinks at the same time we 32 | # set a Kafka sink to publish to a test topic then have a Kafka 33 | # source read from that test topic: 34 | # 35 | # RAYVENS KAFKA SINK => KAFKA => RAYVENS KAFKA SOURCE 36 | # 37 | 38 | # Command line arguments and validation: 39 | if len(sys.argv) < 2: 40 | print(f'usage: {sys.argv[0]} or' 41 | f' {sys.argv[0]} ') 42 | sys.exit(1) 43 | 44 | # Brokers and run mode: 45 | brokers = None 46 | password = None 47 | run_mode = sys.argv[1] 48 | if len(sys.argv) == 4: 49 | brokers = sys.argv[1] 50 | password = sys.argv[2] 51 | run_mode = sys.argv[3] 52 | 53 | if run_mode not in ['local', 'mixed', 'operator']: 54 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 55 | 56 | # The Kafka topic used for communication. 57 | topic = "externalTopic" 58 | 59 | # If using the Kafka broker started by Rayvens the following brokers 60 | # are possible: 61 | # - from inside the cluster: kafka:9092 62 | # - from outside the cluster: localhost:31093 63 | # If using a different Kafka service please provide the brokers in the 64 | # form of host:port,host1:port1, ... . 65 | if brokers is None: 66 | brokers = 'localhost:31093' 67 | if run_mode == 'operator': 68 | brokers = "kafka:9092" 69 | 70 | # Initialize ray either on the cluster or locally otherwise. 71 | if run_mode == 'operator': 72 | ray.init(address='auto') 73 | else: 74 | ray.init() 75 | 76 | # Start rayvens in operator mode. 77 | rayvens.init(mode=run_mode) 78 | 79 | # Create source stream and configuration. 80 | source_stream = rayvens.Stream('kafka-source-stream') 81 | 82 | source_config = dict(kind='kafka-source', 83 | route='/fromkafka', 84 | topic=topic, 85 | brokers=brokers) 86 | if password is not None: 87 | source_config['SASL_password'] = password 88 | source = source_stream.add_source(source_config) 89 | # Log all events from stream-attached sources. 90 | source_stream >> (lambda event: print('KAFKA SOURCE:', event)) 91 | 92 | # Create sink stream and configuration. 93 | sink_stream = rayvens.Stream('kafka-sink-stream') 94 | sink_config = dict(kind='kafka-sink', 95 | route='/tokafka', 96 | topic=topic, 97 | brokers=brokers) 98 | if password is not None: 99 | sink_config['SASL_password'] = password 100 | sink = sink_stream.add_sink(sink_config) 101 | 102 | time.sleep(10) 103 | 104 | # Sends message to all sinks attached to this stream. 105 | sink_stream << f'Sending message to Kafka sink in run mode {run_mode}.' 106 | 107 | # Give a grace period to the message to propagate then disconnect source 108 | # and sink. 109 | time.sleep(30) 110 | source_stream.disconnect_all() 111 | sink_stream.disconnect_all() 112 | -------------------------------------------------------------------------------- /examples/multitask/multitask.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | 21 | # Send message to Slack sink using a multi-tasking operator. The operator 22 | # will be a Ray Task which spawns three other Ray Tasks. Each of the three 23 | # spawned tasks will send their event to the sinks attached to the stream. 24 | 25 | # Command line arguments and validation: 26 | if len(sys.argv) < 4: 27 | print(f'usage: {sys.argv[0]} ') 28 | sys.exit(1) 29 | slack_channel = sys.argv[1] 30 | slack_webhook = sys.argv[2] 31 | run_mode = sys.argv[3] 32 | if run_mode not in ['local', 'mixed', 'operator']: 33 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 34 | 35 | # Initialize ray either on the cluster or locally otherwise. 36 | if run_mode == 'operator': 37 | ray.init(address='auto') 38 | else: 39 | ray.init() 40 | 41 | # Start rayvens in operator mode. 42 | rayvens.init(mode=run_mode) 43 | 44 | # Create stream. 45 | stream = rayvens.Stream('slack') 46 | 47 | 48 | # Operator sub-task: 49 | @ray.remote 50 | def sub_task(context, intermediate_data): 51 | sub_task_outgoing_event = "sub-task " + intermediate_data 52 | print(sub_task_outgoing_event) 53 | context.publish(sub_task_outgoing_event) 54 | 55 | 56 | # Operator task: 57 | @ray.remote 58 | def multi_part_task(context, incoming_event): 59 | print("multi-part-task:", incoming_event) 60 | for i in range(3): 61 | sub_task.remote(context, "sub-event" + str(i)) 62 | 63 | 64 | # Event sink config. 65 | sink_config = dict(kind='slack-sink', 66 | route='/toslack', 67 | channel=slack_channel, 68 | webhook_url=slack_webhook) 69 | 70 | # Add sink to stream. 71 | sink = stream.add_sink(sink_config) 72 | 73 | # Add multi-task operator to stream. 74 | stream.add_multitask_operator(multi_part_task) 75 | 76 | # Sends message to all sinks attached to this stream. 77 | stream << f'Sending message to Slack sink in run mode {run_mode}.' 78 | 79 | # Disconnect any sources or sinks attached to the stream 2 seconds after 80 | # the stream is idle (i.e. no events were propagated by the stream). 81 | stream.disconnect_all(after_idle_for=2) 82 | -------------------------------------------------------------------------------- /examples/slack/slack.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import json 18 | import ray 19 | import rayvens 20 | import sys 21 | import time 22 | 23 | # This example demonstrates how to use Camel 24 | # to receive and emit external events. 25 | # 26 | # fetch AAPL quotes every 3 seconds 27 | # analyze trend (up/down/stable) 28 | # publish trend to slack 29 | # 30 | # http-source -> comparator actor -> slack-sink 31 | 32 | # process command line arguments 33 | if len(sys.argv) < 3: 34 | print(f'usage: {sys.argv[0]} ') 35 | sys.exit(1) 36 | slack_channel = sys.argv[1] 37 | slack_webhook = sys.argv[2] 38 | 39 | # initialize ray 40 | ray.init() 41 | 42 | # initialize rayvens 43 | rayvens.init() 44 | 45 | # create a source stream 46 | source_config = dict( 47 | kind='http-source', 48 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 49 | period=3000) 50 | source = rayvens.Stream('http', source_config=source_config) 51 | 52 | # log incoming events 53 | source >> (lambda event: print('LOG:', event)) 54 | 55 | # create a sink stream 56 | sink_config = dict(kind='slack-sink', 57 | channel=slack_channel, 58 | webhook_url=slack_webhook) 59 | sink = rayvens.Stream('slack', sink_config=sink_config) 60 | 61 | 62 | # actor to compare APPL quote with last quote 63 | @ray.remote 64 | class Comparator: 65 | def __init__(self): 66 | self.last_quote = None 67 | 68 | def append(self, event): 69 | payload = json.loads(event) # parse event string to json 70 | quote = payload['quoteResponse']['result'][0]['regularMarketPrice'] 71 | try: 72 | if self.last_quote: 73 | if quote > self.last_quote: 74 | return 'AAPL is up' 75 | elif quote < self.last_quote: 76 | return 'AAPL is down' 77 | else: 78 | return 'AAPL is stable' 79 | finally: 80 | self.last_quote = quote 81 | 82 | 83 | # create an stream operator 84 | comparator = Comparator.remote() 85 | operator = rayvens.Stream('comparator', operator=comparator) 86 | 87 | # connect source to operator to sink 88 | source >> operator >> sink 89 | 90 | # run for a while 91 | time.sleep(120) 92 | -------------------------------------------------------------------------------- /examples/slack/slack_operator_mode.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import sys 20 | 21 | # Send message to Slack sink using the kamel anywhere operator implementation. 22 | 23 | # Command line arguments and validation: 24 | if len(sys.argv) < 4: 25 | print(f'usage: {sys.argv[0]} ') 26 | sys.exit(1) 27 | slack_channel = sys.argv[1] 28 | slack_webhook = sys.argv[2] 29 | run_mode = sys.argv[3] 30 | if run_mode not in ['local', 'mixed', 'operator']: 31 | raise RuntimeError(f'Invalid run mode provided: {run_mode}') 32 | 33 | # Initialize ray either on the cluster or locally otherwise. 34 | if run_mode == 'operator': 35 | ray.init(address='auto') 36 | else: 37 | ray.init() 38 | 39 | # Start rayvens in operator mode. 40 | rayvens.init(mode=run_mode) 41 | 42 | # Create stream. 43 | stream = rayvens.Stream('slack') 44 | 45 | # Event sink config. 46 | sink_config = dict(kind='slack-sink', 47 | route='/toslack', 48 | channel=slack_channel, 49 | webhook_url=slack_webhook) 50 | 51 | # Add sink to stream. 52 | sink = stream.add_sink(sink_config) 53 | 54 | # Sends message to all sinks attached to this stream. 55 | stream << f'Sending message to Slack sink in run mode {run_mode}.' 56 | 57 | # Disconnect any sources or sinks attached to the stream 2 seconds after 58 | # the stream is idle (i.e. no events were propagated by the stream). 59 | stream.disconnect_all(after_idle_for=2) 60 | -------------------------------------------------------------------------------- /examples/source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import json 18 | import ray 19 | import rayvens 20 | import time 21 | 22 | # This example demonstrates how to subscribe to a Camel event source 23 | # and process incoming events using a Ray actor. 24 | 25 | # initialize ray 26 | ray.init() 27 | 28 | # initialize rayvens 29 | rayvens.init() 30 | 31 | # create a source stream 32 | source_config = dict( 33 | kind='http-source', 34 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 35 | period=3000) 36 | source = rayvens.Stream('http', source_config=source_config) 37 | 38 | # log incoming events 39 | source >> (lambda event: print('LOG:', event)) 40 | 41 | 42 | # actor to compare APPL quote with last quote 43 | @ray.remote 44 | class Comparator: 45 | def __init__(self): 46 | self.last_quote = None 47 | 48 | def append(self, event): 49 | payload = json.loads(event) # parse event string to json 50 | quote = payload['quoteResponse']['result'][0]['regularMarketPrice'] 51 | try: 52 | if self.last_quote: 53 | if quote > self.last_quote: 54 | print('AAPL is up') 55 | elif quote < self.last_quote: 56 | print('AAPL is down') 57 | else: 58 | print('AAPL is stable') 59 | finally: 60 | self.last_quote = quote 61 | 62 | 63 | # comparator instance 64 | comparator = Comparator.remote() 65 | 66 | # subscribe comparator to source 67 | source >> comparator 68 | 69 | # run for a while 70 | time.sleep(120) 71 | -------------------------------------------------------------------------------- /examples/stream.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | 20 | # This example demonstrates how to create a stream of events, 21 | # subscribe to this stream, and append events to the stream. 22 | 23 | # initialize ray 24 | ray.init() 25 | 26 | # initialize rayvens 27 | rayvens.init() 28 | 29 | # create a stream 30 | stream = rayvens.Stream('example') 31 | 32 | # log all future events 33 | stream >> (lambda event: print('LOG:', event)) 34 | 35 | # append two events to the stream in order 36 | stream << 'hello' << 'world' 37 | -------------------------------------------------------------------------------- /examples/task.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | 20 | # This example demonstrates the use of Ray tasks to handle events. 21 | # These tasks are invoked in arbitrary order. The processing order 22 | # is therefore non-deterministic. 23 | 24 | # initialize ray 25 | ray.init() 26 | 27 | # initialize rayvens 28 | rayvens.init() 29 | 30 | # create a stream 31 | stream = rayvens.Stream('example') 32 | 33 | 34 | # define a first event handling task 35 | @ray.remote 36 | def handler1(event): 37 | print('handler1 received', event) 38 | 39 | 40 | # define a second event handling task 41 | @ray.remote 42 | def handler2(event): 43 | print('handler2 received', event) 44 | 45 | 46 | # subscribe tasks to stream 47 | stream >> handler1 48 | stream >> handler2 49 | 50 | # publish a few events 51 | for i in range(10): 52 | stream << f'event {i}' 53 | -------------------------------------------------------------------------------- /rayvens/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from rayvens.api import init, meta, Stream 18 | from rayvens.core.common import OutputEvent 19 | 20 | __all__ = ['init', 'meta', 'Stream', 'OutputEvent'] 21 | -------------------------------------------------------------------------------- /rayvens/cli/README.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # Rayvens CLI 18 | 19 | Rayvens offers a command line interface for creating, launching and deleting integrations that receive or send events to external event sources and sinks respectively. 20 | 21 | Requirements: Python 3.8+, Docker and Kubernetes. 22 | 23 | ## Integrations 24 | 25 | Integrations are processes which interact with event sources and event sinks on behalf of the user. Integrations offer a uniform way to send/receive events between a user application and a diverse set of event sources and sinks. 26 | 27 | The Rayvens CLI uses the `quay.io/rayvens` image registry to store intermediate images. Pre-built images exist of commonly used integrations such as `slack-sink`. 28 | 29 | ### Running a pre-built image locally: 30 | 31 | To run the pre-built Slack sink integration run: 32 | 33 | ``` 34 | rayvens run --image quay.io/rayvens/slack-sink -p channel=<...> webhook_url=<...> 35 | ``` 36 | 37 | This will run the slack sink integration on the local machine as a Docker container. To view the created container perform a `docker ps`. 38 | 39 | When the integration is ready to receive events, the command will output the endpoint on which it expects to receive events. For example: 40 | 41 | ``` 42 | > rayvens run --image quay.io/rayvens/slack-sink -p channel=<...> webhook_url=<...> 43 | http://localhost:53844/my-integration-route 44 | ``` 45 | 46 | To remove the integration run: 47 | 48 | ``` 49 | rayvens delete --name slack-sink 50 | ``` 51 | 52 | This will stop, kill and remove the docker container from the list of existing containers. 53 | 54 | ### Removing an integration with a custom name: 55 | 56 | By default the name of the integration being run is the integration type which is also the image name i.e. `slack-sink`. The name of the application can be set by the user using the `--name` flag: 57 | 58 | ``` 59 | rayvens run --image quay.io/rayvens/slack-sink -p channel=<...> webhook_url=<...> --name my-integration 60 | ``` 61 | 62 | Deleting this integration will now become: 63 | 64 | ``` 65 | rayvens delete --name my-integration 66 | ``` 67 | 68 | The command will delete all the docker containers with a name starting with the string `my-container`. 69 | 70 | ### Running a pre-built image in a Kubernetes cluster: 71 | To run the pre-built Slack sink integration in a Kubernetes cluster add the `--deploy` flag to the previous command line: 72 | 73 | ``` 74 | rayvens run --image quay.io/rayvens/slack-sink -p channel=<...> webhook_url=<...> --deploy 75 | ``` 76 | 77 | To remove a Kubernetes-deployed integration with a default name run: 78 | 79 | ``` 80 | rayvens delete --name slack-sink --deployed 81 | ``` 82 | 83 | This command will remove any Kubernetes deployments and services. 84 | -------------------------------------------------------------------------------- /rayvens/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /rayvens/cli/delete.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from rayvens.cli import utils 18 | import rayvens.cli.kubernetes as kube 19 | import rayvens.cli.docker as docker 20 | 21 | 22 | def delete(args): 23 | # Set verbosity: 24 | utils.verbose = args.verbose 25 | 26 | if args.deployed: 27 | if args.name is not None: 28 | delete_deployment(args) 29 | else: 30 | if args.name is not None: 31 | delete_local_deployment(args) 32 | 33 | if args.all_jobs is not None: 34 | delete_all_jobs(args) 35 | 36 | 37 | def delete_all_jobs(args): 38 | namespace = "default" 39 | if args.namespace is not None: 40 | namespace = args.namespace 41 | 42 | prefix = args.all_jobs 43 | 44 | # Delete the jobs that start with the provided string. 45 | from kubernetes import client, config 46 | config.load_kube_config() 47 | k8s_client = client.BatchV1Api(client.ApiClient()) 48 | 49 | exception_occurred = False 50 | try: 51 | api_response = k8s_client.list_namespaced_job(namespace) 52 | body = client.V1DeleteOptions(propagation_policy='Background') 53 | for job in api_response.items: 54 | if job.metadata.name.startswith(prefix): 55 | k8s_client.delete_namespaced_job(job.metadata.name, 56 | namespace, 57 | body=body) 58 | except client.exceptions.ApiException: 59 | exception_occurred = True 60 | 61 | # Delete the jobs that start with the provided string. 62 | k8s_client = client.CoreV1Api(client.ApiClient()) 63 | 64 | if not exception_occurred: 65 | print(f"Successfully deleted jobs starting with {prefix} from " 66 | f"{namespace} namespace.") 67 | 68 | 69 | def delete_deployment(args, with_job_launcher_privileges=True): 70 | if args.name is None: 71 | raise RuntimeError("No integration name provided.") 72 | 73 | namespace = "default" 74 | if args.namespace is not None: 75 | namespace = args.namespace 76 | 77 | # Delete kubernetes deployment for integration. 78 | from kubernetes import client, config 79 | config.load_kube_config() 80 | k8s_client = client.ApiClient() 81 | 82 | # Delete the integration deployment. 83 | deployment_name = utils.get_kubernetes_integration_name(args.name) 84 | deployment = kube.Deployment(deployment_name, namespace=namespace) 85 | deployment.delete(k8s_client, starting_with=deployment_name) 86 | 87 | # Delete the entrypoint service. 88 | entrypoint_service_name = utils.get_kubernetes_entrypoint_name(args.name) 89 | service = kube.Service(entrypoint_service_name, namespace=namespace) 90 | service.delete(k8s_client, starting_with=entrypoint_service_name) 91 | 92 | # Delete configMap for updating the integration file. 93 | config_map = kube.ConfigMap(namespace=namespace) 94 | config_map.delete(k8s_client, kube.volume_base_name) 95 | 96 | if with_job_launcher_privileges: 97 | # Delete service account: 98 | # job-launcher-service-account 99 | name = utils.job_launcher_service_account 100 | service_account = kube.ServiceAccount(name, namespace=namespace) 101 | service_account.delete(k8s_client) 102 | 103 | # Delete cluster role binding: 104 | # job-launcher-service-account 105 | name = utils.job_launcher_cluster_role_binding 106 | cluster_role_binding = kube.ClusterRoleBinding(name, [], None) 107 | cluster_role_binding.delete(k8s_client) 108 | 109 | # Delete cluster role: 110 | # job-manager-role 111 | name = utils.job_manager_role 112 | cluster_role_binding = kube.ClusterRole(name, namespace=namespace) 113 | cluster_role_binding.delete(k8s_client) 114 | 115 | 116 | def delete_local_deployment(args): 117 | if args.name is None: 118 | raise RuntimeError("No integration name provided.") 119 | 120 | containers = docker.docker_container_ls() 121 | 122 | for container_name in containers: 123 | if container_name.startswith(args.name): 124 | docker.docker_kill(containers[container_name]) 125 | docker.docker_rm(containers[container_name]) 126 | -------------------------------------------------------------------------------- /rayvens/cli/examples/build-base-image.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Example of building the base image used by all integration types. 18 | rayvens base --registry quay.io/rayvens 19 | -------------------------------------------------------------------------------- /rayvens/cli/examples/build-slack-sink-image.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Example of building the Slack sink integration image. This image preloads 18 | # all the dependencies required for running the integration. This step should 19 | # be done once, and then the image reused. 20 | rayvens build --kind slack-sink --registry quay.io/rayvens 21 | -------------------------------------------------------------------------------- /rayvens/cli/examples/delete-slack-sink-local.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Example of deleting a slack sink integration locally. 18 | # Add -v to run this command in verbose mode. 19 | rayvens delete --name slack-sink 20 | -------------------------------------------------------------------------------- /rayvens/cli/examples/run-slack-sink-local.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Example of running a slack sink integration locally. 18 | # Add -v to run this command in verbose mode. 19 | rayvens run --image quay.io/rayvens/slack-sink -p channel=$SLACK_CHANNEL webhook_url=$SLACK_WEBHOOK 20 | -------------------------------------------------------------------------------- /rayvens/cli/kamel.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import subprocess 18 | import rayvens.cli.utils as utils 19 | from rayvens.cli.utils import PRINT 20 | 21 | kamel_tag = "kamel" 22 | 23 | 24 | def kamel_local_build_base_image(args): 25 | command = ["kamel"] 26 | 27 | # Local command: 28 | command.append("local") 29 | 30 | # Build command: 31 | command.append("build") 32 | 33 | # Base image options: 34 | command.append("--base-image") 35 | 36 | # Registry: 37 | registry = utils.get_registry(args) 38 | command.append("--container-registry") 39 | command.append(registry) 40 | 41 | # Wait for docker command to finish before returning: 42 | outcome = subprocess.run(command) 43 | 44 | image_name = utils.get_base_image_name(args) 45 | 46 | if outcome.returncode == 0: 47 | PRINT(f"Base image {image_name} pushed successfully.", tag=kamel_tag) 48 | else: 49 | PRINT(f"Base image {image_name} push failed.", tag=kamel_tag) 50 | 51 | 52 | def kamel_local_build_image(args, integration_file_path): 53 | command = ["kamel"] 54 | 55 | # Local command: 56 | command.append("local") 57 | 58 | # Build command: 59 | command.append("build") 60 | 61 | # Image: 62 | integration_image = utils.get_integration_image(args) 63 | command.append("--image") 64 | command.append(integration_image) 65 | 66 | # Add integration file: 67 | command.append(integration_file_path) 68 | 69 | # Wait for docker command to finish before returning: 70 | outcome = subprocess.run(command) 71 | 72 | if outcome.returncode == 0: 73 | PRINT(f"Base image {integration_image} pushed successfully.", 74 | tag=kamel_tag) 75 | else: 76 | PRINT(f"Base image {integration_image} push failed.", tag=kamel_tag) 77 | -------------------------------------------------------------------------------- /rayvens/cli/rayvens_print.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from rayvens.core.catalog_utils import print_predefined_integrations 18 | from rayvens.core.catalog_utils import print_requirements_summary 19 | 20 | 21 | def rayvens_print(args): 22 | if args.all: 23 | print_predefined_integrations() 24 | if args.kind: 25 | print_requirements_summary(args.kind) 26 | -------------------------------------------------------------------------------- /rayvens/cli/rayvens_setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | 18 | def rayvens_setup(args): 19 | print(args) 20 | -------------------------------------------------------------------------------- /rayvens/cli/run.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import yaml 18 | import rayvens.cli.utils as utils 19 | import rayvens.cli.file as file 20 | import rayvens.cli.docker as docker 21 | import rayvens.cli.kubernetes as kube 22 | import rayvens.core.utils as rayvens_utils 23 | from rayvens.core.catalog import sources, sinks 24 | from rayvens.core.catalog import construct_source, construct_sink 25 | from rayvens.cli.docker import docker_run_integration 26 | run_tag = "run" 27 | 28 | 29 | def run_integration(args): 30 | # Form full image name: 31 | if args.image is None: 32 | raise RuntimeError("Missing image name") 33 | image = utils.get_integration_image(args) 34 | 35 | # Set verbosity: 36 | utils.verbose = args.verbose 37 | 38 | # Get a free port: 39 | docker.free_port = rayvens_utils.random_port() 40 | 41 | # Create a work directory in the current directory: 42 | workspace_directory = file.Directory("workspace") 43 | 44 | # Fetch summary file from integration image. 45 | docker.add_summary_from_image(image, workspace_directory) 46 | 47 | # Retrieve a reference to the summary file. 48 | summary_file = workspace_directory.get_file(file.summary_file_name) 49 | 50 | # Get integration kind from summary file: 51 | kind = summary_file.kind 52 | 53 | # Check if a valid launch image name has been passed: 54 | with_job_launcher = summary_file.launch_image != "None" 55 | 56 | # The name of the integration: 57 | name = kind 58 | if args.name is not None: 59 | name = args.name 60 | 61 | # Check if the source/sink is predefined. 62 | predefined_integration = kind in sources or kind in sinks 63 | 64 | # By default the HTTP transport is used. This is the only supported 65 | # transport for now. 66 | inverted_transport = True 67 | 68 | is_sink = False 69 | if predefined_integration: 70 | # Extract predefined integration kind: 71 | full_config = utils.get_full_config(summary_file, args) 72 | 73 | # Create the integration yaml specification. 74 | route = "/" + name + "-route" 75 | if kind in sources: 76 | spec = construct_source(full_config, 77 | f'platform-http:{route}', 78 | inverted=inverted_transport) 79 | else: 80 | spec = construct_sink(full_config, f'platform-http:{route}') 81 | is_sink = True 82 | 83 | # Write the specification to the file. 84 | modeline_options = utils.get_modeline_config(args, summary_file) 85 | integration_source_file = modeline_options + "\n\n" + yaml.dump(spec) 86 | integration_file = file.File( 87 | utils.get_kubernetes_integration_file_name(name), 88 | contents=integration_source_file) 89 | else: 90 | raise RuntimeError("Not implemented yet") 91 | 92 | # Fetch the variables specified as environment variables. 93 | envvars = utils.get_modeline_envvars(summary_file, args) 94 | 95 | server_address = None 96 | integration_file.emit() 97 | if args.deploy is not None and args.deploy: 98 | # Set the namespace: 99 | namespace = "default" 100 | if args.namespace is not None: 101 | namespace = args.namespace 102 | 103 | # Update integration file on image via configMap: 104 | integration_config_map = kube.ConfigMap(integration_file, 105 | namespace=namespace) 106 | 107 | # Prepare Kubernetes API: 108 | from kubernetes import client, config 109 | config.load_kube_config() 110 | 111 | # Kubernetes client: 112 | k8s_client = client.ApiClient() 113 | 114 | # Create configMap that updates the integration file. 115 | integration_config_map.create(k8s_client) 116 | 117 | # Deploy integration in Kubernetes: 118 | deployment = kube.get_deployment(name, namespace, 119 | utils.get_registry(args), args, 120 | with_job_launcher, 121 | integration_config_map, k8s_client) 122 | 123 | deployment.create(k8s_client) 124 | else: 125 | # Output endpoint for sink: 126 | server_address = f"http://localhost:{docker.free_port}" 127 | 128 | # Run final integration image: 129 | # docker run \ 130 | # -v integration_file_path:/workspace/ \ 131 | # --env ENV_VAR=$ENV_VAR -p :8080 \ 132 | # 133 | docker_run_integration(image, 134 | integration_file.full_path, 135 | integration_file.name, 136 | name, 137 | envvars=envvars, 138 | is_sink=is_sink, 139 | server_address=server_address) 140 | 141 | integration_file.delete() 142 | 143 | if is_sink and server_address is not None: 144 | print(f"{server_address}{route}") 145 | -------------------------------------------------------------------------------- /rayvens/cli/version.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | kamel_version = "1.6.1" 18 | -------------------------------------------------------------------------------- /rayvens/core/FileQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright IBM Corporation 2021 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.util.concurrent.BlockingQueue; 18 | import java.util.concurrent.LinkedBlockingQueue; 19 | import java.io.File; 20 | import java.io.InputStream; 21 | import java.io.FileInputStream; 22 | 23 | import org.apache.camel.BindToRegistry; 24 | import org.apache.camel.Exchange; 25 | import org.apache.camel.Processor; 26 | import org.apache.camel.builder.RouteBuilder; 27 | 28 | class Recv implements Processor { 29 | BlockingQueue queue; 30 | 31 | public Recv(BlockingQueue queue) { 32 | this.queue = queue; 33 | } 34 | 35 | public void process(Exchange exchange) throws Exception { 36 | File file = exchange.getIn().getBody(File.class); 37 | long fileSize = file.length(); 38 | 39 | // TODO: find a better way to read the file content. 40 | byte[] allBytes = new byte[(int) fileSize]; 41 | InputStream inputStream = new FileInputStream(file); 42 | inputStream.read(allBytes); 43 | inputStream.close(); 44 | queue.add(allBytes); 45 | } 46 | } 47 | 48 | class Send implements Processor { 49 | BlockingQueue queue; 50 | 51 | public Send(BlockingQueue queue) { 52 | this.queue = queue; 53 | } 54 | 55 | public void process(Exchange exchange) throws Exception { 56 | Object body = queue.take(); 57 | exchange.getIn().setBody(body); 58 | } 59 | } 60 | 61 | public class FileQueue extends RouteBuilder { 62 | BlockingQueue queue = new LinkedBlockingQueue(); 63 | 64 | @BindToRegistry 65 | public Recv addToFileQueue() { 66 | return new Recv(queue); 67 | } 68 | 69 | @BindToRegistry 70 | public Send takeFromFileQueue() { 71 | return new Send(queue); 72 | } 73 | 74 | @Override 75 | public void configure() throws Exception { 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /rayvens/core/FileQueueJson.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright IBM Corporation 2021 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.util.concurrent.BlockingQueue; 18 | import java.util.concurrent.LinkedBlockingQueue; 19 | import java.io.File; 20 | 21 | import org.apache.camel.BindToRegistry; 22 | import org.apache.camel.Exchange; 23 | import org.apache.camel.Processor; 24 | import org.apache.camel.builder.RouteBuilder; 25 | import org.json.simple.JSONObject; 26 | 27 | class Recv implements Processor { 28 | BlockingQueue queue; 29 | 30 | public Recv(BlockingQueue queue) { 31 | this.queue = queue; 32 | } 33 | 34 | public void process(Exchange exchange) throws Exception { 35 | JSONObject returnJsonObject = new JSONObject(); 36 | String body = exchange.getIn().getBody(String.class); 37 | returnJsonObject.put("body", body); 38 | 39 | Object key = exchange.getIn().getHeader("CamelAwsS3Key"); 40 | returnJsonObject.put("filename", key.toString()); 41 | queue.add(returnJsonObject.toString()); 42 | } 43 | } 44 | 45 | class Send implements Processor { 46 | BlockingQueue queue; 47 | 48 | public Send(BlockingQueue queue) { 49 | this.queue = queue; 50 | } 51 | 52 | public void process(Exchange exchange) throws Exception { 53 | Object body = queue.take(); 54 | exchange.getIn().setBody(body); 55 | } 56 | } 57 | 58 | public class FileQueueJson extends RouteBuilder { 59 | BlockingQueue queue = new LinkedBlockingQueue(); 60 | 61 | @BindToRegistry 62 | public Recv addToFileJsonQueue() { 63 | return new Recv(queue); 64 | } 65 | 66 | @BindToRegistry 67 | public Send takeFromFileJsonQueue() { 68 | return new Send(queue); 69 | } 70 | 71 | @Override 72 | public void configure() throws Exception { 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /rayvens/core/FileQueueName.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright IBM Corporation 2021 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.util.concurrent.BlockingQueue; 18 | import java.util.concurrent.LinkedBlockingQueue; 19 | import java.io.File; 20 | import java.io.InputStream; 21 | import java.io.FileInputStream; 22 | 23 | import org.apache.camel.BindToRegistry; 24 | import org.apache.camel.Exchange; 25 | import org.apache.camel.Processor; 26 | import org.apache.camel.builder.RouteBuilder; 27 | 28 | import org.json.simple.JSONObject; 29 | 30 | class Recv implements Processor { 31 | BlockingQueue queue; 32 | 33 | public Recv(BlockingQueue queue) { 34 | this.queue = queue; 35 | } 36 | 37 | public void process(Exchange exchange) throws Exception { 38 | JSONObject returnJsonObject = new JSONObject(); 39 | String body = exchange.getIn().getBody(String.class); 40 | returnJsonObject.put("body", body); 41 | 42 | Object key = exchange.getIn().getHeader("CamelFileName"); 43 | returnJsonObject.put("filename", key.toString()); 44 | queue.add(returnJsonObject.toString()); 45 | } 46 | } 47 | 48 | class Send implements Processor { 49 | BlockingQueue queue; 50 | 51 | public Send(BlockingQueue queue) { 52 | this.queue = queue; 53 | } 54 | 55 | public void process(Exchange exchange) throws Exception { 56 | Object body = queue.take(); 57 | exchange.getIn().setBody(body); 58 | } 59 | } 60 | 61 | public class FileQueueName extends RouteBuilder { 62 | BlockingQueue queue = new LinkedBlockingQueue(); 63 | 64 | @BindToRegistry 65 | public Recv addToFileQueueName() { 66 | return new Recv(queue); 67 | } 68 | 69 | @BindToRegistry 70 | public Send takeFromFileQueueName() { 71 | return new Send(queue); 72 | } 73 | 74 | @Override 75 | public void configure() throws Exception { 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /rayvens/core/FileWatchQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright IBM Corporation 2021 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.util.concurrent.BlockingQueue; 18 | import java.util.concurrent.LinkedBlockingQueue; 19 | import java.io.File; 20 | import java.io.InputStream; 21 | import java.io.FileInputStream; 22 | 23 | import org.apache.camel.BindToRegistry; 24 | import org.apache.camel.Exchange; 25 | import org.apache.camel.Processor; 26 | import org.apache.camel.builder.RouteBuilder; 27 | import org.json.simple.JSONObject; 28 | 29 | class Recv implements Processor { 30 | BlockingQueue queue; 31 | 32 | public Recv(BlockingQueue queue) { 33 | this.queue = queue; 34 | } 35 | 36 | public void process(Exchange exchange) throws Exception { 37 | JSONObject returnJsonObject = new JSONObject(); 38 | 39 | // Record event type: 40 | Object eventType = exchange.getIn().getHeader("CamelFileEventType"); 41 | returnJsonObject.put("event_type", eventType.toString()); 42 | 43 | // Record event type: 44 | File file = exchange.getIn().getBody(File.class); 45 | returnJsonObject.put("filename", file.toString()); 46 | queue.add(returnJsonObject.toString()); 47 | } 48 | } 49 | 50 | class Send implements Processor { 51 | BlockingQueue queue; 52 | 53 | public Send(BlockingQueue queue) { 54 | this.queue = queue; 55 | } 56 | 57 | public void process(Exchange exchange) throws Exception { 58 | Object body = queue.take(); 59 | exchange.getIn().setBody(body); 60 | } 61 | } 62 | 63 | public class FileWatchQueue extends RouteBuilder { 64 | BlockingQueue queue = new LinkedBlockingQueue(); 65 | 66 | @BindToRegistry 67 | public Recv addToFileWatchQueue() { 68 | return new Recv(queue); 69 | } 70 | 71 | @BindToRegistry 72 | public Send takeFromFileWatchQueue() { 73 | return new Send(queue); 74 | } 75 | 76 | @Override 77 | public void configure() throws Exception { 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /rayvens/core/MetaEventQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright IBM Corporation 2021 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.util.concurrent.BlockingQueue; 18 | import java.util.concurrent.LinkedBlockingQueue; 19 | import java.io.File; 20 | import java.io.InputStream; 21 | import java.io.FileInputStream; 22 | 23 | import org.apache.camel.BindToRegistry; 24 | import org.apache.camel.Exchange; 25 | import org.apache.camel.Processor; 26 | import org.apache.camel.builder.RouteBuilder; 27 | import org.json.simple.JSONObject; 28 | 29 | class Recv implements Processor { 30 | BlockingQueue queue; 31 | 32 | public Recv(BlockingQueue queue) { 33 | this.queue = queue; 34 | } 35 | 36 | public void process(Exchange exchange) throws Exception { 37 | JSONObject returnJsonObject = new JSONObject(); 38 | Object key = exchange.getIn().getHeader("CamelAwsS3Key"); 39 | returnJsonObject.put("filename", key.toString()); 40 | queue.add(returnJsonObject.toString()); 41 | } 42 | } 43 | 44 | class Send implements Processor { 45 | BlockingQueue queue; 46 | 47 | public Send(BlockingQueue queue) { 48 | this.queue = queue; 49 | } 50 | 51 | public void process(Exchange exchange) throws Exception { 52 | Object body = queue.take(); 53 | exchange.getIn().setBody(body); 54 | } 55 | } 56 | 57 | public class MetaEventQueue extends RouteBuilder { 58 | BlockingQueue queue = new LinkedBlockingQueue(); 59 | 60 | @BindToRegistry 61 | public Recv addToMetaEventQueue() { 62 | return new Recv(queue); 63 | } 64 | 65 | @BindToRegistry 66 | public Send takeFromMetaEventQueue() { 67 | return new Send(queue); 68 | } 69 | 70 | @Override 71 | public void configure() throws Exception { 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /rayvens/core/ProcessFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright IBM Corporation 2021 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.nio.file.Path; 18 | import java.nio.file.Paths; 19 | import java.io.File; 20 | 21 | import org.apache.camel.BindToRegistry; 22 | import org.apache.camel.Exchange; 23 | import org.apache.camel.Processor; 24 | import org.apache.camel.builder.RouteBuilder; 25 | import org.apache.camel.component.aws2.s3.AWS2S3Constants; 26 | 27 | class FileProcessor implements Processor { 28 | public void process(Exchange exchange) throws Exception { 29 | // Input is a file and all we do is set the header entry to the file name. 30 | File file = exchange.getIn().getBody(File.class); 31 | Path path = Paths.get(file.toString()); 32 | exchange.getIn().setHeader(AWS2S3Constants.KEY, path.getFileName()); 33 | } 34 | } 35 | 36 | public class ProcessFile extends RouteBuilder { 37 | @BindToRegistry 38 | public FileProcessor processFile() { 39 | return new FileProcessor(); 40 | } 41 | 42 | @Override 43 | public void configure() throws Exception { 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /rayvens/core/ProcessPath.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright IBM Corporation 2021 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.nio.file.Path; 18 | import java.nio.file.Paths; 19 | 20 | import org.apache.camel.BindToRegistry; 21 | import org.apache.camel.Exchange; 22 | import org.apache.camel.Processor; 23 | import org.apache.camel.builder.RouteBuilder; 24 | import org.apache.camel.component.aws2.s3.AWS2S3Constants; 25 | 26 | class PathProcessor implements Processor { 27 | public void process(Exchange exchange) throws Exception { 28 | // Input is a string with the path to the file. 29 | Path path = Paths.get(exchange.getIn().getBody(String.class)); 30 | exchange.getIn().setHeader(AWS2S3Constants.KEY, path.getFileName()); 31 | exchange.getIn().setBody(path.toFile()); 32 | } 33 | } 34 | 35 | public class ProcessPath extends RouteBuilder { 36 | @BindToRegistry 37 | public PathProcessor processPath() { 38 | return new PathProcessor(); 39 | } 40 | 41 | @Override 42 | public void configure() throws Exception { 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /rayvens/core/Queue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright IBM Corporation 2021 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import java.util.concurrent.BlockingQueue; 18 | import java.util.concurrent.LinkedBlockingQueue; 19 | 20 | import org.apache.camel.BindToRegistry; 21 | import org.apache.camel.Exchange; 22 | import org.apache.camel.Processor; 23 | import org.apache.camel.builder.RouteBuilder; 24 | 25 | class Recv implements Processor { 26 | BlockingQueue queue; 27 | 28 | public Recv(BlockingQueue queue) { 29 | this.queue = queue; 30 | } 31 | 32 | public void process(Exchange exchange) throws Exception { 33 | Object body = exchange.getIn().getBody(); 34 | queue.add(body); 35 | } 36 | } 37 | 38 | class Send implements Processor { 39 | BlockingQueue queue; 40 | 41 | public Send(BlockingQueue queue) { 42 | this.queue = queue; 43 | } 44 | 45 | public void process(Exchange exchange) throws Exception { 46 | Object body = queue.take(); 47 | exchange.getIn().setBody(body); 48 | } 49 | } 50 | 51 | public class Queue extends RouteBuilder { 52 | BlockingQueue queue = new LinkedBlockingQueue(); 53 | 54 | @BindToRegistry 55 | public Recv addToQueue() { 56 | return new Recv(queue); 57 | } 58 | 59 | @BindToRegistry 60 | public Send takeFromQueue() { 61 | return new Send(queue); 62 | } 63 | 64 | @Override 65 | public void configure() throws Exception { 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /rayvens/core/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /rayvens/core/harness.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | import psutil 19 | import signal 20 | import subprocess 21 | import sys 22 | 23 | is_local = sys.argv[1] == "kamel" 24 | 25 | # Start child process. 26 | process = None 27 | if is_local: 28 | process = subprocess.Popen(sys.argv[1:], start_new_session=True) 29 | else: 30 | process = subprocess.Popen(sys.argv[3:], start_new_session=True) 31 | 32 | 33 | def clean_up(): 34 | if sys.platform == "win32": 35 | os.kill(process.pid, signal.CTRL_C_EVENT) 36 | else: 37 | os.killpg(os.getpgid(process.pid), signal.SIGKILL) 38 | 39 | # Delete integration file. 40 | os.remove(sys.argv[-1]) 41 | 42 | 43 | def sigterm_handler(*args): 44 | clean_up() 45 | sys.exit() 46 | 47 | 48 | signal.signal(signal.SIGTERM, sigterm_handler) 49 | 50 | # Wait for parent process. 51 | psutil.wait_procs([psutil.Process().parent()]) 52 | 53 | # Terminate child process. 54 | if is_local: 55 | clean_up() 56 | else: 57 | # Operator commands can only be terminated by running a kamel delete 58 | # command. 59 | command = ["kamel", "delete", sys.argv[1], "-n", sys.argv[2]] 60 | subprocess.Popen(command, start_new_session=True) 61 | -------------------------------------------------------------------------------- /rayvens/core/kafka.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import rayvens.core.catalog as catalog 18 | from rayvens.core.common import get_run_mode, brokers 19 | from rayvens.core.common import kafka_send_to, kafka_recv_from 20 | from rayvens.core.integration import Integration 21 | 22 | 23 | def start(camel_mode, check_port, release): 24 | return Camel(get_run_mode(camel_mode, check_port, release)) 25 | 26 | 27 | class Camel: 28 | def __init__(self, mode): 29 | self.mode = mode 30 | self.mode.transport = 'kafka' 31 | 32 | def add_source(self, stream, config, source_name): 33 | integration = Integration(stream.name, source_name, config) 34 | spec = catalog.construct_source( 35 | config, 36 | f'kafka:{integration.kafka_transport_topic}?brokers={brokers()}') 37 | integration.prepare_environment(self.mode) 38 | integration.invoke_local_run(self.mode, spec) 39 | integration.thread = kafka_send_to( 40 | integration.kafka_transport_topic, 41 | integration.kafka_transport_partitions, stream.actor) 42 | return integration 43 | 44 | def add_sink(self, stream, config, sink_name): 45 | integration = Integration(stream.name, sink_name, config) 46 | spec = catalog.construct_sink( 47 | config, 48 | f'kafka:{integration.kafka_transport_topic}?brokers={brokers()}') 49 | integration.prepare_environment(self.mode) 50 | integration.invoke_local_run(self.mode, spec) 51 | kafka_recv_from(sink_name, integration.kafka_transport_topic, 52 | stream.actor) 53 | return integration 54 | 55 | def disconnect(self, integration): 56 | integration.disconnect(self.mode) 57 | -------------------------------------------------------------------------------- /rayvens/core/kamel_backend.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import requests 19 | from ray import serve 20 | 21 | 22 | class SinkEvent: 23 | def __init__(self, route, integration_name): 24 | self.route = route 25 | self.integration_name = integration_name 26 | self.data = None 27 | 28 | def get_data(self): 29 | if self.data is None: 30 | raise RuntimeError( 31 | "Attempting to send event with None data field.") 32 | return self.data 33 | 34 | 35 | class KamelEventHandler: 36 | def __init__(self, mode, topic): 37 | self.mode = mode 38 | self.topic = topic 39 | 40 | async def __call__(self, request): 41 | body = await request.body() 42 | if isinstance(body, SinkEvent): 43 | endpoint = self.mode._get_server_address( 44 | body.integration_name) + body.route 45 | requests.post(endpoint, data=body.get_data()) 46 | return {"message": "Success"} 47 | 48 | if self.topic is None: 49 | return {"message": "Failure"} 50 | print("Body:", body) 51 | self.topic.append.remote(body) 52 | return {"message": "Success"} 53 | 54 | 55 | class KamelBackend: 56 | backendName = "kamel_backend" 57 | 58 | def __init__(self, mode, topic=None): 59 | # When the backend runs on a local machine we must allocate its 60 | # actor at least 1 CPU. 61 | # actor_options = {'num_cpus': 0} 62 | # if mode.is_local() or mode.is_mixed(): 63 | # actor_options = {'num_cpus': 1} 64 | serve.create_backend(self.backendName, KamelEventHandler, mode, topic) 65 | self.endpoint_to_event = {} 66 | 67 | def createProxyEndpoint(self, endpoint_name, route, integration_name): 68 | self.endpoint_to_event[endpoint_name] = SinkEvent( 69 | route, integration_name) 70 | 71 | # Create endpoint with method as POST. 72 | serve.create_endpoint(endpoint_name, 73 | backend=self.backendName, 74 | route=route, 75 | methods=["POST"]) 76 | 77 | def _post_event(self, endpointHandle, endpoint_name, data): 78 | # Get partial event.s 79 | event = self.endpoint_to_event[endpoint_name] 80 | 81 | # Populate data field. 82 | event.data = data 83 | 84 | # Send request to backend. 85 | return ray.get(endpointHandle.remote(event)) 86 | 87 | def postToProxyEndpoint(self, endpoint_name, data): 88 | return self._post_event(serve.get_handle(endpoint_name), endpoint_name, 89 | data) 90 | 91 | def postToProxyEndpointHandle(self, endpointHandle, endpoint_name, data): 92 | return self._post_event(endpointHandle, endpoint_name, data) 93 | 94 | def removeProxyEndpoint(self, endpoint_name): 95 | serve.delete_endpoint(endpoint_name) 96 | self.endpoint_to_event.pop(endpoint_name) 97 | 98 | 99 | # Method which send post request to external Camel-K sink. 100 | 101 | 102 | @ray.remote 103 | class SinkSubscriber(object): 104 | def __init__(self, route): 105 | self.route = route 106 | 107 | def sendToSink(self, data): 108 | requests.post("http://0.0.0.0:8080" + self.route, data=data) 109 | -------------------------------------------------------------------------------- /rayvens/core/kamel_utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from enum import Enum 18 | from rayvens.core import invocation 19 | 20 | 21 | class KamelCommand(Enum): 22 | INSTALL = 1 23 | BUILD = 2 24 | RUN = 3 25 | LOCAL_BUILD = 4 26 | LOCAL_RUN = 5 27 | DELETE = 6 28 | LOG = 7 29 | UNINSTALL = 100 30 | 31 | 32 | def kamel_command_type(command_options): 33 | command = ' '.join(command_options) 34 | if command.startswith("install"): 35 | return KamelCommand.INSTALL 36 | if command.startswith("build"): 37 | return KamelCommand.BUILD 38 | if command.startswith("run"): 39 | return KamelCommand.RUN 40 | if command.startswith("local build"): 41 | return KamelCommand.LOCAL_BUILD 42 | if command.startswith("local run"): 43 | return KamelCommand.LOCAL_RUN 44 | if command.startswith("uninstall"): 45 | return KamelCommand.UNINSTALL 46 | if command.startswith("delete"): 47 | return KamelCommand.DELETE 48 | if command.startswith("log"): 49 | return KamelCommand.LOG 50 | raise RuntimeError('unsupported kamel subcommand: %s' % command) 51 | 52 | 53 | def kamel_command_str(command_type): 54 | if command_type == KamelCommand.INSTALL: 55 | return "install" 56 | if command_type == KamelCommand.BUILD: 57 | return "build" 58 | if command_type == KamelCommand.RUN: 59 | return "run" 60 | if command_type == KamelCommand.LOCAL_BUILD: 61 | return "local build" 62 | if command_type == KamelCommand.LOCAL_RUN: 63 | return "local run" 64 | if command_type == KamelCommand.UNINSTALL: 65 | return "uninstall" 66 | if command_type == KamelCommand.DELETE: 67 | return "delete" 68 | if command_type == KamelCommand.LOG: 69 | return "log" 70 | raise RuntimeError('unsupported kamel subcommand') 71 | 72 | 73 | def kamel_command_end_condition(subcommand_type, base_name): 74 | if subcommand_type == KamelCommand.INSTALL: 75 | return "Camel K installed in namespace" 76 | if subcommand_type == KamelCommand.BUILD: 77 | return "" 78 | if subcommand_type == KamelCommand.RUN: 79 | return f"Integration \"{base_name}\"" 80 | if subcommand_type == KamelCommand.LOCAL_BUILD: 81 | return "" 82 | if subcommand_type == KamelCommand.LOCAL_RUN: 83 | return "Installed features:" 84 | if subcommand_type == KamelCommand.UNINSTALL: 85 | return "Camel K Service Accounts removed from namespace" 86 | if subcommand_type == KamelCommand.DELETE: 87 | return f"Integration {base_name} deleted" 88 | if subcommand_type == KamelCommand.LOG: 89 | return "" 90 | raise RuntimeError('unsupported kamel subcommand: %s' % 91 | kamel_command_str(subcommand_type)) 92 | 93 | 94 | # Helper for ongoing local commands like kamel local run. 95 | 96 | 97 | def invoke_kamel_command(command, 98 | mode, 99 | integration_name, 100 | integration_content=[], 101 | message=None): 102 | # Invoke command using the Kamel invocation actor. 103 | kamel_invocation = invocation.KamelInvocation( 104 | command, 105 | mode, 106 | integration_name=integration_name, 107 | integration_content=integration_content) 108 | 109 | # Wait for kamel command to finish launching the integration. 110 | if kamel_invocation.invoke(message): 111 | return kamel_invocation 112 | 113 | return None 114 | -------------------------------------------------------------------------------- /rayvens/core/kubernetes.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | from rayvens.core import utils 19 | from rayvens.core import kubernetes_utils 20 | 21 | # Port used by the Quarkus Runtime to listen to HTTP requests. 22 | quarkus_listener_port = "8080" 23 | 24 | # Wait for pod to reach running state. 25 | 26 | 27 | def pod_running_status(integration_name, namespace): 28 | # TODO: adapt this to support multiple namespaces. 29 | command = ["get", "pods", "-w"] 30 | 31 | # Namespace 32 | if namespace != "default": 33 | command.append("-n") 34 | command.append(namespace) 35 | 36 | return kubernetes_utils.pod_status(command, integration_name) 37 | 38 | 39 | # Wait for integration to reach running state. 40 | 41 | 42 | def integration_status(mode, pod_name, message=None): 43 | # TODO: adapt this to support multiple namespaces. 44 | command = ["logs", pod_name] 45 | 46 | # Namespace 47 | command.append("-n") 48 | command.append(mode.namespace) 49 | 50 | # Stream output from this command. 51 | command.append("--follow=true") 52 | 53 | return kubernetes_utils.invoke_kubectl_command(command, 54 | message=message, 55 | ongoing=True) 56 | 57 | 58 | # Create service that Ray can talk to from outside the cluster. 59 | 60 | 61 | def create_kamel_external_service(mode, service_name, integration_name): 62 | # Compose yaml file. 63 | yaml_file = f""" 64 | kind: Service 65 | apiVersion: v1 66 | metadata: 67 | name: {service_name} 68 | spec: 69 | ports: 70 | - nodePort: {utils.externalized_cluster_port} 71 | port: {quarkus_listener_port} 72 | protocol: TCP 73 | targetPort: {quarkus_listener_port} 74 | selector: 75 | camel.apache.org/integration: {integration_name} 76 | type: NodePort 77 | """ 78 | 79 | # Write to output yaml file. 80 | output_file_name = os.path.abspath(service_name + ".yaml") 81 | output_file = open(output_file_name, "w") 82 | output_file.write(yaml_file) 83 | output_file.close() 84 | 85 | # Start service and check that it has started. 86 | service_started = False 87 | command = ["apply", "-f", output_file_name] 88 | 89 | # Namespace 90 | command.append("-n") 91 | command.append(mode.namespace) 92 | 93 | if kubernetes_utils.invoke_kubectl_command(command, 94 | service_name=service_name): 95 | command = ["get", "services", "-w"] 96 | 97 | # Namespace 98 | command.append("-n") 99 | command.append(mode.namespace) 100 | 101 | service_started = kubernetes_utils.invoke_kubectl_command( 102 | command, service_name=service_name, ongoing=True) 103 | 104 | if service_started: 105 | print("Service %s has been started successfully." % service_name) 106 | 107 | # Remove intermediate file. 108 | if output_file_name is not None: 109 | os.remove(output_file_name) 110 | 111 | 112 | # Create service from yaml configuration: 113 | 114 | 115 | def deploy_image_as_service(service_name, namespace, configuration_path): 116 | # TODO: adapt this to support multiple namespaces. 117 | command = ["apply"] 118 | 119 | # Namespace 120 | if namespace != "default": 121 | command.append("-n") 122 | command.append(namespace) 123 | 124 | # Service configuration 125 | command.append("-f") 126 | command.append(configuration_path) 127 | 128 | service_started = kubernetes_utils.invoke_kubectl_command( 129 | command, service_name=service_name, with_output=True) 130 | 131 | pod_running_status(service_name, namespace) 132 | 133 | if service_started: 134 | print("Service %s has been started successfully." % service_name) 135 | 136 | 137 | # Delete service. 138 | 139 | 140 | def delete_service(mode, service_name): 141 | command = ["delete", "service", service_name] 142 | 143 | # Namespace 144 | command.append("-n") 145 | command.append(mode.namespace) 146 | 147 | return kubernetes_utils.invoke_kubectl_command(command, 148 | service_name=service_name) 149 | -------------------------------------------------------------------------------- /rayvens/core/local.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import rayvens.core.catalog as catalog 18 | from rayvens.core.integration import Integration 19 | from rayvens.core.common import get_run_mode, send_to, recv_from, await_start 20 | 21 | 22 | def start(camel_mode, check_port, release): 23 | return Camel(get_run_mode(camel_mode, check_port, release)) 24 | 25 | 26 | class Camel: 27 | def __init__(self, mode): 28 | self.mode = mode 29 | self.mode.transport = 'http' 30 | 31 | def add_source(self, stream, config, source_name): 32 | integration = Integration(stream.name, source_name, config) 33 | route = integration.route(default='/source') 34 | spec = catalog.construct_source(config, 35 | f'platform-http:{route}', 36 | inverted=True) 37 | integration.prepare_environment(self.mode) 38 | integration.invoke_local_run(self.mode, spec) 39 | integration.thread = send_to(stream.actor, 40 | self.mode.server_address(integration), 41 | route) 42 | if not await_start(self.mode, integration): 43 | raise RuntimeError('Could not start source') 44 | return integration 45 | 46 | def add_sink(self, stream, config, sink_name): 47 | integration = Integration(stream.name, sink_name, config) 48 | route = integration.route(default='/sink') 49 | spec = catalog.construct_sink(config, f'platform-http:{route}') 50 | integration.prepare_environment(self.mode) 51 | integration.invoke_local_run(self.mode, spec) 52 | recv_from(stream.actor, sink_name, 53 | self.mode.server_address(integration), route) 54 | if not await_start(self.mode, integration): 55 | raise RuntimeError('Could not start source') 56 | return integration 57 | 58 | def disconnect(self, integration): 59 | integration.disconnect(self.mode) 60 | -------------------------------------------------------------------------------- /rayvens/core/mode.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from enum import Enum 18 | from rayvens.core import utils 19 | 20 | # Port for internal communication inside the cluster. 21 | internal_cluster_port = "80" 22 | 23 | # Cluster source port. 24 | internal_cluster_port_for_source = "8000" 25 | 26 | 27 | class RayvensMode(Enum): 28 | # Ray and Kamel running locally. 29 | LOCAL = 1 30 | 31 | # Ray running locally, Kamel local running in a container in the cluster. 32 | MIXED_LOCAL = 2 33 | 34 | # Ray running locally, Kamel operator running in the cluster. 35 | MIXED_OPERATOR = 3 36 | 37 | # Ray in cluster, Kamel local running in a container in the cluster. 38 | CLUSTER_LOCAL = 4 39 | 40 | # Ray in cluster, Kamel operator running in the cluster. 41 | CLUSTER_OPERATOR = 5 42 | 43 | # Ray operator in cluster, Kamel local running in a container in cluster. 44 | OPERATOR_LOCAL = 6 45 | 46 | # Ray operator in cluster, Kamel operator running in the cluster. 47 | OPERATOR_OPERATOR = 7 48 | 49 | 50 | class RunMode: 51 | def __init__(self, run_mode=RayvensMode.LOCAL): 52 | self.run_mode = run_mode 53 | self.namespace = "ray" 54 | self.transport = None 55 | self.check_port = True 56 | self.release = False 57 | 58 | def server_address(self, integration): 59 | return self._get_server_address(integration.integration_name, 60 | port=integration.port) 61 | 62 | def is_local(self): 63 | return self.run_mode == RayvensMode.LOCAL 64 | 65 | def is_mixed(self): 66 | return self.run_mode == RayvensMode.MIXED_OPERATOR 67 | 68 | def is_cluster(self): 69 | return self.run_mode == RayvensMode.CLUSTER_OPERATOR 70 | 71 | def _get_server_address(self, 72 | integration_name, 73 | serve_source=False, 74 | port=None): 75 | if self.run_mode == RayvensMode.LOCAL: 76 | # Default setup: "http://0.0.0.0:8080" 77 | if port is None: 78 | raise RuntimeError('port is not specified') 79 | return f'http://localhost:{port}' 80 | if self.run_mode == RayvensMode.MIXED_OPERATOR: 81 | return "http://localhost:%s" % utils.externalized_cluster_port 82 | if self.run_mode == RayvensMode.CLUSTER_OPERATOR: 83 | if integration_name == "": 84 | raise RuntimeError("integration name is not set") 85 | if serve_source: 86 | return "http://%s.%s.svc.cluster.local:%s" % ( 87 | integration_name, self.namespace, 88 | internal_cluster_port_for_source) 89 | return "http://%s.%s.svc.cluster.local:%s" % ( 90 | integration_name, self.namespace, internal_cluster_port) 91 | raise RuntimeError("unreachable") 92 | 93 | 94 | # Default execution mode. 95 | mode = RunMode() 96 | -------------------------------------------------------------------------------- /rayvens/core/name.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import re 18 | 19 | 20 | # name if provided else kind 21 | def name_source(config): 22 | if 'kind' not in config: 23 | raise TypeError('A Camel source needs a kind.') 24 | return config.get('name', config['kind']) 25 | 26 | 27 | # name if provided else kind 28 | def name_sink(config): 29 | if 'kind' not in config: 30 | raise TypeError('A Camel sink needs a kind.') 31 | return config.get('name', config['kind']) 32 | 33 | 34 | # sanitize name for use as Kubernetes name and file name 35 | def sanitize(name): 36 | name = str(name) 37 | name = name.lower() 38 | name = re.sub('[^a-z0-9]', '-', name) 39 | name = re.sub('-{2,}', '-', name) 40 | name = name[-253:] 41 | name = name.strip('-') 42 | return name 43 | 44 | 45 | # combine stream name and source/sink name 46 | def name_integration(stream_name, name_or_kind): 47 | return sanitize(stream_name + '-' + name_or_kind) 48 | -------------------------------------------------------------------------------- /rayvens/core/operator.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from rayvens.core.common import get_run_mode, send_to, await_start 18 | from rayvens.core.common import ProducerActor 19 | from rayvens.core.catalog import construct_source, construct_sink 20 | from rayvens.core.integration import Integration 21 | 22 | 23 | def start(camel_mode, check_port, release): 24 | return Camel(get_run_mode(camel_mode, check_port, release)) 25 | 26 | 27 | class Camel: 28 | def __init__(self, mode): 29 | self.mode = mode 30 | self.mode.transport = 'http' 31 | 32 | def add_source(self, stream, source, source_name): 33 | # Construct integration 34 | integration = Integration(stream.name, source_name, source) 35 | 36 | # Construct endpoint. First, get route: 37 | route = integration.route() 38 | 39 | # Prepare env: 40 | integration.prepare_environment(self.mode) 41 | 42 | # Determine the `to` endpoint value made up of a base address and 43 | # a custom route provided by the user. Use this to construct the 44 | # integration source code. 45 | integration_content = construct_source(source, 46 | f'platform-http:{route}', 47 | inverted=True) 48 | 49 | # Start running the source integration. 50 | integration.invoke_run(self.mode, integration_content) 51 | 52 | # Set up source for the HTTP connector case. 53 | integration.thread = send_to(stream.actor, 54 | self.mode.server_address(integration), 55 | route) 56 | 57 | if not await_start(self.mode, integration): 58 | raise RuntimeError('Could not start source') 59 | return integration 60 | 61 | def add_sink(self, stream, sink, sink_name): 62 | # Construct integration 63 | integration = Integration(stream.name, sink_name, sink) 64 | 65 | # Extract integration properties: 66 | route = integration.route() 67 | 68 | # Prepare env: 69 | integration.prepare_environment(self.mode) 70 | 71 | # Get integration source code. 72 | integration_content = construct_sink(sink, f'platform-http:{route}') 73 | 74 | # Start running the integration. 75 | integration.invoke_run(self.mode, integration_content) 76 | 77 | helper = ProducerActor.remote( 78 | self.mode.server_address(integration) + route) 79 | stream.actor.send_to.remote(helper, sink_name) 80 | 81 | # Wait for integration to finish. 82 | if not await_start(self.mode, integration): 83 | raise RuntimeError('Could not start sink') 84 | 85 | return integration 86 | 87 | def disconnect(self, integration): 88 | integration.disconnect(self.mode) 89 | -------------------------------------------------------------------------------- /rayvens/core/operator_kafka.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | from rayvens.core.common import get_run_mode, await_start, brokers 18 | from rayvens.core.common import kafka_send_to, kafka_recv_from 19 | from rayvens.core.catalog import construct_source, construct_sink 20 | from rayvens.core.integration import Integration 21 | 22 | 23 | def start(camel_mode, check_port, release): 24 | return Camel(get_run_mode(camel_mode, check_port, release)) 25 | 26 | 27 | class Camel: 28 | def __init__(self, mode): 29 | self.mode = mode 30 | self.mode.transport = 'kafka' 31 | 32 | def add_source(self, stream, source, source_name): 33 | # Construct integration 34 | integration = Integration(stream.name, source_name, source) 35 | 36 | # Prepare env: 37 | integration.prepare_environment(self.mode) 38 | 39 | # Determine the `to` endpoint value made up of a base address and 40 | # a custom route provided by the user. Use this to construct the 41 | # integration source code. 42 | integration_content = construct_source( 43 | source, 44 | f'kafka:{integration.kafka_transport_topic}?brokers={brokers()}') 45 | 46 | # Start running the source integration. 47 | integration.invoke_run(self.mode, integration_content) 48 | 49 | # Set up source for the HTTP connector case. 50 | integration.thread = kafka_send_to( 51 | integration.kafka_transport_topic, 52 | integration.kafka_transport_partitions, stream.actor) 53 | 54 | if not await_start(self.mode, integration): 55 | raise RuntimeError('Could not start source') 56 | return integration 57 | 58 | def add_sink(self, stream, sink, sink_name): 59 | # Construct integration 60 | integration = Integration(stream.name, sink_name, sink) 61 | 62 | # Prepare env: 63 | integration.prepare_environment(self.mode) 64 | 65 | # Get integration source code. 66 | integration_content = construct_sink( 67 | sink, 68 | f'kafka:{integration.kafka_transport_topic}?brokers={brokers()}') 69 | 70 | # Start running the integration. 71 | integration.invoke_run(self.mode, integration_content) 72 | 73 | kafka_recv_from(sink_name, integration.kafka_transport_topic, 74 | stream.actor) 75 | 76 | # Wait for integration to finish. 77 | if not await_start(self.mode, integration): 78 | raise RuntimeError('Could not start sink') 79 | 80 | return integration 81 | 82 | def disconnect(self, integration): 83 | integration.disconnect(self.mode) 84 | -------------------------------------------------------------------------------- /rayvens/core/ray_serve.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | from ray import serve 19 | from rayvens.core.kamel_backend import KamelBackend 20 | from rayvens.core import utils 21 | from rayvens.core import common 22 | from rayvens.core.catalog import construct_source, construct_sink 23 | from rayvens.core.common import await_start 24 | from rayvens.core.integration import Integration 25 | 26 | 27 | def start(camel_mode, check_port, release): 28 | return Camel(common.get_run_mode(camel_mode, check_port, release)) 29 | 30 | 31 | class Camel: 32 | def __init__(self, mode): 33 | self.mode = mode 34 | self.mode.transport = 'ray-serve' 35 | 36 | # The Ray Serve backend used for Sinks. Sink are a special case and 37 | # can use one backend to support multiple sinks. 38 | self.kamel_backend = None 39 | 40 | # Start server is using a backend. 41 | serve.start() 42 | 43 | def add_source(self, stream, source, source_name): 44 | # Construct integration 45 | integration = Integration(stream.name, source_name, source) 46 | 47 | # Construct endpoint. First, get route: 48 | route = integration.route() 49 | 50 | # Determine the `to` endpoint value made up of a base address and 51 | # a custom route provided by the user. The computation depends on 52 | # the connector type used for the implementation. 53 | server_pod_name = "" 54 | if self.mode.is_cluster(): 55 | server_pod_name = utils.get_server_pod_name() 56 | endpoint_base = self.mode._get_server_address(server_pod_name, 57 | serve_source=True) 58 | 59 | # Construct integration source code. When the ray serve connector is 60 | # not enabled, use an HTTP inverted connection. 61 | integration_content = construct_source(source, 62 | f'{endpoint_base}{route}') 63 | 64 | # Set endpoint and integration names. 65 | endpoint_name = self._get_endpoint_name(integration.integration_name) 66 | 67 | # Create backend for this topic. 68 | source_backend = KamelBackend(self.mode, topic=stream.actor) 69 | 70 | # Create endpoint. 71 | source_backend.createProxyEndpoint(endpoint_name, route, 72 | integration.integration_name) 73 | 74 | # Start running the source integration. 75 | integration.invoke_run(self.mode, integration_content) 76 | 77 | if not await_start(self.mode, integration): 78 | raise RuntimeError('Could not start source') 79 | return integration 80 | 81 | def add_sink(self, stream, sink, sink_name): 82 | # Construct integration 83 | integration = Integration(stream.name, sink_name, sink) 84 | 85 | # Extract integration properties: 86 | route = integration.route() 87 | use_backend = integration.use_backend() 88 | integration_name = integration.integration_name 89 | 90 | # Get integration source code. 91 | integration_content = construct_sink(sink, f'platform-http:{route}') 92 | 93 | # Create backend if one hasn't been created so far. 94 | if use_backend and self.kamel_backend is None: 95 | self.kamel_backend = KamelBackend(self.mode) 96 | 97 | # Start running the integration. 98 | integration.invoke_run(self.mode, integration_content) 99 | 100 | if use_backend: 101 | endpoint_name = self._get_endpoint_name(stream.name) 102 | self.kamel_backend.createProxyEndpoint(endpoint_name, route, 103 | integration_name) 104 | 105 | helper = HelperWithBackend.remote(self.kamel_backend, 106 | serve.get_handle(endpoint_name), 107 | endpoint_name) 108 | else: 109 | helper = common.ProducerActor.remote( 110 | self.mode.server_address(integration_name) + route) 111 | stream.actor.send_to.remote(helper, sink_name) 112 | 113 | # Wait for integration to finish. 114 | if not await_start(self.mode, integration): 115 | raise RuntimeError('Could not start sink') 116 | 117 | return integration 118 | 119 | def disconnect(self, integration): 120 | integration.disconnect(self.mode) 121 | 122 | def _get_endpoint_name(self, integration_name): 123 | return "_".join(["endpoint", integration_name]) 124 | 125 | 126 | @ray.remote(num_cpus=0) 127 | class HelperWithBackend: 128 | def __init__(self, backend, endpoint_handle, endpoint_name): 129 | self.backend = backend 130 | self.endpoint_name = endpoint_name 131 | self.endpoint_handle = endpoint_handle 132 | 133 | def append(self, data): 134 | if data is not None: 135 | answer = self.backend.postToProxyEndpointHandle( 136 | self.endpoint_handle, self.endpoint_name, data) 137 | print(answer) 138 | -------------------------------------------------------------------------------- /rayvens/core/utils.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | import random 19 | import threading 20 | from queue import Queue, Empty 21 | 22 | # Port externalized by the cluster. 23 | externalized_cluster_port = "31095" 24 | 25 | # Check if the executable exists in PATH. This method should work 26 | # in Windows, Linux and MacOS. Python >= 3.3 required. 27 | 28 | rayvens_random = random.Random() 29 | rayvens_random.seed() 30 | 31 | 32 | def random_port(check_port=True, start=49152, end=65535): 33 | port = rayvens_random.randint(start, end) 34 | if not check_port: 35 | return port 36 | 37 | while not _port_is_free(port): 38 | print(f"Port {port} busy, trying new port.") 39 | port = rayvens_random.randint(start, end) 40 | 41 | return port 42 | 43 | 44 | def _port_is_free(port): 45 | import socket 46 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 47 | 48 | # Attempt to bind to the port if it fails, port must be busy. 49 | port_is_free = True 50 | try: 51 | sock.bind(('127.0.0.1', port)) 52 | except socket.error as message: 53 | print(f'Port {port} is in use (Error: {str(message)}).') 54 | port_is_free = False 55 | 56 | sock.close() 57 | return port_is_free 58 | 59 | 60 | def executable_is_available(executable): 61 | # If this is a path to the executable file return true. 62 | if os.path.isfile(executable): 63 | return True 64 | 65 | # Look up executable in path. 66 | from shutil import which 67 | return which(executable) is not None 68 | 69 | 70 | def subprocess_tag(subprocess_name): 71 | return "[%s subprocess]" % subprocess_name 72 | 73 | 74 | def print_log_from_subprocess(subprocess_name, stdout, with_output): 75 | output = stdout.readline().decode("utf-8") 76 | output = output.strip() 77 | 78 | if output != "" and with_output: 79 | print(subprocess_tag(subprocess_name), output) 80 | 81 | return output 82 | 83 | 84 | class LogThread(threading.Thread): 85 | def __init__(self, stdout): 86 | threading.Thread.__init__(self) 87 | self.stop_flag = threading.Event() 88 | self.read_flag = threading.Event() 89 | self.stdout = stdout 90 | self.queue = Queue() 91 | 92 | def run(self): 93 | while not self.stop_flag.is_set(): 94 | if self.read_flag.is_set(): 95 | line = self.stdout.readline().decode("utf-8").strip() 96 | if line != "": 97 | self.queue.put(line) 98 | self.read_flag.clear() 99 | 100 | print("[Logging thread] Kamel command logging terminated.") 101 | 102 | 103 | def print_log_from_queue(subprocess_name, queue, with_output): 104 | try: 105 | line = queue.get_nowait() 106 | except Empty: 107 | return None 108 | else: 109 | if with_output: 110 | print(subprocess_tag(subprocess_name), line) 111 | return line 112 | 113 | 114 | def print_log(subprocess_name, message): 115 | print(subprocess_tag(subprocess_name), message) 116 | 117 | 118 | def get_server_pod_name(): 119 | with open('/etc/podinfo/labels', 'r') as f: 120 | for line in f: 121 | k, v = line.partition('=')[::2] 122 | if k == 'component': 123 | return f'{v[1:-2]}' 124 | 125 | raise RuntimeError("Cannot find server pod name") 126 | 127 | 128 | def create_partitioned_topic(topic, partitions, brokers): 129 | # Create new topic 130 | from confluent_kafka.admin import AdminClient, NewTopic 131 | admin_client = AdminClient({"bootstrap.servers": brokers}) 132 | 133 | # TODO: Smart choice for replication factor, for now use a 134 | # replication factor of 1. 135 | topics = [NewTopic(topic, num_partitions=partitions, replication_factor=1)] 136 | admin_client.create_topics(topics) 137 | 138 | # Wait for topic to be ready: 139 | topic_ready = False 140 | while not topic_ready: 141 | for enabled_topic in admin_client.list_topics().topics: 142 | if topic == enabled_topic: 143 | topic_ready = True 144 | break 145 | print(f"Topic {topic} is ready.") 146 | -------------------------------------------------------------------------------- /rayvens/core/verify.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import time 19 | from rayvens.core import kamel 20 | 21 | 22 | def verify_do(handle, _global_camel, action, *args, **kwargs): 23 | if action == 'verify_log': 24 | return _verify_log(handle, _global_camel, *args, **kwargs) 25 | raise RuntimeError('invalid meta action') 26 | 27 | 28 | def wait_for_event(handle): 29 | event_count = 0 30 | countdown = 20 31 | while event_count == 0: 32 | event_count = ray.get(handle.event_count.remote()) 33 | time.sleep(1) 34 | countdown -= 1 35 | if countdown == 0: 36 | break 37 | if event_count == 0: 38 | return False 39 | return True 40 | 41 | 42 | def _verify_log(handle, 43 | _global_camel, 44 | sink_source_name, 45 | message, 46 | wait_for_events=False): 47 | log = "FAIL" 48 | 49 | # Wait for at least one event to happen. 50 | if wait_for_events: 51 | if not wait_for_event(handle): 52 | print("[LOG CHECK]:", "NO EVENTS RECEIVED") 53 | return False 54 | 55 | if _global_camel.mode.is_local(): 56 | # In the local case the integration run is ongoing and we can 57 | # access the logs directly. 58 | outcome = ray.get( 59 | handle._integration_invoke.remote(sink_source_name, message)) 60 | else: 61 | # When running using the operator then the integration run command 62 | # is non-blocking and returns immediately. The logs can be queried 63 | # using the kamel log command. 64 | integration_name = ray.get( 65 | handle._get_integration_name.remote(sink_source_name)) 66 | invocation = kamel.log(_global_camel.mode, integration_name, message) 67 | outcome = invocation is not None 68 | invocation.kill() 69 | 70 | if outcome: 71 | log = "SUCCESS" 72 | print("[LOG CHECK]:", log) 73 | return outcome 74 | -------------------------------------------------------------------------------- /rayvens/rayvens: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright IBM Corporation 2021 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | python_command=python 20 | 21 | if command -v python3 &> /dev/null 22 | then 23 | python_command=python3 24 | fi 25 | 26 | if ! command -v $python_command &> /dev/null 27 | then 28 | echo "a valid python command could not be found" 29 | exit 30 | fi 31 | 32 | # Find full path to rayvens_cli.sh file: 33 | rayvens_cli_file=`$python_command -c "import subprocess; import platform; cmd = \"where\" if platform.system() == \"Windows\" else \"which\"; subprocess.call([cmd, \"rayvens_cli.py\"]);"` 34 | 35 | # Invoke command line processing: 36 | $python_command $rayvens_cli_file $@ -------------------------------------------------------------------------------- /resources/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/project-codeflare/rayvens/197831984cc2f5cce0dd1696e0db3fd1f09ffba8/resources/logo.png -------------------------------------------------------------------------------- /resources/logo.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/project-codeflare/rayvens/197831984cc2f5cce0dd1696e0db3fd1f09ffba8/resources/logo.pptx -------------------------------------------------------------------------------- /scripts/Preloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright IBM Corporation 2021 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import org.apache.camel.BindToRegistry; 18 | import org.apache.camel.Exchange; 19 | import org.apache.camel.Processor; 20 | import org.apache.camel.builder.RouteBuilder; 21 | 22 | class Exit implements Processor { 23 | public void process(Exchange exchange) throws Exception { 24 | System.exit(0); 25 | } 26 | } 27 | 28 | public class Preloader extends RouteBuilder { 29 | @Override 30 | public void configure() throws Exception { 31 | from("timer:tick").to("bean:exit"); 32 | from("platform-http:/null").to("http:null"); 33 | } 34 | 35 | @BindToRegistry 36 | public Exit exit() { 37 | return new Exit(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /scripts/camel-test-source.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | - from: 18 | uri: timer:tick?period=3000 19 | steps: 20 | - to: https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL 21 | - log: 22 | message: "${body}" 23 | -------------------------------------------------------------------------------- /scripts/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | version: "3" 18 | services: 19 | zookeeper: 20 | image: zookeeper:3.5 21 | hostname: zookeeper 22 | container_name: zookeeper 23 | restart: ${RESTART_POLICY:-no} 24 | kafka: 25 | image: wurstmeister/kafka:2.13-2.6.0 26 | hostname: kafka 27 | container_name: kafka 28 | command: [start-kafka.sh] 29 | ports: 30 | - "31093:9093" 31 | environment: 32 | HOSTNAME_COMMAND: "hostname -f" 33 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: ONBUS:PLAINTEXT,EXTERNAL:PLAINTEXT 34 | KAFKA_LISTENERS: ONBUS://:9092,EXTERNAL://:9093 35 | KAFKA_ADVERTISED_LISTENERS: ONBUS://_{HOSTNAME_COMMAND}:9092,EXTERNAL://localhost:31093 36 | KAFKA_INTER_BROKER_LISTENER_NAME: ONBUS 37 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 38 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 39 | depends_on: 40 | - "zookeeper" 41 | -------------------------------------------------------------------------------- /scripts/kafka.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | apiVersion: apps/v1 18 | kind: StatefulSet 19 | metadata: 20 | name: zookeeper 21 | spec: 22 | replicas: 1 23 | selector: 24 | matchLabels: 25 | name: zookeeper 26 | serviceName: zookeeper 27 | template: 28 | metadata: 29 | labels: 30 | name: zookeeper 31 | spec: 32 | containers: 33 | - name: zk 34 | image: docker.io/zookeeper:3.5 35 | ports: 36 | - name: zookeeper 37 | containerPort: 2181 38 | - name: server 39 | containerPort: 2888 40 | - name: leader-election 41 | containerPort: 3888 42 | livenessProbe: 43 | tcpSocket: 44 | port: 2181 45 | readinessProbe: 46 | exec: 47 | command: 48 | - /bin/bash 49 | - -c 50 | - "echo ruok | nc -w 1 localhost 2181 | grep imok" 51 | env: 52 | - name: ZOO_4LW_COMMANDS_WHITELIST 53 | value: "srvr,ruok" 54 | --- 55 | apiVersion: v1 56 | kind: Service 57 | metadata: 58 | name: zookeeper 59 | spec: 60 | type: ClusterIP 61 | clusterIP: None 62 | selector: 63 | name: zookeeper 64 | ports: 65 | - name: zookeeper 66 | port: 2181 67 | targetPort: 2181 68 | - name: server 69 | port: 2888 70 | targetPort: 2888 71 | - name: leader-election 72 | port: 3888 73 | targetPort: 3888 74 | --- 75 | apiVersion: apps/v1 76 | kind: StatefulSet 77 | metadata: 78 | name: kafka 79 | spec: 80 | replicas: 1 81 | selector: 82 | matchLabels: 83 | name: kafka 84 | serviceName: kafka 85 | template: 86 | metadata: 87 | labels: 88 | name: kafka 89 | spec: 90 | initContainers: 91 | - name: wait-for-zookeeper 92 | image: busybox 93 | command: 94 | [ 95 | "sh", 96 | "-c", 97 | 'result=1; until [ $result -eq 0 ]; do OK=$(echo ruok | nc -w 1 zookeeper 2181); if [ "$OK" == "imok" ]; then result=0; echo "zookeeper returned imok!"; else echo waiting for zookeeper to be ready; sleep 1; fi; done; echo "Success: zookeeper is up"', 98 | ] 99 | containers: 100 | - name: kafka 101 | image: docker.io/wurstmeister/kafka:2.13-2.6.0 102 | ports: 103 | - name: kafka-internal 104 | containerPort: 9092 105 | - name: kafka-external 106 | containerPort: 9093 107 | readinessProbe: 108 | initialDelaySeconds: 10 109 | timeoutSeconds: 5 110 | periodSeconds: 10 111 | exec: 112 | command: 113 | - /opt/kafka/bin/kafka-topics.sh 114 | - localhost:9092 115 | - --version 116 | env: 117 | - name: HOSTNAME_COMMAND 118 | value: hostname -f 119 | - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP 120 | value: INCLUSTER:PLAINTEXT,EXTERNAL:PLAINTEXT 121 | - name: KAFKA_LISTENERS 122 | value: INCLUSTER://:9092,EXTERNAL://:9093 123 | - name: KAFKA_ADVERTISED_LISTENERS 124 | value: INCLUSTER://_{HOSTNAME_COMMAND}:9092,EXTERNAL://localhost:31093 125 | - name: KAFKA_INTER_BROKER_LISTENER_NAME 126 | value: INCLUSTER 127 | - name: KAFKA_ZOOKEEPER_CONNECT 128 | value: zookeeper:2181 129 | - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE 130 | value: "true" 131 | - name: KAFKA_PORT 132 | value: "9092" 133 | --- 134 | apiVersion: v1 135 | kind: Service 136 | metadata: 137 | name: kafka 138 | spec: 139 | type: NodePort 140 | selector: 141 | name: kafka 142 | ports: 143 | - name: kafka-internal 144 | port: 9092 145 | targetPort: 9092 146 | - name: kafka-external 147 | port: 9093 148 | targetPort: 9093 149 | nodePort: 31093 150 | -------------------------------------------------------------------------------- /scripts/kind.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | kind: Cluster 18 | apiVersion: kind.x-k8s.io/v1alpha4 19 | containerdConfigPatches: 20 | - |- 21 | [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"] 22 | endpoint = ["http://registry:5000"] 23 | [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"] 24 | endpoint = ["http://registry:5000"] 25 | nodes: 26 | - role: control-plane 27 | extraPortMappings: 28 | - containerPort: 31093 29 | hostPort: 31093 30 | protocol: TCP 31 | - containerPort: 31095 32 | hostPort: 31095 33 | protocol: TCP 34 | -------------------------------------------------------------------------------- /scripts/start-kind.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright IBM Corporation 2021 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | cd $(python -c 'import os,sys;print(os.path.dirname(os.path.realpath(sys.argv[1])))' "$0") 20 | 21 | echo "--- starting private docker registry" 22 | docker run -d --restart=always -p 5000:5000 --name registry registry:2 23 | 24 | echo "--- building the rayvens image" 25 | docker build .. -t localhost:5000/rayvens 26 | docker push localhost:5000/rayvens 27 | 28 | echo "--- starting kind cluster" 29 | kind delete cluster 30 | kind create cluster --config kind.yaml 31 | docker network connect kind registry 32 | 33 | echo "--- installing kafka" 34 | kubectl create namespace ray 35 | kubectl apply -n ray -f kafka.yaml 36 | 37 | echo "--- installing kamel operator" 38 | kubectl create serviceaccount kamel -n ray 39 | kubectl create clusterrolebinding kamel --clusterrole=cluster-admin --serviceaccount=ray:kamel 40 | kubectl run --rm -i -t kamel --image=apache/camel-k:1.5.1 --restart=Never --serviceaccount=kamel -n ray -- \ 41 | kamel install --registry-insecure --namespace ray --registry registry:5000 42 | kubectl delete clusterrolebinding kamel 43 | kubectl delete serviceaccount kamel -n ray 44 | 45 | echo "--- starting ray cluster" 46 | ray up cluster.yaml --no-config-cache --yes 47 | -------------------------------------------------------------------------------- /scripts/travis.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | set -eu 18 | 19 | KAMEL_VERSION=v1.5.1 20 | KIND_VERSION=v0.11.1 21 | KUBECTL_VERSION=v1.18.8 22 | 23 | # Download and install command line tools 24 | pushd /tmp 25 | # kubectl 26 | echo 'installing kubectl' 27 | curl -Lo ./kubectl https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl 28 | chmod +x kubectl 29 | sudo cp kubectl /usr/local/bin/kubectl 30 | 31 | # kind 32 | echo 'installing kind' 33 | curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/$KIND_VERSION/kind-linux-amd64 34 | chmod +x kind 35 | sudo cp kind /usr/local/bin/kind 36 | 37 | # kamel 38 | echo 'installing kamel' 39 | curl -L https://github.com/apache/camel-k/releases/download/$KAMEL_VERSION/camel-k-client-${KAMEL_VERSION#?}-linux-64bit.tar.gz | tar zx ./kamel 40 | sudo cp kamel /usr/local/bin/kamel 41 | 42 | popd 43 | -------------------------------------------------------------------------------- /scripts/update-rayvens.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright IBM Corporation 2021 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | cd $(python -c 'import os,sys;print(os.path.dirname(os.path.realpath(sys.argv[1])))' "$0") 20 | 21 | docker build .. -t localhost:5000/rayvens 22 | docker push localhost:5000/rayvens 23 | ray down cluster.yaml --yes 24 | ray up cluster.yaml --no-config-cache --yes 25 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import re 18 | from setuptools import setup 19 | 20 | long_description = '''Rayvens augments [Ray](https://ray.io) with events. With 21 | Rayvens, Ray applications can subscribe to event streams, process and produce 22 | events. Rayvens leverages [Apache Camel](https://camel.apache.org) to make it 23 | possible for data scientists to access hundreds of data services with little 24 | effort. 25 | 26 | For the full documentation see 27 | [https://github.com/project-codeflare/rayvens](https://github.com/project-codeflare/rayvens). 28 | ''' 29 | 30 | with open('scripts/rayvens-setup.sh') as f: 31 | version = re.findall('rayvens_version=([0-9.]+)', f.read())[0] 32 | 33 | setup( 34 | name='rayvens', 35 | long_description=long_description, 36 | long_description_content_type='text/markdown', 37 | packages=['rayvens', 'rayvens.core', 'rayvens.cli'], 38 | package_data={'rayvens.core': ['*.java']}, 39 | install_requires=[ 40 | 'confluent-kafka>=1.6.1', 'ray[default,serve,k8s]>=1.3.0' 41 | ], 42 | scripts=[ 43 | 'scripts/rayvens-setup.sh', 'rayvens/rayvens', 'rayvens/rayvens_cli.py' 44 | ], 45 | version=version, 46 | python_requires='>=3.6', 47 | description='Rayvens augments Ray with events.', 48 | license='Apache 2.0', 49 | author='Rayvens authors', 50 | author_email='tardieu@us.ibm.com, gheorghe-teod.bercea@ibm.com', 51 | keywords=("ray events kubernetes"), 52 | url='https://github.com/project-codeflare/rayvens', 53 | classifiers=[ 54 | 'Development Status :: 4 - Beta', 55 | 'Intended Audience :: Developers', 56 | 'License :: OSI Approved :: Apache Software License', 57 | 'Programming Language :: Java', 58 | 'Programming Language :: Python', 59 | 'Programming Language :: Python :: 3', 60 | 'Programming Language :: Python :: 3 :: Only', 61 | 'Operating System :: OS Independent', 62 | 'Topic :: System :: Distributed Computing', 63 | ], 64 | project_urls={ 65 | 'Bug Reports': 'https://github.com/project-codeflare/rayvens/issues', 66 | 'Source': 'https://github.com/project-codeflare/rayvens', 67 | }, 68 | ) 69 | -------------------------------------------------------------------------------- /tests/generic_sink.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import os 20 | 21 | # Initialize ray based on where ray will run inside the cluster using the 22 | # kamel operator. 23 | 24 | # Default run mode. 25 | run_mode = 'operator' 26 | env_run_mode = os.getenv('RAYVENS_TEST_MODE') 27 | if env_run_mode is not None: 28 | run_mode = env_run_mode 29 | 30 | # Select appropriate Ray init method. 31 | if run_mode == 'operator': 32 | ray.init(address='auto') 33 | else: 34 | ray.init(object_store_memory=78643200) 35 | 36 | # Start the test. 37 | 38 | # Start rayvens in operator mode. 39 | rayvens.init(mode=run_mode) 40 | 41 | # Create stream. 42 | stream = rayvens.Stream('test-sink') 43 | 44 | # Event sink config. 45 | test_sink_config = dict(kind='generic-sink', 46 | route='/togenericsink', 47 | spec=""" 48 | - from: 49 | steps: 50 | - log: 51 | message: '"${body}"' 52 | """) 53 | 54 | # Add sink to stream. 55 | sink = stream.add_sink(test_sink_config) 56 | 57 | # Sends message to all sinks attached to this stream. 58 | output_message = f'Sending message to Slack sink in run mode {run_mode}.' 59 | stream << output_message 60 | 61 | # Verify outcome. 62 | rayvens.meta(stream, 'verify_log', sink, output_message, wait_for_events=True) 63 | 64 | # Delete all integrations from stream. 65 | stream.disconnect_all(after_idle_for=5) 66 | -------------------------------------------------------------------------------- /tests/generic_source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import asyncio 18 | import json 19 | import os 20 | import ray 21 | import rayvens 22 | 23 | # Default run mode. 24 | run_mode = 'operator' 25 | env_run_mode = os.getenv('RAYVENS_TEST_MODE') 26 | if env_run_mode is not None: 27 | run_mode = env_run_mode 28 | 29 | # Select appropriate Ray init method. 30 | if run_mode == 'operator': 31 | ray.init(address='auto') 32 | else: 33 | ray.init(object_store_memory=78643200) 34 | 35 | 36 | # Actor class for processing the events from the source. 37 | @ray.remote 38 | class Counter: 39 | def __init__(self): 40 | self.count = 0 41 | self.ready = asyncio.Event() 42 | 43 | def append(self, event): 44 | print( 45 | 'AAPL is', 46 | json.loads(event)['quoteResponse']['result'][0] 47 | ['regularMarketPrice']) 48 | self.count += 1 49 | if self.count > 5: 50 | self.ready.set() 51 | 52 | async def wait(self): 53 | await self.ready.wait() 54 | 55 | 56 | # Start the test. 57 | 58 | # Start rayvens in the desired mode. 59 | rayvens.init(mode=run_mode) 60 | 61 | # Config for the source. 62 | source_config = dict(kind='generic-source', 63 | spec=""" 64 | - from: 65 | uri: timer:tick?period=3000 66 | steps: 67 | - to: https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL 68 | """) 69 | 70 | # Create stream where we can attach sinks, sources and operators. 71 | stream = rayvens.Stream('http') 72 | 73 | # Attach a source to the stream. 74 | source = stream.add_source(source_config) 75 | 76 | # Instantiate the processor class for the events. 77 | counter = Counter.remote() 78 | 79 | # Send all events from the source to the processor. 80 | stream >> counter 81 | 82 | ray.get(counter.wait.remote(), timeout=180) 83 | 84 | # Delete all integrations from stream. 85 | stream.disconnect_all() 86 | -------------------------------------------------------------------------------- /tests/kafka.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | import ray 19 | import rayvens 20 | import time 21 | 22 | # Initialize run mode. 23 | run_mode = 'operator' 24 | env_run_mode = os.getenv('RAYVENS_TEST_MODE') 25 | if env_run_mode is not None: 26 | run_mode = env_run_mode 27 | 28 | if run_mode == 'operator': 29 | ray.init(address='auto') 30 | else: 31 | ray.init(object_store_memory=78643200) 32 | 33 | # The Kafka topic used for communication. 34 | topic = "testTopic" 35 | 36 | # If using the Kafka broker started by Rayvens the following brokers 37 | # are possible: 38 | # - from inside the cluster: kafka:9092 39 | # - from outside the cluster: localhost:31093 40 | broker = 'localhost:31093' 41 | if run_mode == 'operator': 42 | broker = "kafka:9092" 43 | 44 | rayvens.init(mode=run_mode) 45 | 46 | # Create sink stream and configuration. 47 | sink_stream = rayvens.Stream('kafka-sink-stream') 48 | sink_config = dict(kind='kafka-sink', 49 | route='/tokafka', 50 | topic=topic, 51 | brokers=broker) 52 | sink = sink_stream.add_sink(sink_config) 53 | 54 | # Create source stream and configuration. 55 | source_stream = rayvens.Stream('kafka-source-stream') 56 | source_config = dict(kind='kafka-source', 57 | route='/fromkafka', 58 | topic=topic, 59 | brokers=broker) 60 | source = source_stream.add_source(source_config) 61 | # Log all events from stream-attached sources. 62 | source_stream >> (lambda event: print('MESSAGE:', event)) 63 | 64 | # Sends message to all sinks attached to this stream. 65 | time.sleep(5) 66 | output_message = f'Sending message to Kafka sink in run mode {run_mode}.' 67 | sink_stream << output_message 68 | 69 | # Disconnect source and sink. 70 | time.sleep(5) 71 | source_stream.disconnect_all() 72 | sink_stream.disconnect_all() 73 | -------------------------------------------------------------------------------- /tests/kafka_scaling_transport.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | import sys 19 | import ray 20 | import rayvens 21 | 22 | # Initialize run mode. 23 | if len(sys.argv) < 2: 24 | run_mode = 'local' 25 | else: 26 | run_mode = sys.argv[1] 27 | 28 | if os.getenv('RAYVENS_TEST_MODE') == 'local': 29 | ray.init(object_store_memory=78643200) 30 | else: 31 | ray.init(address='auto') 32 | 33 | # The Kafka topic used for communication. 34 | topic = "testTopicPartitioned" 35 | 36 | rayvens.init(mode=run_mode, transport='kafka') 37 | 38 | # Create source stream and configuration. 39 | source_stream = rayvens.Stream('kafka-source-stream') 40 | 41 | # Event sink config. 42 | test_sink_config = dict(kind='test-sink') 43 | 44 | # Add sink to stream. 45 | test_sink = source_stream.add_sink(test_sink_config) 46 | 47 | source_config = dict( 48 | kind='http-source', 49 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 50 | route='/from-http', 51 | period=2000, 52 | kafka_transport_topic=topic, 53 | kafka_transport_partitions=3) 54 | source = source_stream.add_source(source_config) 55 | 56 | # Verify outcome. Since events are going through the Kafka infrastructure 57 | # we need to disable checks based on event counts. 58 | rayvens.meta(source_stream, 59 | 'verify_log', 60 | test_sink, 61 | "quoteResponse", 62 | wait_for_events=False) 63 | 64 | # Disconnect source and sink. 65 | source_stream.disconnect_all(after=10) 66 | -------------------------------------------------------------------------------- /tests/kafka_transport.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | import sys 19 | import ray 20 | import rayvens 21 | 22 | # Initialize run mode. 23 | if len(sys.argv) < 2: 24 | run_mode = 'local' 25 | else: 26 | run_mode = sys.argv[1] 27 | 28 | if os.getenv('RAYVENS_TEST_MODE') == 'local': 29 | ray.init(object_store_memory=78643200) 30 | else: 31 | ray.init(address='auto') 32 | 33 | # The Kafka topic used for communication. 34 | topic = "testTopic" 35 | 36 | rayvens.init(mode=run_mode, transport='kafka') 37 | 38 | # Create source stream and configuration. 39 | source_stream = rayvens.Stream('kafka-source-stream') 40 | 41 | # Event sink config. 42 | test_sink_config = dict(kind='test-sink') 43 | 44 | # Add sink to stream. 45 | test_sink = source_stream.add_sink(test_sink_config) 46 | 47 | source_config = dict( 48 | kind='http-source', 49 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 50 | route='/from-http', 51 | period=2000) 52 | source = source_stream.add_source(source_config) 53 | 54 | # Verify outcome. 55 | rayvens.meta(source_stream, 56 | 'verify_log', 57 | test_sink, 58 | "quoteResponse", 59 | wait_for_events=True) 60 | 61 | # Disconnect source and sink. 62 | source_stream.disconnect_all(after=5) 63 | -------------------------------------------------------------------------------- /tests/sink.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import ray 18 | import rayvens 19 | import os 20 | 21 | # Initialize ray based on where ray will run inside the cluster using the 22 | # kamel operator. 23 | 24 | # Default run mode. 25 | run_mode = 'operator' 26 | env_run_mode = os.getenv('RAYVENS_TEST_MODE') 27 | if env_run_mode is not None: 28 | run_mode = env_run_mode 29 | 30 | # Select appropriate Ray init method. 31 | if run_mode == 'operator': 32 | ray.init(address='auto') 33 | else: 34 | ray.init(object_store_memory=78643200) 35 | 36 | # Start the test. 37 | 38 | # Start rayvens in operator mode. 39 | rayvens.init(mode=run_mode) 40 | 41 | # Create stream. 42 | stream = rayvens.Stream('test-sink') 43 | 44 | # Event sink config. 45 | test_sink_config = dict(kind='test-sink', route='/totestsink') 46 | 47 | # Add sink to stream. 48 | sink = stream.add_sink(test_sink_config) 49 | 50 | # Sends message to all sinks attached to this stream. 51 | output_message = f'Sending message to Slack sink in run mode {run_mode}.' 52 | stream << output_message 53 | 54 | # Verify outcome. 55 | rayvens.meta(stream, 'verify_log', sink, output_message, wait_for_events=True) 56 | 57 | # Delete all integrations from stream. 58 | stream.disconnect_all(after_idle_for=5) 59 | -------------------------------------------------------------------------------- /tests/source.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import asyncio 18 | import json 19 | import os 20 | import ray 21 | import rayvens 22 | 23 | if os.getenv('RAYVENS_TEST_MODE') == 'local': 24 | ray.init(object_store_memory=78643200) 25 | else: 26 | ray.init(address='auto') 27 | 28 | rayvens.init() 29 | 30 | source_config = dict( 31 | kind='http-source', 32 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 33 | period=3000) 34 | source = rayvens.Stream('http', source_config=source_config) 35 | 36 | 37 | @ray.remote 38 | class Counter: 39 | def __init__(self): 40 | self.count = 0 41 | self.ready = asyncio.Event() 42 | 43 | def append(self, event): 44 | print( 45 | 'AAPL is', 46 | json.loads(event)['quoteResponse']['result'][0] 47 | ['regularMarketPrice']) 48 | self.count += 1 49 | if self.count > 5: 50 | self.ready.set() 51 | 52 | async def wait(self): 53 | await self.ready.wait() 54 | 55 | 56 | counter = Counter.remote() 57 | 58 | source >> counter 59 | 60 | ray.get(counter.wait.remote(), timeout=180) 61 | -------------------------------------------------------------------------------- /tests/source_operator.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import asyncio 18 | import json 19 | import ray 20 | import rayvens 21 | 22 | # Initialize ray based on where ray will run inside the cluster using the 23 | # kamel operator. 24 | ray.init(address='auto') 25 | run_mode = 'operator' 26 | 27 | 28 | # Actor class for processing the events from the source. 29 | @ray.remote 30 | class Counter: 31 | def __init__(self): 32 | self.count = 0 33 | self.ready = asyncio.Event() 34 | 35 | def append(self, event): 36 | print( 37 | 'AAPL is', 38 | json.loads(event)['quoteResponse']['result'][0] 39 | ['regularMarketPrice']) 40 | self.count += 1 41 | if self.count > 5: 42 | self.ready.set() 43 | 44 | async def wait(self): 45 | await self.ready.wait() 46 | 47 | 48 | # Start the test. 49 | 50 | # Start rayvens in the desired mode. 51 | rayvens.init(mode=run_mode) 52 | 53 | # Config for the source. 54 | source_config = dict( 55 | kind='http-source', 56 | url='https://query1.finance.yahoo.com/v7/finance/quote?symbols=AAPL', 57 | route='/test-from-http', 58 | period=3000) 59 | 60 | # Create stream where we can attach sinks, sources and operators. 61 | stream = rayvens.Stream('http') 62 | 63 | # Attach a source to the stream. 64 | source = stream.add_source(source_config) 65 | 66 | # Instantiate the processor class for the events. 67 | counter = Counter.remote() 68 | 69 | # Send all events from the source to the processor. 70 | stream >> counter 71 | 72 | ray.get(counter.wait.remote(), timeout=180) 73 | 74 | # Delete all integrations from stream. 75 | stream.disconnect_all() 76 | -------------------------------------------------------------------------------- /tests/stream.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright IBM Corporation 2021 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | import os 18 | import ray 19 | import rayvens 20 | 21 | if os.getenv('RAYVENS_TEST_MODE') == 'local': 22 | ray.init(object_store_memory=78643200) 23 | else: 24 | ray.init(address='auto') 25 | 26 | rayvens.init() 27 | 28 | stream = rayvens.Stream('example') 29 | 30 | 31 | def handler1(event): 32 | print('handler1 received', event) 33 | 34 | 35 | def handler2(event): 36 | print('handler2 received', event) 37 | 38 | 39 | stream >> handler1 40 | stream >> handler2 41 | 42 | for i in range(10): 43 | stream << f'event {i}' 44 | --------------------------------------------------------------------------------