├── beam ├── forward-beam-service.sh ├── topic-details.http ├── produce-consume.py └── values.yaml ├── forward-broker-web-service.sh ├── pulsar-sql ├── forward-pulsar-sql-service.sh ├── show-tables.http ├── PrestoExample.java └── values.yaml ├── starlight-for-kafka ├── forward-s4k-service.sh └── values.yaml ├── starlight-for-rabbitmq ├── forward-s4r-service.sh ├── test-queue.py └── values.yaml ├── install.sh ├── README.adoc └── client.conf /beam/forward-beam-service.sh: -------------------------------------------------------------------------------- 1 | kubectl port-forward -n datastax-pulsar service/pulsar-proxy 8085:8085 -------------------------------------------------------------------------------- /forward-broker-web-service.sh: -------------------------------------------------------------------------------- 1 | kubectl port-forward -n datastax-pulsar service/pulsar-broker 8080:8080 -------------------------------------------------------------------------------- /pulsar-sql/forward-pulsar-sql-service.sh: -------------------------------------------------------------------------------- 1 | kubectl port-forward -n datastax-pulsar service/pulsar-sql 8090:8090 -------------------------------------------------------------------------------- /starlight-for-kafka/forward-s4k-service.sh: -------------------------------------------------------------------------------- 1 | kubectl port-forward -n datastax-pulsar service/pulsar-proxy 9092:9092 -------------------------------------------------------------------------------- /starlight-for-rabbitmq/forward-s4r-service.sh: -------------------------------------------------------------------------------- 1 | kubectl port-forward -n datastax-pulsar service/pulsar-proxy 5672:5672 2 | -------------------------------------------------------------------------------- /pulsar-sql/show-tables.http: -------------------------------------------------------------------------------- 1 | POST http://localhost:8090/v1/statement HTTP/1.1 2 | content-type: application/json 3 | X-Presto-User: ddieruf 4 | 5 | show tables in pulsar."public/default" 6 | 7 | -------------------------------------------------------------------------------- /beam/topic-details.http: -------------------------------------------------------------------------------- 1 | # Produce message 2 | POST http://127.0.0.1:8085/v2/firehose/persistent/public/default/my-beam-topic 3 | 4 | Hi there 5 | 6 | ### 7 | 8 | # Consume message 9 | GET http://127.0.0.1:8085/v2/poll/persistent/public/default/my-beam-topic?SubscriptionType=shared&SubscriptionInitialPosition=earliest&SubscriptionName=my-subscription2 -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | # tag::helm-repo-add[] 2 | helm repo add datastax-pulsar https://datastax.github.io/pulsar-helm-chart 3 | # end::helm-repo-add[] 4 | 5 | # tag::helm-install[] 6 | helm install \ 7 | --namespace datastax-pulsar \ 8 | --create-namespace \ 9 | --values $VALUES_URL \ 10 | --version 3.0.4 \ 11 | my-pulsar-cluster \ 12 | datastax-pulsar/pulsar 13 | # end::helm-install[] 14 | 15 | # tag::wait-for-broker[] 16 | kubectl -n datastax-pulsar wait --for=condition=Ready pod/pulsar-broker-0 --timeout=120s 17 | # end::wait-for-broker[] -------------------------------------------------------------------------------- /starlight-for-rabbitmq/test-queue.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import pika 3 | 4 | connection = pika.BlockingConnection(pika.ConnectionParameters(port=5672)) 5 | channel = connection.channel() 6 | 7 | try: 8 | channel.queue_declare("test-queue") 9 | print("created test-queue queue") 10 | 11 | channel.basic_publish(exchange="", routing_key="test-queue", body="test".encode('utf-8')) 12 | print("published message test") 13 | 14 | _, _, res = channel.basic_get(queue="test-queue", auto_ack=True) 15 | assert res is not None, "should have received a message" 16 | print("received message: " + res.decode()) 17 | 18 | channel.queue_delete("test-queue") 19 | print("deleted test-queue queue") 20 | 21 | finally: 22 | connection.close() -------------------------------------------------------------------------------- /pulsar-sql/PrestoExample.java: -------------------------------------------------------------------------------- 1 | import java.sql.*; 2 | 3 | class PrestoExample { 4 | public static void main(String[] args) { 5 | try { 6 | Connection conn = DriverManager.getConnection("jdbc:presto://localhost:8090/pulsar?user=test"); 7 | System.out.println("Connection established......"); 8 | 9 | Statement stmt = conn.createStatement(); 10 | 11 | try { 12 | ResultSet rs = stmt.executeQuery("select * from pulsar.\"public/default\".mytopic limit 10"); 13 | while(rs.next()) { 14 | String str = rs.getString(1); 15 | System.out.println(String.format("%s", str)); 16 | } 17 | } 18 | finally { 19 | stmt.close(); 20 | conn.close(); 21 | } 22 | } 23 | catch (Exception e) { 24 | e.printStackTrace(); 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /beam/produce-consume.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | 4 | beamUrl = "http://127.0.0.1:8085"; 5 | 6 | tenantName = "public"; 7 | namespace = "default"; 8 | topicName = "my-beam-topic"; 9 | 10 | topic = "persistent://{0}/{1}/{2}".format(tenantName, namespace, topicName) 11 | 12 | # Produce a message 13 | response = requests.post(beamUrl + "/v2/firehose/persistent/{0}/{1}/{2}".format(tenantName, namespace, topicName), data="Hi there!") 14 | print("Published 1 message ({0})".format(response.status_code)) 15 | 16 | # Consume a message 17 | response = requests.get(beamUrl + "/v2/poll/persistent/{0}/{1}/{2}?SubscriptionType=shared&SubscriptionInitialPosition=earliest&SubscriptionName=my-beam-subscription".format(tenantName, namespace, topicName)) 18 | print("Consumed messages ({0}):\n".format(response.status_code) + json.dumps(response.json(),indent=2)) -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | [[top-of-page,[Back to top]]] 2 | = DataStax Luna Streaming Examples 3 | :toc: 4 | :toclevels: 1 5 | :toc-placement!: 6 | 7 | A collection of examples that can be used with https://docs.datastax.com/en/streaming/luna-streaming/2.10_1.x/index.html[DataStax Luna Streaming]. 8 | 9 | NOTE: There's not much explanation within these example scripts and applications. If you are wanting a little more direction, head over to the https://docs.datastax.com/en/streaming/luna-streaming/2.10_1.x/index.html[Luna Streaming docs]. There, you'll find a guided experience. 10 | 11 | toc::[] 12 | 13 | == Pulsar SQL 14 | 15 | link:./pulsar-sql/values.yaml[Values]: The bare minimum helm values needed to deploy a working cluster that includes Presto SQL workers 16 | link:./pulsar-sql/forward-pulsar-sql-service.sh[Forward Service]: Script to forward Presto commands 17 | 18 | == Pulsar Beam 19 | 20 | link:https://github.com/kafkaesque-io/pulsar-beam[Project source^] 21 | link:https://kafkaesque-io.github.io/pulsar-beam-swagger[Endpoints swagger^] 22 | 23 | link:./beam/values.yaml[Values]: The bare minimum helm values needed to deploy a working cluster that includes beam 24 | link:./beam/forward-beam-service.sh[Forward Service]: Script to forward beam requests -------------------------------------------------------------------------------- /pulsar-sql/values.yaml: -------------------------------------------------------------------------------- 1 | extra: 2 | broker: false 3 | brokerSts: true 4 | pulsarSQL: true 5 | proxy: false 6 | wsproxy: false 7 | bastion: false 8 | 9 | pulsarSQL: 10 | server: 11 | workers: 1 12 | service: 13 | type: ClusterIP 14 | 15 | zookeeper: 16 | replicaCount: 1 17 | 18 | bookkeeper: 19 | replicaCount: 1 20 | 21 | broker: 22 | component: "broker" # this has to match brokerSts naming due to a bug 23 | replicaCount: 0 24 | 25 | brokerSts: 26 | component: "broker" # this has to match broker naming due to a bug 27 | replicaCount: 1 28 | functionsWorkerEnabled: true # to overcome the need for functions working 29 | ledger: # the default assumes 2 bookies but this chart only deploys 1 30 | defaultEnsembleSize: 1 31 | defaultAckQuorum: 1 32 | defaultWriteQuorum: 1 33 | 34 | image: 35 | brokerSts: 36 | repository: datastax/lunastreaming-all #because we want connectors too 37 | pullPolicy: IfNotPresent 38 | tag: 2.10_2.4 39 | zookeeper: 40 | repository: datastax/lunastreaming 41 | pullPolicy: IfNotPresent 42 | tag: 2.10_2.4 43 | bookkeeper: 44 | repository: datastax/lunastreaming 45 | pullPolicy: IfNotPresent 46 | tag: 2.10_2.4 47 | pulsarSQL: 48 | repository: datastax/lunastreaming 49 | tag: 2.10_2.4 50 | pullPolicy: IfNotPresent -------------------------------------------------------------------------------- /client.conf: -------------------------------------------------------------------------------- 1 | # Configuration for pulsar-client and pulsar-admin CLI tools 2 | 3 | # URL for Pulsar REST API (for admin operations) 4 | # For TLS: 5 | # webServiceUrl=https://localhost:8443/ 6 | webServiceUrl=http://127.0.0.1:8080/ 7 | 8 | # URL for Pulsar Binary Protocol (for produce and consume operations) 9 | # For TLS: 10 | # brokerServiceUrl=pulsar+ssl://localhost:6651/ 11 | brokerServiceUrl=pulsar://127.0.0.1:6650/ 12 | 13 | # Authentication plugin to authenticate with servers 14 | # e.g. for TLS 15 | # authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls 16 | authPlugin= 17 | 18 | # Parameters passed to authentication plugin. 19 | # A comma separated list of key:value pairs. 20 | # Keys depend on the configured authPlugin. 21 | # e.g. for TLS 22 | # authParams=tlsCertFile:/path/to/client-cert.pem,tlsKeyFile:/path/to/client-key.pem 23 | authParams= 24 | 25 | # Allow TLS connections to servers whose certificate cannot be 26 | # be verified to have been signed by a trusted certificate 27 | # authority. 28 | tlsAllowInsecureConnection=false 29 | 30 | # Whether server hostname must match the common name of the certificate 31 | # the server is using. 32 | tlsEnableHostnameVerification=false 33 | 34 | # Path for the trusted TLS certificate file. 35 | # This cert is used to verify that any cert presented by a server 36 | # is signed by a certificate authority. If this verification 37 | # fails, then the cert is untrusted and the connection is dropped. 38 | tlsTrustCertsFilePath= 39 | 40 | # Enable TLS with KeyStore type configuration in broker. 41 | useKeyStoreTls=false 42 | 43 | # TLS KeyStore type configuration: JKS, PKCS12 44 | tlsTrustStoreType=JKS 45 | 46 | # TLS TrustStore path 47 | tlsTrustStorePath= 48 | 49 | # TLS TrustStore password 50 | tlsTrustStorePassword= -------------------------------------------------------------------------------- /beam/values.yaml: -------------------------------------------------------------------------------- 1 | extra: 2 | broker: false 3 | brokerSts: true 4 | pulsarBeam: true 5 | proxy: true 6 | wsproxy: false 7 | bastion: false 8 | 9 | zookeeper: 10 | replicaCount: 1 11 | 12 | bookkeeper: 13 | replicaCount: 1 14 | 15 | broker: 16 | component: "broker" # this has to match brokerSts naming 17 | replicaCount: 0 18 | 19 | brokerSts: 20 | component: "broker" # this has to match broker naming 21 | replicaCount: 1 22 | functionsWorkerEnabled: true # to overcome the need for functions working 23 | ledger: # the default assumes 2 bookies but this chart only deploys 1 24 | defaultEnsembleSize: 1 25 | defaultAckQuorum: 1 26 | defaultWriteQuorum: 1 27 | 28 | # this is a legacy value that the beam sub-chart uses 29 | tokenServer: 30 | allowedRoles: superuser,admin,websocket,proxy 31 | 32 | proxy: 33 | component: "proxy" 34 | replicaCount: 1 35 | service: 36 | ports: 37 | - name: "pulsarbeam" 38 | port: 8085 39 | protocol: TCP 40 | - name: "http" 41 | port: 8080 42 | protocol: TCP 43 | - name: "pulsar" 44 | port: 6650 45 | protocol: TCP 46 | 47 | pulsarBeam: 48 | component: "pulsarbeam" 49 | replicaCount: 1 50 | 51 | image: 52 | proxy: 53 | repository: datastax/lunastreaming 54 | pullPolicy: IfNotPresent 55 | tag: 2.10_2.4 56 | brokerSts: 57 | repository: datastax/lunastreaming 58 | pullPolicy: IfNotPresent 59 | tag: 2.10_2.4 60 | zookeeper: 61 | repository: datastax/lunastreaming 62 | pullPolicy: IfNotPresent 63 | tag: 2.10_2.4 64 | bookkeeper: 65 | repository: datastax/lunastreaming 66 | pullPolicy: IfNotPresent 67 | tag: 2.10_2.4 68 | pulsarBeam: 69 | repository: kesque/pulsar-beam 70 | pullPolicy: IfNotPresent 71 | tag: "1.0.0" -------------------------------------------------------------------------------- /starlight-for-rabbitmq/values.yaml: -------------------------------------------------------------------------------- 1 | extra: 2 | broker: false 3 | brokerSts: true 4 | proxy: true 5 | wsproxy: false 6 | bastion: false 7 | 8 | zookeeper: 9 | replicaCount: 1 10 | 11 | bookkeeper: 12 | replicaCount: 1 13 | 14 | broker: 15 | component: "broker" # this has to match brokerSts naming 16 | replicaCount: 0 17 | 18 | brokerSts: 19 | component: "broker" # this has to match broker naming 20 | replicaCount: 1 21 | functionsWorkerEnabled: true # to overcome the need for functions working 22 | ledger: # the default assumes 2 bookies but this chart only deploys 1 23 | defaultEnsembleSize: 1 24 | defaultAckQuorum: 1 25 | defaultWriteQuorum: 1 26 | service: 27 | ports: 28 | - name: http 29 | port: 8080 30 | - name: pulsar 31 | port: 6650 32 | 33 | proxy: 34 | component: "proxy" 35 | replicaCount: 1 36 | service: 37 | ports: 38 | - name: "http" 39 | port: 8080 40 | protocol: TCP 41 | - name: "pulsar" 42 | port: 6650 43 | protocol: TCP 44 | configData: 45 | PULSAR_PREFIX_amqpListeners: "amqp://0.0.0.0:5672" 46 | extensions: 47 | enabled: true 48 | extensions: "rabbitmq" 49 | containerPorts: 50 | - name: amqp 51 | containerPort: 5672 52 | servicePorts: 53 | - name: amqp 54 | port: 5672 55 | protocol: TCP 56 | targetPort: amqp 57 | 58 | image: 59 | proxy: 60 | repository: datastax/lunastreaming-all # so we get the starlight for rabbitmq extension 61 | pullPolicy: IfNotPresent 62 | tag: 2.10_2.4 63 | brokerSts: 64 | repository: datastax/lunastreaming 65 | pullPolicy: IfNotPresent 66 | tag: 2.10_2.4 67 | zookeeper: 68 | repository: datastax/lunastreaming 69 | pullPolicy: IfNotPresent 70 | tag: 2.10_2.4 71 | bookkeeper: 72 | repository: datastax/lunastreaming 73 | pullPolicy: IfNotPresent 74 | tag: 2.10_2.4 -------------------------------------------------------------------------------- /starlight-for-kafka/values.yaml: -------------------------------------------------------------------------------- 1 | extra: 2 | broker: false 3 | brokerSts: true 4 | proxy: true 5 | wsproxy: false 6 | bastion: false 7 | 8 | zookeeper: 9 | replicaCount: 1 10 | 11 | bookkeeper: 12 | replicaCount: 1 13 | 14 | broker: 15 | component: "broker" # this has to match brokerSts naming 16 | replicaCount: 0 17 | 18 | brokerSts: 19 | component: "broker" # this has to match broker naming 20 | replicaCount: 1 21 | functionsWorkerEnabled: true # to overcome the need for functions working 22 | ledger: # the default assumes 2 bookies but this chart only deploys 1 23 | defaultEnsembleSize: 1 24 | defaultAckQuorum: 1 25 | defaultWriteQuorum: 1 26 | service: 27 | ports: 28 | - name: http 29 | port: 8080 30 | - name: pulsar 31 | port: 6650 32 | - name: kafkaplaintext 33 | port: 9092 34 | - name: kafkaschemareg 35 | port: 8001 36 | configData: 37 | PULSAR_PREFIX_protocolHandlerDirectory: "./protocols" 38 | PULSAR_PREFIX_messagingProtocols: "kafka" 39 | PULSAR_PREFIX_brokerDeleteInactiveTopicsEnabled: "false" 40 | PULSAR_PREFIX_saslAllowedMechanisms: PLAIN 41 | PULSAR_PREFIX_brokerEntryMetadataInterceptors: "org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor,org.apache.pulsar.common.intercept.AppendBrokerTimestampMetadataInterceptor" 42 | PULSAR_PREFIX_kopSchemaRegistryEnable: "true" 43 | PULSAR_PREFIX_allowAutoTopicCreationType: "partitioned" 44 | PULSAR_PREFIX_kafkaListeners: "SASL_PLAINTEXT://0.0.0.0:9092" 45 | PULSAR_PREFIX_kafkaAdvertisedListeners: "SASL_PLAINTEXT://advertisedAddress:9092" 46 | PULSAR_PREFIX_kafkaTransactionCoordinatorEnabled: "true" 47 | 48 | proxy: 49 | component: "proxy" 50 | replicaCount: 1 51 | service: 52 | ports: 53 | - name: "s4k" 54 | port: 9095 55 | protocol: TCP 56 | - name: "http" 57 | port: 8080 58 | protocol: TCP 59 | - name: "pulsar" 60 | port: 6650 61 | protocol: TCP 62 | configData: 63 | PULSAR_PREFIX_kafkaListeners: "SASL_PLAINTEXT://0.0.0.0:9092" 64 | # here you have to customize the advertised name if you want to access 65 | # Kafka from outside the kube 66 | # Drop the SASL_PLAINTEXT endpoint if you want to allow only TLS (here and above) 67 | # The Proxy uses the same TLS certificate configured for the Pulsar protocol 68 | # PULSAR_PREFIX_kafkaAdvertisedListeners: "SASL_PLAINTEXT://pulsar-proxy:9092,SASL_SSL://pulsar-proxy:9093" 69 | # if you use port forwarding to localhost then you have to set 'localhost' 70 | PULSAR_PREFIX_kafkaAdvertisedListeners: "SASL_PLAINTEXT://localhost:9092" 71 | PULSAR_PREFIX_saslAllowedMechanisms: PLAIN 72 | PULSAR_PREFIX_kafkaProxySuperUserRole: superuser 73 | PULSAR_PREFIX_kopSchemaRegistryProxyEnableTls: "true" 74 | PULSAR_PREFIX_kopSchemaRegistryEnable: "true" 75 | PULSAR_PREFIX_kopSchemaRegistryProxyPort: "8081" 76 | PULSAR_PREFIX_kafkaTransactionCoordinatorEnabled: "true" 77 | extensions: 78 | enabled: true 79 | extensions: "kafka" 80 | containerPorts: 81 | - name: kafkaplaintext 82 | containerPort: 9092 83 | - name: kafkaschemareg 84 | containerPort: 8081 85 | servicePorts: 86 | - name: kafkaplaintext 87 | port: 9092 88 | protocol: TCP 89 | targetPort: kafkaplaintext 90 | - name: kafkaschemareg 91 | port: 8081 92 | protocol: TCP 93 | targetPort: kafkaschemareg 94 | 95 | image: 96 | proxy: 97 | repository: datastax/lunastreaming-all # so we get the starlight for kafka extension 98 | pullPolicy: IfNotPresent 99 | tag: 2.10_2.4 100 | brokerSts: 101 | repository: datastax/lunastreaming-all # so we get the starlight for kafka extension 102 | pullPolicy: IfNotPresent 103 | tag: 2.10_2.4 104 | zookeeper: 105 | repository: datastax/lunastreaming 106 | pullPolicy: IfNotPresent 107 | tag: 2.10_2.4 108 | bookkeeper: 109 | repository: datastax/lunastreaming 110 | pullPolicy: IfNotPresent 111 | tag: 2.10_2.4 --------------------------------------------------------------------------------