├── .gitignore ├── README.md ├── pom.xml └── src ├── main ├── assemblies │ └── plugin.xml ├── java │ └── org │ │ └── xbib │ │ └── elasticsearch │ │ ├── action │ │ ├── cluster │ │ │ └── admin │ │ │ │ └── websocket │ │ │ │ ├── TransportWebsocketInfoAction.java │ │ │ │ ├── WebsocketInfo.java │ │ │ │ ├── WebsocketInfoAction.java │ │ │ │ ├── WebsocketInfoRequest.java │ │ │ │ ├── WebsocketInfoRequestBuilder.java │ │ │ │ └── WebsocketInfoResponse.java │ │ └── websocket │ │ │ ├── bulk │ │ │ ├── BulkDeleteAction.java │ │ │ ├── BulkFlushAction.java │ │ │ ├── BulkHandler.java │ │ │ └── BulkIndexAction.java │ │ │ └── pubsub │ │ │ ├── Checkpointer.java │ │ │ ├── ForwardAction.java │ │ │ ├── PubSubIndexName.java │ │ │ ├── PublishAction.java │ │ │ ├── PublishSubscribe.java │ │ │ ├── SubscribeAction.java │ │ │ └── UnsubscribeAction.java │ │ ├── common │ │ ├── bytes │ │ │ ├── BytesArray.java │ │ │ ├── BytesReference.java │ │ │ ├── ChannelBufferBytesReference.java │ │ │ ├── PagedBytesReference.java │ │ │ ├── ReleasableBytesReference.java │ │ │ └── ReleasablePagedBytesReference.java │ │ ├── io │ │ │ ├── ReleasableBytesStream.java │ │ │ └── stream │ │ │ │ ├── BytesStreamOutput.java │ │ │ │ ├── CachedStreamOutput.java │ │ │ │ └── ReleasableBytesStreamOutput.java │ │ └── netty │ │ │ ├── KeepFrameDecoder.java │ │ │ ├── NettyStaticSetup.java │ │ │ ├── OpenChannelsHandler.java │ │ │ └── ReleaseChannelFutureListener.java │ │ ├── http │ │ ├── BindHttpException.java │ │ ├── HttpChannel.java │ │ ├── HttpException.java │ │ ├── HttpInfo.java │ │ ├── HttpRequest.java │ │ ├── HttpServer.java │ │ ├── HttpServerAdapter.java │ │ ├── HttpServerModule.java │ │ ├── HttpServerTransport.java │ │ ├── HttpStats.java │ │ ├── WebSocketServerAdapter.java │ │ └── netty │ │ │ ├── NettyHttpChannel.java │ │ │ ├── NettyHttpRequest.java │ │ │ ├── NettyHttpRequestHandler.java │ │ │ ├── NettyHttpServerPipelineFactory.java │ │ │ ├── NettyInteractiveChannel.java │ │ │ ├── NettyInteractiveRequest.java │ │ │ ├── NettyInteractiveResponse.java │ │ │ ├── NettyWebSocketRequestHandler.java │ │ │ ├── NettyWebSocketServerPipelineFactory.java │ │ │ ├── NettyWebSocketServerTransport.java │ │ │ ├── NettyWebSocketServerTransportModule.java │ │ │ └── client │ │ │ ├── NettyWebSocketBulkRequest.java │ │ │ ├── NettyWebSocketClient.java │ │ │ ├── NettyWebSocketClientFactory.java │ │ │ ├── NettyWebSocketClientHandler.java │ │ │ └── NettyWebSocketException.java │ │ ├── plugin │ │ └── websocket │ │ │ ├── Build.java │ │ │ ├── WebSocketModule.java │ │ │ └── WebSocketPlugin.java │ │ ├── rest │ │ ├── HttpPatchRestController.java │ │ ├── RestXContentBuilder.java │ │ ├── XContentRestResponse.java │ │ ├── XContentThrowableRestResponse.java │ │ └── action │ │ │ └── websocket │ │ │ ├── RestPublishAction.java │ │ │ └── RestUnsubscribeAction.java │ │ ├── transport │ │ └── netty │ │ │ ├── ChannelBufferStreamInput.java │ │ │ ├── ChannelBufferStreamInputFactory.java │ │ │ ├── MessageChannelHandler.java │ │ │ ├── NettyHeader.java │ │ │ ├── NettyTransport.java │ │ │ ├── NettyTransportChannel.java │ │ │ └── SizeHeaderFrameDecoder.java │ │ └── websocket │ │ ├── BaseInteractiveHandler.java │ │ ├── InteractiveActionModule.java │ │ ├── InteractiveChannel.java │ │ ├── InteractiveController.java │ │ ├── InteractiveHandler.java │ │ ├── InteractiveRequest.java │ │ ├── InteractiveResponse.java │ │ ├── Presence.java │ │ └── client │ │ ├── WebSocketActionListener.java │ │ ├── WebSocketClient.java │ │ ├── WebSocketClientBulkRequest.java │ │ ├── WebSocketClientFactory.java │ │ └── WebSocketClientRequest.java └── resources │ └── es-plugin.properties ├── site ├── resources │ ├── elasticsearch-websocket.png │ ├── publish-subscribe.png │ └── transport-modules.png └── site.xml └── test ├── java └── org │ └── xbib │ └── elasticsearch │ └── websocket │ ├── BulkTest.java │ ├── HelloWorldWebSocketTest.java │ ├── PublishSubscribeRequestTest.java │ ├── SimplePublishSubscribeTest.java │ └── helper │ ├── AbstractNodeRandomTestHelper.java │ └── AbstractNodeTestHelper.java └── resources └── log4j.properties /.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | /work 3 | /logs 4 | /.idea 5 | /target 6 | .DS_Store 7 | *.iml 8 | /.settings 9 | /.classpath 10 | /.project 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | :warning: **This repository will be removed soon without any further notice** 2 | 3 | # Elasticsearch WebSocket transport plugin 4 | 5 | Follow [@xbib](https://twitter/xbib) on Twitter 6 | 7 | > NOTE: I plan to move this plugin over to a Ratpack-based plugin: 8 | > https://github.com/jprante/elasticsearch-plugin-ratpack 9 | > If you have questions, concerns, comments, feel free to open an issue. 10 | 11 | This is an implementation of WebSockets for Elasticsearch. 12 | 13 | WebSockets are implemented as an [Elasticsearch transport plugin](http://www.elasticsearch.org/guide/reference/modules/plugins.html) 14 | using the latest implementation of WebSockets in the [Netty project](http://netty.io). 15 | 16 | The [WebSocket protocol specification](http://tools.ietf.org/html/rfc6455) defines an API that 17 | enables web clients to use the WebSockets protocol for two-way communication with a remote host. 18 | It defines a full-duplex communication channel that operates through a single socket over the Web. 19 | WebSockets provide an enormous reduction in unnecessary network traffic and latency compared to the 20 | unscalable polling and long-polling solutions that were used to simulate a full-duplex connection by 21 | maintaining two connections. WebSocket-based applications place less burden on servers, allowing 22 | existing machines to support more concurrent connections. 23 | 24 | Elasticsearch offers a HTTP REST API for nearly all the features available, so using it via `curl` 25 | or via script languages is comparable to a HTTP client connecting to a HTTP server. Some limitations 26 | apply when using the REST API and WebSockets come to the rescue. 27 | 28 | Motivations for implementing an Elasticsearch WebSocket transport layer are 29 | 30 | - to supersede the HTTP request/response model by a full-duplex communication channel 31 | 32 | - to implement scalable and responsive real-time apps, like distributed publish/subscribe services 33 | 34 | - to attach thousands of clients to an Elasticsearch node without service degradation 35 | 36 | - to allow sequences of bulk index and delete operations on a single connection 37 | 38 | - to implement new types of streaming applications like subscribing to change streams from ElasticSearch indexes 39 | 40 | 41 | ## Versions 42 | 43 | | Elasticsearch version | Plugin | Release date | 44 | | ------------------------ | ----------- | -------------| 45 | | 1.4.0 | 1.4.0.0 | Jan 8, 2015 | 46 | | 1.3.1 | 1.3.1.0 | Jul 30, 2014 | 47 | | 1.2.1 | 1.2.1.1 | Jun 9, 2014 | 48 | 49 | ## Installation 50 | 51 | ./bin/plugin -install websocket -url http://xbib.org/repository/org/xbib/elasticsearch/plugin/elasticsearch-transport-websocket/1.4.0.0/elasticsearch-transport-websocket-1.4.0.0-plugin.zip 52 | 53 | Do not forget to restart the node after installing. 54 | 55 | ## Project docs 56 | 57 | The Maven project site is available at [Github](http://jprante.github.io/elasticsearch-transport-websocket) 58 | 59 | ## Issues 60 | 61 | All feedback is welcome! If you find issues, please post them at [Github](https://github.com/jprante/elasticsearch-transport-websocket/issues) 62 | 63 | Overview 64 | -------- 65 | 66 | ![Modules](https://github.com/jprante/elasticsearch-transport-websocket/blob/master/src/site/resources/transport-modules.png?raw=true) 67 | 68 | The transport plugin uses Netty WebSockets for server and clients. WebSocket clients can connect to an 69 | Elasticsearch Node with the transport plugin installed. Between nodes, WebSockets connections may establish 70 | when needed for forwarding messages. The more nodes are installed with websocket transport, 71 | the more clients can get connected. 72 | 73 | ## WebSocket Module 74 | 75 | ![Websocket](https://github.com/jprante/elasticsearch-transport-websocket/blob/master/src/site/resources/elasticsearch-websocket.png?raw=true) 76 | 77 | The WebSocket module includes a module that allows to expose the Elasticsearch API over HTTP. It is superseding the 78 | standard HTTP module on port 9200-9299. 79 | 80 | The http mechanism is completely asynchronous in nature, meaning that there is no blocking thread waiting for a 81 | response. The benefit of using asynchronous communication for HTTP is solving the 82 | [C10k problem](http://en.wikipedia.org/wiki/C10k_problem). 83 | 84 | When possible, consider using [HTTP keep alive](http://en.wikipedia.org/wiki/Keepalive#HTTP_Keepalive) 85 | when connecting for better performance and try to get your favorite client not to do 86 | [HTTP chunking](http://en.wikipedia.org/wiki/Chunked_transfer_encoding). 87 | 88 | | Setting | Description | 89 | | -------------------------------- | -------------------------------------------------------------------------------------- | 90 | | **websocket.port** | A bind port range. Defaults to **9400-9499**. | 91 | | **websocket.max_content_length** | The max content of an HTTP request. Defaults to **100mb** | 92 | | **websocket.compression** | Support for compression when possible (with Accept-Encoding). Defaults to **false**. | 93 | | **websocket.compression_level** | Defines the compression level to use. Defaults to **6**. | 94 | 95 | Node level network settings allows to set common settings that will be shared among all network based modules (unless explicitly overridden in each module). 96 | 97 | 98 | The **network.bind_host** setting allows to control the host different network components will bind on. By default, the bind host will be **anyLocalAddress** (typically **0.0.0.0** or **::0**). 99 | 100 | 101 | The **network.publish_host** setting allows to control the host the node will publish itself within the cluster so other nodes will be able to connect to it. Of course, this can't be the **anyLocalAddress**, and by default, it will be the first non loopback address (if possible), or the local address. 102 | 103 | 104 | The **network.host** setting is a simple setting to automatically set both **network.bind_host** and **network.publish_host** to the same host value. 105 | 106 | 107 | Both settings allows to be configured with either explicit host address or host name. The settings also accept logical setting values explained in the following table: 108 | 109 | 110 | | Logical Host Setting Value | Description | 111 | | -------------------------------- | ------------------------------------------------------------------------------------------ | 112 | | **_local_** | Will be resolved to the local ip address. | 113 | | **_non_loopback_** | The first non loopback address. | 114 | | **_non_loopback:ipv4_** | The first non loopback IPv4 address. | 115 | | **_non_loopback:ipv6_** | The first non loopback IPv6 address. | 116 | | **_[networkInterface]_** | Resolves to the ip address of the provided network interface. For example **_en0_**. | 117 | | **_[networkInterface]:ipv4_** | Resolves to the ipv4 address of the provided network interface. For example **_en0:ipv4_**.| 118 | | **_[networkInterface]:ipv6_** | Resolves to the ipv6 address of the provided network interface. For example **_en0:ipv6_**.| 119 | 120 | 121 | ## TCP Settings 122 | 123 | | Setting | Description | 124 | | ------------------------------------ | ------------------------------------------------------------------------------------------------ | 125 | | **network.tcp.no_delay** | Enable or disable tcp no delay setting. Defaults to **true**. | 126 | | **network.tcp.keep_alive** | Enable or disable tcp keep alive. By default not explicitly set. | 127 | | **network.tcp.reuse_address** | Should an address be reused or not. Defaults to **true** on none windows machines. | 128 | | **network.tcp.send_buffer_size** | The size of the tcp send buffer size (in size setting format). By default not explicitly set. | 129 | | **network.tcp.receive_buffer_size** | The size of the tcp receive buffer size (in size setting format). By default not explicitly set. | 130 | 131 | ## Disable WebSocket 132 | 133 | The websocket module can be completely disabled and not started by setting **websocket.enabled** to **false**. 134 | 135 | # License 136 | 137 | Elasticsearch Websocket Transport Plugin 138 | 139 | Copyright (C) 2012 Jörg Prante 140 | 141 | Licensed under the Apache License, Version 2.0 (the "License"); 142 | you may not use this file except in compliance with the License. 143 | You may obtain a copy of the License at 144 | 145 | http://www.apache.org/licenses/LICENSE-2.0 146 | 147 | Unless required by applicable law or agreed to in writing, software 148 | distributed under the License is distributed on an "AS IS" BASIS, 149 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 150 | See the License for the specific language governing permissions and 151 | limitations 152 | -------------------------------------------------------------------------------- /src/main/assemblies/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | plugin 4 | 5 | zip 6 | 7 | false 8 | 9 | 10 | / 11 | true 12 | false 13 | compile 14 | 15 | org.xbib.elasticsearch.plugin 16 | io.netty:netty 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/cluster/admin/websocket/TransportWebsocketInfoAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.cluster.admin.websocket; 2 | 3 | import org.elasticsearch.ElasticsearchException; 4 | import org.elasticsearch.action.support.ActionFilters; 5 | import org.elasticsearch.action.support.nodes.NodeOperationRequest; 6 | import org.elasticsearch.action.support.nodes.TransportNodesOperationAction; 7 | import org.elasticsearch.cluster.ClusterName; 8 | import org.elasticsearch.cluster.ClusterService; 9 | import org.elasticsearch.common.inject.Inject; 10 | import org.elasticsearch.common.io.stream.StreamInput; 11 | import org.elasticsearch.common.io.stream.StreamOutput; 12 | import org.elasticsearch.common.settings.Settings; 13 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 14 | import org.elasticsearch.discovery.Discovery; 15 | import org.elasticsearch.threadpool.ThreadPool; 16 | import org.elasticsearch.transport.TransportService; 17 | import org.xbib.elasticsearch.http.HttpServer; 18 | 19 | import java.io.IOException; 20 | import java.util.List; 21 | import java.util.concurrent.atomic.AtomicReferenceArray; 22 | 23 | import static org.elasticsearch.common.collect.Lists.newLinkedList; 24 | 25 | public class TransportWebsocketInfoAction extends TransportNodesOperationAction { 26 | 27 | private final Discovery discovery; 28 | 29 | private final HttpServer httpServer; 30 | 31 | @Inject 32 | public TransportWebsocketInfoAction(Settings settings, ClusterName clusterName, ThreadPool threadPool, 33 | ClusterService clusterService, TransportService transportService, 34 | Discovery discovery, HttpServer httpServer, ActionFilters actionFilters) { 35 | super(settings, WebsocketInfoAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters); 36 | this.discovery = discovery; 37 | this.httpServer = httpServer; 38 | } 39 | 40 | @Override 41 | protected String executor() { 42 | return ThreadPool.Names.MANAGEMENT; 43 | } 44 | 45 | @Override 46 | protected WebsocketInfoResponse newResponse(WebsocketInfoRequest nodesInfoRequest, AtomicReferenceArray responses) { 47 | final List nodesInfos = newLinkedList(); 48 | for (int i = 0; i < responses.length(); i++) { 49 | Object resp = responses.get(i); 50 | if (resp instanceof WebsocketInfo) { 51 | nodesInfos.add((WebsocketInfo) resp); 52 | } 53 | } 54 | return new WebsocketInfoResponse(clusterName, nodesInfos); 55 | } 56 | 57 | @Override 58 | protected WebsocketInfoRequest newRequest() { 59 | return new WebsocketInfoRequest(); 60 | } 61 | 62 | @Override 63 | protected TransportWebsocketInfoRequest newNodeRequest() { 64 | return new TransportWebsocketInfoRequest(); 65 | } 66 | 67 | @Override 68 | protected TransportWebsocketInfoRequest newNodeRequest(String nodeId, WebsocketInfoRequest request) { 69 | return new TransportWebsocketInfoRequest(nodeId, request); 70 | } 71 | 72 | @Override 73 | protected WebsocketInfo newNodeResponse() { 74 | return new WebsocketInfo(); 75 | } 76 | 77 | @Override 78 | protected WebsocketInfo nodeOperation(TransportWebsocketInfoRequest nodeRequest) throws ElasticsearchException { 79 | return new WebsocketInfo(discovery.localNode(), (InetSocketTransportAddress)httpServer.address()); 80 | } 81 | 82 | @Override 83 | protected boolean accumulateExceptions() { 84 | return false; 85 | } 86 | 87 | static class TransportWebsocketInfoRequest extends NodeOperationRequest { 88 | 89 | WebsocketInfoRequest request; 90 | 91 | TransportWebsocketInfoRequest() { 92 | } 93 | 94 | TransportWebsocketInfoRequest(String nodeId, WebsocketInfoRequest request) { 95 | super(request, nodeId); 96 | this.request = request; 97 | } 98 | 99 | @Override 100 | public void readFrom(StreamInput in) throws IOException { 101 | super.readFrom(in); 102 | request = new WebsocketInfoRequest(); 103 | request.readFrom(in); 104 | } 105 | 106 | @Override 107 | public void writeTo(StreamOutput out) throws IOException { 108 | super.writeTo(out); 109 | request.writeTo(out); 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/cluster/admin/websocket/WebsocketInfo.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.cluster.admin.websocket; 2 | 3 | import org.elasticsearch.action.support.nodes.NodeOperationResponse; 4 | import org.elasticsearch.cluster.node.DiscoveryNode; 5 | import org.elasticsearch.common.io.stream.StreamInput; 6 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 7 | 8 | import java.io.IOException; 9 | 10 | public class WebsocketInfo extends NodeOperationResponse { 11 | 12 | private InetSocketTransportAddress address; 13 | 14 | WebsocketInfo() { 15 | } 16 | 17 | public WebsocketInfo(DiscoveryNode node, InetSocketTransportAddress address) { 18 | super(node); 19 | this.address = address; 20 | } 21 | 22 | public InetSocketTransportAddress getAddress() { 23 | return address; 24 | } 25 | 26 | public static WebsocketInfo readInfo(StreamInput in) throws IOException { 27 | WebsocketInfo info = new WebsocketInfo(); 28 | info.readFrom(in); 29 | return info; 30 | } 31 | 32 | @Override 33 | public void readFrom(StreamInput in) throws IOException { 34 | super.readFrom(in); 35 | address = InetSocketTransportAddress.readInetSocketTransportAddress(in); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/cluster/admin/websocket/WebsocketInfoAction.java: -------------------------------------------------------------------------------- 1 | 2 | package org.xbib.elasticsearch.action.cluster.admin.websocket; 3 | 4 | import org.elasticsearch.action.admin.cluster.ClusterAction; 5 | import org.elasticsearch.client.ClusterAdminClient; 6 | 7 | /** 8 | */ 9 | public class WebsocketInfoAction extends ClusterAction { 10 | 11 | public static final WebsocketInfoAction INSTANCE = new WebsocketInfoAction(); 12 | public static final String NAME = "org.xbib.elasticsearch.action.websocket.info"; 13 | 14 | private WebsocketInfoAction() { 15 | super(NAME); 16 | } 17 | 18 | @Override 19 | public WebsocketInfoResponse newResponse() { 20 | return new WebsocketInfoResponse(); 21 | } 22 | 23 | @Override 24 | public WebsocketInfoRequestBuilder newRequestBuilder(ClusterAdminClient client) { 25 | return new WebsocketInfoRequestBuilder(client); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/cluster/admin/websocket/WebsocketInfoRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.cluster.admin.websocket; 2 | 3 | import org.elasticsearch.action.support.nodes.NodesOperationRequest; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.elasticsearch.common.io.stream.StreamOutput; 6 | 7 | import java.io.IOException; 8 | 9 | public class WebsocketInfoRequest extends NodesOperationRequest { 10 | 11 | public WebsocketInfoRequest() { 12 | } 13 | 14 | public WebsocketInfoRequest(String... nodesIds) { 15 | super(nodesIds); 16 | } 17 | 18 | @Override 19 | public void readFrom(StreamInput in) throws IOException { 20 | super.readFrom(in); 21 | } 22 | 23 | @Override 24 | public void writeTo(StreamOutput out) throws IOException { 25 | super.writeTo(out); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/cluster/admin/websocket/WebsocketInfoRequestBuilder.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.cluster.admin.websocket; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; 5 | import org.elasticsearch.client.ClusterAdminClient; 6 | 7 | public class WebsocketInfoRequestBuilder extends NodesOperationRequestBuilder { 8 | 9 | public WebsocketInfoRequestBuilder(ClusterAdminClient clusterClient) { 10 | super(clusterClient, new WebsocketInfoRequest()); 11 | } 12 | 13 | @Override 14 | protected void doExecute(ActionListener listener) { 15 | client.execute(WebsocketInfoAction.INSTANCE, request, listener); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/cluster/admin/websocket/WebsocketInfoResponse.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.cluster.admin.websocket; 2 | 3 | import org.elasticsearch.action.support.nodes.NodesOperationResponse; 4 | import org.elasticsearch.cluster.ClusterName; 5 | import org.elasticsearch.common.io.stream.StreamInput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | import org.elasticsearch.common.xcontent.ToXContent; 8 | import org.elasticsearch.common.xcontent.XContentBuilder; 9 | import org.elasticsearch.common.xcontent.XContentFactory; 10 | 11 | import java.io.IOException; 12 | import java.util.List; 13 | 14 | public class WebsocketInfoResponse extends NodesOperationResponse implements ToXContent { 15 | 16 | public WebsocketInfoResponse() { 17 | } 18 | 19 | public WebsocketInfoResponse(ClusterName clusterName, List nodes) { 20 | super(clusterName, nodes.toArray(new WebsocketInfo[nodes.size()])); 21 | } 22 | 23 | @Override 24 | public void readFrom(StreamInput in) throws IOException { 25 | super.readFrom(in); 26 | nodes = new WebsocketInfo[in.readVInt()]; 27 | for (int i = 0; i < nodes.length; i++) { 28 | nodes[i] = WebsocketInfo.readInfo(in); 29 | } 30 | } 31 | 32 | @Override 33 | public void writeTo(StreamOutput out) throws IOException { 34 | super.writeTo(out); 35 | out.writeVInt(nodes.length); 36 | for (WebsocketInfo node : nodes) { 37 | node.writeTo(out); 38 | } 39 | } 40 | 41 | @Override 42 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 43 | builder.field("cluster_name", getClusterName().value(), XContentBuilder.FieldCaseConversion.NONE); 44 | 45 | builder.startObject("nodes"); 46 | for (WebsocketInfo nodeInfo : this) { 47 | builder.startObject(nodeInfo.getNode().id(), XContentBuilder.FieldCaseConversion.NONE); 48 | builder.field("name", nodeInfo.getNode().name(), XContentBuilder.FieldCaseConversion.NONE); 49 | builder.field("transport_address", nodeInfo.getNode().address().toString()); 50 | builder.field("host", nodeInfo.getNode().getHostName(), XContentBuilder.FieldCaseConversion.NONE); 51 | builder.field("ip", nodeInfo.getNode().getHostAddress(), XContentBuilder.FieldCaseConversion.NONE); 52 | builder.field("websocket_address", nodeInfo.getAddress(), XContentBuilder.FieldCaseConversion.NONE); 53 | builder.endObject(); 54 | } 55 | builder.endObject(); 56 | return builder; 57 | } 58 | 59 | @Override 60 | public String toString() { 61 | try { 62 | XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); 63 | builder.startObject(); 64 | toXContent(builder, EMPTY_PARAMS); 65 | builder.endObject(); 66 | return builder.string(); 67 | } catch (IOException e) { 68 | return "{ \"error\" : \"" + e.getMessage() + "\"}"; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/websocket/bulk/BulkDeleteAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.websocket.bulk; 2 | 3 | import org.elasticsearch.action.delete.DeleteRequest; 4 | import org.elasticsearch.client.Client; 5 | import org.elasticsearch.client.Requests; 6 | import org.elasticsearch.common.inject.Inject; 7 | import org.elasticsearch.common.settings.Settings; 8 | import org.xbib.elasticsearch.websocket.InteractiveChannel; 9 | import org.xbib.elasticsearch.websocket.InteractiveController; 10 | import org.xbib.elasticsearch.websocket.InteractiveRequest; 11 | 12 | import java.io.IOException; 13 | 14 | /** 15 | * Bulk delete action 16 | */ 17 | public class BulkDeleteAction extends BulkHandler { 18 | 19 | private final static String TYPE = "delete"; 20 | 21 | @Inject 22 | public BulkDeleteAction(Settings settings, Client client, InteractiveController controller) { 23 | super(settings, client); 24 | controller.registerHandler(TYPE, this); 25 | } 26 | 27 | @Override 28 | public void handleRequest(final InteractiveRequest request, final InteractiveChannel channel) { 29 | String index = request.paramAsString("index"); 30 | String type = request.paramAsString("type"); 31 | String id = request.paramAsString("id"); 32 | try { 33 | if (index == null) { 34 | channel.sendResponse(TYPE, new IllegalArgumentException("index is null")); 35 | return; 36 | } 37 | if (type == null) { 38 | channel.sendResponse(TYPE, new IllegalArgumentException("type is null")); 39 | return; 40 | } 41 | if (id == null) { 42 | channel.sendResponse(TYPE, new IllegalArgumentException("id is null")); 43 | return; 44 | } 45 | DeleteRequest deleteRequest = Requests.deleteRequest(index).type(type).id(id); 46 | add(deleteRequest, channel); 47 | } catch (IOException ex) { 48 | try { 49 | channel.sendResponse(TYPE, ex); 50 | } catch (IOException ex1) { 51 | logger.error("error while sending exception"); 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/websocket/bulk/BulkFlushAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.websocket.bulk; 2 | 3 | import org.elasticsearch.client.Client; 4 | import org.elasticsearch.common.inject.Inject; 5 | import org.elasticsearch.common.settings.Settings; 6 | import org.xbib.elasticsearch.websocket.InteractiveChannel; 7 | import org.xbib.elasticsearch.websocket.InteractiveController; 8 | import org.xbib.elasticsearch.websocket.InteractiveRequest; 9 | 10 | /** 11 | * Bulk flush action. This action forces bulk requests to get 12 | * sent to the cluster. 13 | */ 14 | public class BulkFlushAction extends BulkHandler { 15 | 16 | private final static String TYPE = "flush"; 17 | 18 | @Inject 19 | public BulkFlushAction(Settings settings, Client client, InteractiveController controller) { 20 | super(settings, client); 21 | controller.registerHandler(TYPE, this); 22 | } 23 | 24 | @Override 25 | public void handleRequest(final InteractiveRequest request, final InteractiveChannel channel) { 26 | this.flush(channel); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/websocket/bulk/BulkIndexAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.websocket.bulk; 2 | 3 | import org.elasticsearch.action.index.IndexRequest; 4 | import org.elasticsearch.client.Client; 5 | import org.elasticsearch.client.Requests; 6 | import org.elasticsearch.common.inject.Inject; 7 | import org.elasticsearch.common.settings.Settings; 8 | import org.xbib.elasticsearch.websocket.InteractiveChannel; 9 | import org.xbib.elasticsearch.websocket.InteractiveController; 10 | import org.xbib.elasticsearch.websocket.InteractiveRequest; 11 | 12 | import java.io.IOException; 13 | import java.util.Map; 14 | 15 | public class BulkIndexAction extends BulkHandler { 16 | 17 | private final static String TYPE = "index"; 18 | 19 | @Inject 20 | public BulkIndexAction(Settings settings, Client client, InteractiveController controller) { 21 | super(settings, client); 22 | controller.registerHandler(TYPE, this); 23 | } 24 | 25 | @Override 26 | public void handleRequest(final InteractiveRequest request, final InteractiveChannel channel) { 27 | String index = request.paramAsString("index"); 28 | String type = request.paramAsString("type"); 29 | String id = request.paramAsString("id"); 30 | try { 31 | if (index == null) { 32 | channel.sendResponse(TYPE, new IllegalArgumentException("index is null")); 33 | return; 34 | } 35 | if (type == null) { 36 | channel.sendResponse(TYPE, new IllegalArgumentException("type is null")); 37 | return; 38 | } 39 | if (id == null) { 40 | channel.sendResponse(TYPE, new IllegalArgumentException("id is null")); 41 | return; 42 | } 43 | IndexRequest indexRequest = Requests.indexRequest(index).type(type).id(id) 44 | .source((Map) request.asMap().get("data")); 45 | add(indexRequest, channel); 46 | } catch (IOException ex) { 47 | try { 48 | channel.sendResponse(TYPE, ex); 49 | } catch (IOException ex1) { 50 | logger.error("error while sending exception"); 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/websocket/pubsub/Checkpointer.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.websocket.pubsub; 2 | 3 | import org.elasticsearch.ElasticsearchException; 4 | import org.elasticsearch.action.delete.DeleteRequest; 5 | import org.elasticsearch.action.get.GetResponse; 6 | import org.elasticsearch.action.index.IndexRequest; 7 | import org.elasticsearch.client.Client; 8 | import org.elasticsearch.client.Requests; 9 | import org.elasticsearch.common.component.AbstractLifecycleComponent; 10 | import org.elasticsearch.common.inject.Inject; 11 | import org.elasticsearch.common.logging.ESLogger; 12 | import org.elasticsearch.common.logging.ESLoggerFactory; 13 | import org.elasticsearch.common.settings.Settings; 14 | import org.xbib.elasticsearch.websocket.InteractiveChannel; 15 | import org.xbib.elasticsearch.action.websocket.bulk.BulkHandler; 16 | 17 | import java.io.IOException; 18 | 19 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 20 | 21 | /* 22 | * Checkpoint management for topics and subscribers. 23 | */ 24 | public class Checkpointer extends AbstractLifecycleComponent { 25 | 26 | private final ESLogger logger = ESLoggerFactory.getLogger(Checkpointer.class.getSimpleName()); 27 | 28 | private final String pubSubIndexName; 29 | 30 | private final Client client; 31 | 32 | private final static String TYPE = "checkpoint"; 33 | 34 | private final BulkHandler bulkHandler; 35 | 36 | @Inject 37 | public Checkpointer(Settings settings, Client client) { 38 | super(settings); 39 | this.client = client; 40 | this.bulkHandler = new BulkHandler(settings, client); 41 | this.pubSubIndexName = PubSubIndexName.Conf.indexName(settings); 42 | } 43 | 44 | @Override 45 | protected void doStart() throws ElasticsearchException { 46 | } 47 | 48 | @Override 49 | protected void doStop() throws ElasticsearchException { 50 | } 51 | 52 | @Override 53 | protected void doClose() throws ElasticsearchException { 54 | } 55 | 56 | /** 57 | * Checkpointing a topic or a subscriber. The current timestamp is written 58 | * to the checkpoint index type. Note that bulk index requests are used by 59 | * checkpointing and flushCheckpoint() needs to be called after all is done. 60 | * 61 | * @param id topic or subscriber 62 | * @throws IOException if this method fails 63 | */ 64 | public void checkpoint(String id) throws IOException { 65 | indexBulk(Requests.indexRequest(pubSubIndexName).type(TYPE).id(id) 66 | .source(jsonBuilder().startObject().field("timestamp", System.currentTimeMillis()).endObject()), null); 67 | } 68 | 69 | public void flushCheckpoint() throws IOException { 70 | flushBulk(null); 71 | } 72 | 73 | public Long checkpointedAt(String id) throws IOException { 74 | GetResponse response = client.prepareGet(pubSubIndexName, TYPE, id) 75 | .setFields("timestamp") 76 | .execute().actionGet(); 77 | boolean failed = !response.isExists(); 78 | if (failed) { 79 | logger.warn("can't get checkpoint for {}", id); 80 | return null; 81 | } else { 82 | return (Long) response.getFields().get("timestamp").getValue(); 83 | } 84 | } 85 | 86 | /** 87 | * Perform bulk indexing 88 | * 89 | * @param request the index request 90 | * @param channel the interactive channel 91 | * @throws IOException if this method fails 92 | */ 93 | public void indexBulk(IndexRequest request, InteractiveChannel channel) throws IOException { 94 | bulkHandler.add(request, channel); 95 | } 96 | 97 | /** 98 | * Perform bulk delete 99 | * 100 | * @param request the delete request 101 | * @param channel the interactive channel 102 | * @throws IOException if this method fails 103 | */ 104 | public void deleteBulk(DeleteRequest request, InteractiveChannel channel) throws IOException { 105 | bulkHandler.add(request, channel); 106 | } 107 | 108 | /** 109 | * Flush bulk 110 | * 111 | * @param channel the interactive channel 112 | * @throws IOException if this method fails 113 | */ 114 | public void flushBulk(InteractiveChannel channel) throws IOException { 115 | bulkHandler.flush(); 116 | } 117 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/websocket/pubsub/ForwardAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.websocket.pubsub; 2 | 3 | import org.elasticsearch.client.Client; 4 | import org.elasticsearch.common.inject.Inject; 5 | import org.elasticsearch.common.settings.Settings; 6 | import org.jboss.netty.channel.Channel; 7 | import org.xbib.elasticsearch.websocket.BaseInteractiveHandler; 8 | import org.xbib.elasticsearch.websocket.InteractiveChannel; 9 | import org.xbib.elasticsearch.websocket.InteractiveController; 10 | import org.xbib.elasticsearch.websocket.InteractiveRequest; 11 | import org.xbib.elasticsearch.http.HttpServerTransport; 12 | import org.xbib.elasticsearch.http.netty.NettyInteractiveResponse; 13 | 14 | import java.io.IOException; 15 | import java.util.Map; 16 | 17 | /** 18 | * Forwarding a message to a destination. 19 | */ 20 | public class ForwardAction extends BaseInteractiveHandler { 21 | 22 | private final String TYPE = "forward"; 23 | private final HttpServerTransport transport; 24 | 25 | @Inject 26 | public ForwardAction(Settings settings, 27 | Client client, 28 | HttpServerTransport transport, 29 | InteractiveController controller) { 30 | super(settings, client); 31 | this.transport = transport; 32 | controller.registerHandler(TYPE, this); 33 | } 34 | 35 | @Override 36 | public void handleRequest(final InteractiveRequest request, final InteractiveChannel channel) { 37 | Map m = request.asMap(); 38 | Integer id = (Integer) m.get("channel"); 39 | Channel ch = transport.channel(id); 40 | try { 41 | if (ch != null) { 42 | ch.write(new NettyInteractiveResponse("message", (Map) m.get("message")).response()); 43 | // don't send a success message back to the channel 44 | } else { 45 | // delivery failed, channel not present 46 | channel.sendResponse(TYPE, new IOException("channel " + id + " gone")); 47 | } 48 | } catch (IOException ex) { 49 | logger.error("error while delivering forward message {}: {}", m, ex); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/websocket/pubsub/PubSubIndexName.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.websocket.pubsub; 2 | 3 | import org.elasticsearch.common.inject.BindingAnnotation; 4 | import org.elasticsearch.common.settings.Settings; 5 | 6 | import java.lang.annotation.Documented; 7 | import java.lang.annotation.Retention; 8 | import java.lang.annotation.Target; 9 | 10 | import static java.lang.annotation.ElementType.FIELD; 11 | import static java.lang.annotation.ElementType.PARAMETER; 12 | import static java.lang.annotation.RetentionPolicy.RUNTIME; 13 | 14 | /** 15 | * Configuration for the pubsub index name. 16 | */ 17 | @BindingAnnotation 18 | @Target({FIELD, PARAMETER}) 19 | @Retention(RUNTIME) 20 | @Documented 21 | public @interface PubSubIndexName { 22 | 23 | static class Conf { 24 | public static final String DEFAULT_INDEX_NAME = "pubsub"; 25 | 26 | public static String indexName(Settings settings) { 27 | return settings.get("pubsub.index_name", DEFAULT_INDEX_NAME); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/websocket/pubsub/PublishAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.websocket.pubsub; 2 | 3 | import org.elasticsearch.action.index.IndexResponse; 4 | import org.elasticsearch.action.search.SearchResponse; 5 | import org.elasticsearch.action.search.SearchType; 6 | import org.elasticsearch.client.Client; 7 | import org.elasticsearch.common.inject.Inject; 8 | import org.elasticsearch.common.settings.Settings; 9 | import org.elasticsearch.common.unit.TimeValue; 10 | import org.elasticsearch.common.xcontent.XContentBuilder; 11 | import org.elasticsearch.index.query.QueryBuilder; 12 | import org.elasticsearch.search.SearchHit; 13 | import org.elasticsearch.search.SearchHitField; 14 | import org.xbib.elasticsearch.websocket.InteractiveChannel; 15 | import org.xbib.elasticsearch.websocket.InteractiveController; 16 | import org.xbib.elasticsearch.websocket.InteractiveRequest; 17 | import org.xbib.elasticsearch.http.HttpServerTransport; 18 | 19 | import java.io.IOException; 20 | import java.util.Map; 21 | 22 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 23 | import static org.elasticsearch.index.query.QueryBuilders.termQuery; 24 | 25 | /** 26 | * Publish action 27 | */ 28 | public class PublishAction extends PublishSubscribe { 29 | 30 | protected final static String TYPE = "publish"; 31 | 32 | @Inject 33 | public PublishAction(Settings settings, 34 | Client client, 35 | HttpServerTransport transport, 36 | InteractiveController controller, 37 | Checkpointer service) { 38 | super(settings, client, transport, controller, service); 39 | controller.registerHandler(TYPE, this); 40 | } 41 | 42 | @Override 43 | public void handleRequest(final InteractiveRequest request, final InteractiveChannel channel) { 44 | String topic = request.hasParam("topic") ? request.paramAsString("topic") : "*"; 45 | try { 46 | // advertise phase - save message in the index (for disconnected subscribers) 47 | final XContentBuilder messageBuilder = createPublishMessage(request); 48 | final XContentBuilder responseBuilder = jsonBuilder().startObject(); 49 | IndexResponse indexResponse = client.prepareIndex() 50 | .setIndex(pubSubIndexName) 51 | .setType(TYPE) 52 | .setSource(messageBuilder) 53 | .setRefresh(request.paramAsBoolean("refresh", true)) 54 | .execute().actionGet(); 55 | responseBuilder.field("id", indexResponse.getId()); 56 | // push phase - scroll over subscribers for this topic currently connected 57 | QueryBuilder queryBuilder = termQuery("topic", topic); 58 | SearchResponse searchResponse = client.prepareSearch() 59 | .setIndices(pubSubIndexName) 60 | .setTypes("subscribe") 61 | .setSearchType(SearchType.SCAN) 62 | .setScroll(new TimeValue(60000)) 63 | .setQuery(queryBuilder) 64 | .addField("subscriber.channel") 65 | .setSize(100) 66 | .execute().actionGet(); 67 | boolean failed = searchResponse.getFailedShards() > 0 || searchResponse.isTimedOut(); 68 | if (failed) { 69 | logger.error("searching for subscribers for topic {} failed: failed shards={} timeout={}", 70 | topic, searchResponse.getFailedShards(), searchResponse.isTimedOut()); 71 | responseBuilder.field("subscribers", 0).field("failed", true); 72 | channel.sendResponse(TYPE, responseBuilder.endObject()); 73 | responseBuilder.close(); 74 | return; 75 | } 76 | // look for subscribers 77 | long totalHits = searchResponse.getHits().getTotalHits(); 78 | boolean zero = totalHits == 0L; 79 | if (zero) { 80 | responseBuilder.field("subscribers", 0).field("failed", false); 81 | channel.sendResponse(TYPE, responseBuilder.endObject()); 82 | responseBuilder.close(); 83 | return; 84 | } 85 | // report the total number of subscribers online to the publisher 86 | responseBuilder.field("subscribers", totalHits); 87 | channel.sendResponse(TYPE, responseBuilder.endObject()); 88 | messageBuilder.close(); 89 | responseBuilder.close(); 90 | // checkpoint topic 91 | service.checkpoint(topic); 92 | // push phase - write the message to the subscribers. We have 60 seconds per 100 subscribers. 93 | while (true) { 94 | searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()) 95 | .setScroll(new TimeValue(60000)) 96 | .execute().actionGet(); 97 | for (SearchHit hit : searchResponse.getHits()) { 98 | // for message sync - update all subscribers with the current timestamp 99 | service.checkpoint(hit.getId()); 100 | // find node address and channel ID 101 | SearchHitField channelField = hit.field("subscriber.channel"); 102 | Map channelfieldMap = channelField.getValue(); 103 | String nodeAddress = (String) channelfieldMap.get("localAddress"); 104 | Integer id = (Integer) channelfieldMap.get("id"); 105 | // forward to node 106 | transport.forward(nodeAddress, id, messageBuilder); 107 | } 108 | if (searchResponse.getHits().hits().length == 0) { 109 | break; 110 | } 111 | } 112 | service.flushCheckpoint(); 113 | } catch (Exception e) { 114 | logger.error("exception while processing publish request", e); 115 | try { 116 | channel.sendResponse(TYPE, e); 117 | } catch (IOException e1) { 118 | logger.error("exception while sending exception response", e1); 119 | } 120 | } 121 | } 122 | 123 | private XContentBuilder createPublishMessage(InteractiveRequest request) { 124 | try { 125 | return jsonBuilder().startObject() 126 | .field("timestamp", request.paramAsLong("timestamp", System.currentTimeMillis())) 127 | .field("data", request.asMap()) 128 | .endObject(); 129 | } catch (IOException e) { 130 | return null; 131 | } 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/websocket/pubsub/PublishSubscribe.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.websocket.pubsub; 2 | 3 | import org.elasticsearch.client.Client; 4 | import org.elasticsearch.common.settings.Settings; 5 | import org.elasticsearch.common.unit.TimeValue; 6 | import org.xbib.elasticsearch.websocket.BaseInteractiveHandler; 7 | import org.xbib.elasticsearch.websocket.InteractiveController; 8 | import org.xbib.elasticsearch.http.HttpServerTransport; 9 | 10 | /** 11 | * Base class for Publish/Subscribe actions 12 | */ 13 | public abstract class PublishSubscribe extends BaseInteractiveHandler { 14 | 15 | protected final String pubSubIndexName; 16 | 17 | protected final HttpServerTransport transport; 18 | 19 | protected final Checkpointer service; 20 | 21 | protected final TimeValue scrollTimeout = new TimeValue(60000); 22 | 23 | protected final int scrollSize = 100; 24 | 25 | public PublishSubscribe(Settings settings, 26 | Client client, 27 | HttpServerTransport transport, 28 | InteractiveController controller, 29 | Checkpointer service) { 30 | super(settings, client); 31 | this.pubSubIndexName = PubSubIndexName.Conf.indexName(settings); 32 | this.transport = transport; 33 | this.service = service; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/websocket/pubsub/SubscribeAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.websocket.pubsub; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.index.IndexResponse; 5 | import org.elasticsearch.action.search.SearchResponse; 6 | import org.elasticsearch.action.search.SearchType; 7 | import org.elasticsearch.client.Client; 8 | import org.elasticsearch.common.inject.Inject; 9 | import org.elasticsearch.common.settings.Settings; 10 | import org.elasticsearch.common.xcontent.XContentBuilder; 11 | import org.elasticsearch.index.query.QueryBuilder; 12 | import org.elasticsearch.index.query.RangeFilterBuilder; 13 | import org.elasticsearch.search.SearchHit; 14 | import org.jboss.netty.channel.Channel; 15 | import org.xbib.elasticsearch.websocket.InteractiveChannel; 16 | import org.xbib.elasticsearch.websocket.InteractiveController; 17 | import org.xbib.elasticsearch.websocket.InteractiveRequest; 18 | import org.xbib.elasticsearch.http.HttpServerTransport; 19 | import org.xbib.elasticsearch.http.netty.NettyInteractiveResponse; 20 | 21 | import java.io.IOException; 22 | import java.util.Map; 23 | 24 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 25 | import static org.elasticsearch.index.query.FilterBuilders.rangeFilter; 26 | import static org.elasticsearch.index.query.QueryBuilders.termQuery; 27 | 28 | /** 29 | * Subscribe action. It performs the subscription of a client to 30 | * the pubsub index under a given topic. 31 | */ 32 | public class SubscribeAction extends PublishSubscribe { 33 | 34 | protected final static String TYPE = "subscribe"; 35 | 36 | @Inject 37 | public SubscribeAction(Settings settings, 38 | Client client, 39 | HttpServerTransport transport, 40 | InteractiveController controller, 41 | Checkpointer service) { 42 | super(settings, client, transport, controller, service); 43 | controller.registerHandler(TYPE, this); 44 | } 45 | 46 | @Override 47 | public void handleRequest(final InteractiveRequest request, final InteractiveChannel channel) { 48 | final String topic = request.hasParam("topic") ? request.paramAsString("topic") : "*"; 49 | final String subscriberId = request.hasParam("subscriber") ? request.paramAsString("subscriber") : null; 50 | if (subscriberId == null) { 51 | try { 52 | channel.sendResponse(TYPE, new IllegalArgumentException("no subscriber")); 53 | } catch (IOException e) { 54 | logger.error("error while sending failure response", e); 55 | } 56 | } 57 | try { 58 | client.prepareIndex() 59 | .setIndex(pubSubIndexName) 60 | .setType(TYPE) 61 | .setId(subscriberId) 62 | .setSource(createSubscriberMessage(topic, channel)) 63 | .setRefresh(request.paramAsBoolean("refresh", true)) 64 | .execute(new ActionListener() { 65 | @Override 66 | public void onResponse(IndexResponse response) { 67 | try { 68 | XContentBuilder builder = jsonBuilder(); 69 | builder.startObject().field("ok", true).field("id", response.getId()).endObject(); 70 | channel.sendResponse(TYPE, builder); 71 | // receive outstanding messages 72 | sync(subscriberId, topic, channel.getChannel()); 73 | } catch (Exception e) { 74 | onFailure(e); 75 | } 76 | } 77 | 78 | @Override 79 | public void onFailure(Throwable e) { 80 | logger.error("error while processing subscribe request", e); 81 | try { 82 | channel.sendResponse(TYPE, e); 83 | } catch (IOException ex) { 84 | logger.error("error while sending error response", ex); 85 | } 86 | } 87 | }); 88 | } catch (Exception e) { 89 | logger.error("exception while processing subscribe request", e); 90 | try { 91 | channel.sendResponse(TYPE, e); 92 | } catch (IOException e1) { 93 | logger.error("exception while sending exception response", e1); 94 | } 95 | } 96 | } 97 | 98 | private XContentBuilder createSubscriberMessage(String topic, InteractiveChannel channel) { 99 | Integer channelId = channel.getChannel().getId(); 100 | String localAddress = channel.getChannel().getLocalAddress().toString(); 101 | String remoteAddress = channel.getChannel().getRemoteAddress().toString(); 102 | try { 103 | return jsonBuilder() 104 | .startObject() 105 | .field("topic", topic) 106 | .startObject("subscriber") 107 | .startObject("channel") 108 | .field("id", channelId) 109 | .field("localAddress", localAddress) 110 | .field("remoteAddress", remoteAddress) 111 | .endObject() 112 | .endObject() 113 | .endObject(); 114 | } catch (IOException e) { 115 | return null; 116 | } 117 | } 118 | 119 | /** 120 | * Synchronize the subscriber with the current messages. 121 | * 122 | * @param subscriberId 123 | * @param topic 124 | * @param channel 125 | * @throws IOException 126 | */ 127 | private void sync(final String subscriberId, final String topic, final Channel channel) throws IOException { 128 | Long lastSeen = service.checkpointedAt(subscriberId); 129 | Long topicSeen = service.checkpointedAt(topic); 130 | // if client appearance is later than topic, do not search for any messages 131 | if (lastSeen == null || topicSeen == null || lastSeen >= topicSeen) { 132 | return; 133 | } 134 | // message sync - update subscriber with the current timestamp 135 | service.checkpoint(subscriberId); 136 | service.flushCheckpoint(); 137 | // there are unreceived messages, get all outstanding messages since last seen 138 | QueryBuilder queryBuilder = termQuery("topic", topic); 139 | RangeFilterBuilder filterBuilder = rangeFilter("timestamp").gte(lastSeen); 140 | SearchResponse searchResponse = client.prepareSearch() 141 | .setIndices(pubSubIndexName) 142 | .setTypes("publish") 143 | .setSearchType(SearchType.SCAN) 144 | .setScroll(scrollTimeout) 145 | .setQuery(queryBuilder) 146 | .setPostFilter(filterBuilder) 147 | .addField("data") 148 | .addField("timestamp") 149 | .setSize(scrollSize) 150 | .execute().actionGet(); 151 | boolean failed = searchResponse.getFailedShards() > 0 || searchResponse.isTimedOut(); 152 | if (failed) { 153 | logger.error("searching for messages for topic {} failed: failed shards={} timeout={}", 154 | topic, searchResponse.getFailedShards(), searchResponse.isTimedOut()); 155 | return; 156 | } 157 | // look for messages 158 | long totalHits = searchResponse.getHits().getTotalHits(); 159 | boolean zero = totalHits == 0L; 160 | if (zero) { 161 | return; 162 | } 163 | // slurp in all outstanding messages 164 | while (true) { 165 | searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()) 166 | .setScroll(scrollTimeout) 167 | .execute().actionGet(); 168 | for (SearchHit hit : searchResponse.getHits()) { 169 | Long timestamp = (Long) hit.field("timestamp").getValues().get(0); 170 | Map data = hit.field("data").getValue(); 171 | channel.write(new NettyInteractiveResponse("message", createPublishMessage(timestamp, data)).response()); 172 | } 173 | if (searchResponse.getHits().hits().length == 0) { 174 | break; 175 | } 176 | } 177 | } 178 | 179 | private XContentBuilder createPublishMessage(long timestamp, Map data) { 180 | try { 181 | return jsonBuilder().startObject() 182 | .field("timestamp", timestamp) 183 | .field("data", data) 184 | .endObject(); 185 | } catch (IOException e) { 186 | return null; 187 | } 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/action/websocket/pubsub/UnsubscribeAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.websocket.pubsub; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.delete.DeleteResponse; 5 | import org.elasticsearch.client.Client; 6 | import org.elasticsearch.common.inject.Inject; 7 | import org.elasticsearch.common.settings.Settings; 8 | import org.elasticsearch.common.xcontent.XContentBuilder; 9 | import org.xbib.elasticsearch.websocket.BaseInteractiveHandler; 10 | import org.xbib.elasticsearch.websocket.InteractiveChannel; 11 | import org.xbib.elasticsearch.websocket.InteractiveController; 12 | import org.xbib.elasticsearch.websocket.InteractiveRequest; 13 | 14 | import java.io.IOException; 15 | 16 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 17 | 18 | /** 19 | * Unsubscribe action. It removes a subscription. 20 | */ 21 | public class UnsubscribeAction extends BaseInteractiveHandler { 22 | 23 | private final String TYPE = "unsubscribe"; 24 | private final String pubSubIndexName; 25 | 26 | @Inject 27 | public UnsubscribeAction(Settings settings, 28 | Client client, 29 | InteractiveController controller) { 30 | super(settings, client); 31 | this.pubSubIndexName = PubSubIndexName.Conf.indexName(settings); 32 | controller.registerHandler(TYPE, this); 33 | } 34 | 35 | @Override 36 | public void handleRequest(final InteractiveRequest request, final InteractiveChannel channel) { 37 | String subscriberId = request.hasParam("subscriber") ? request.paramAsString("subscriber") : null; 38 | if (subscriberId == null) { 39 | try { 40 | channel.sendResponse(TYPE, new IllegalArgumentException("no subscriber")); 41 | } catch (IOException e) { 42 | logger.error("error while sending failure response", e); 43 | } 44 | } 45 | try { 46 | client.prepareDelete(pubSubIndexName, SubscribeAction.TYPE, subscriberId) 47 | .execute(new ActionListener() { 48 | @Override 49 | public void onResponse(DeleteResponse response) { 50 | try { 51 | XContentBuilder builder = jsonBuilder(); 52 | builder.startObject().field("ok", true).field("id", response.getId()).endObject(); 53 | channel.sendResponse(TYPE, builder); 54 | } catch (Exception e) { 55 | onFailure(e); 56 | } 57 | } 58 | 59 | @Override 60 | public void onFailure(Throwable e) { 61 | logger.error("error while processing unsubscribe request", e); 62 | try { 63 | channel.sendResponse(TYPE, e); 64 | } catch (IOException ex) { 65 | logger.error("error while sending error response", ex); 66 | } 67 | } 68 | }); 69 | } catch (Exception e) { 70 | logger.error("exception while processing unsubscribe request", e); 71 | try { 72 | channel.sendResponse(TYPE, e); 73 | } catch (IOException e1) { 74 | logger.error("exception while sending exception response", e1); 75 | } 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/bytes/BytesArray.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.common.bytes; 2 | 3 | import org.apache.lucene.util.BytesRef; 4 | import org.elasticsearch.ElasticsearchIllegalArgumentException; 5 | import org.elasticsearch.common.base.Charsets; 6 | import org.elasticsearch.common.io.stream.BytesStreamInput; 7 | import org.elasticsearch.common.io.stream.StreamInput; 8 | import org.jboss.netty.buffer.ChannelBuffer; 9 | import org.jboss.netty.buffer.ChannelBuffers; 10 | 11 | import java.io.IOException; 12 | import java.io.OutputStream; 13 | import java.nio.ByteBuffer; 14 | import java.nio.channels.GatheringByteChannel; 15 | import java.util.Arrays; 16 | 17 | /** 18 | * BytesArray for un-relocated netty. 19 | */ 20 | public class BytesArray implements BytesReference { 21 | 22 | private byte[] bytes; 23 | private int offset; 24 | private int length; 25 | 26 | public BytesArray(String bytes) { 27 | BytesRef bytesRef = new BytesRef(bytes); 28 | this.bytes = bytesRef.bytes; 29 | this.offset = bytesRef.offset; 30 | this.length = bytesRef.length; 31 | } 32 | 33 | public BytesArray(BytesRef bytesRef) { 34 | this(bytesRef, false); 35 | } 36 | 37 | public BytesArray(BytesRef bytesRef, boolean deepCopy) { 38 | if (deepCopy) { 39 | BytesRef copy = BytesRef.deepCopyOf(bytesRef); 40 | bytes = copy.bytes; 41 | offset = copy.offset; 42 | length = copy.length; 43 | } else { 44 | bytes = bytesRef.bytes; 45 | offset = bytesRef.offset; 46 | length = bytesRef.length; 47 | } 48 | } 49 | 50 | public BytesArray(byte[] bytes) { 51 | this.bytes = bytes; 52 | this.offset = 0; 53 | this.length = bytes.length; 54 | } 55 | 56 | public BytesArray(byte[] bytes, int offset, int length) { 57 | this.bytes = bytes; 58 | this.offset = offset; 59 | this.length = length; 60 | } 61 | 62 | @Override 63 | public byte get(int index) { 64 | return bytes[offset + index]; 65 | } 66 | 67 | @Override 68 | public int length() { 69 | return length; 70 | } 71 | 72 | @Override 73 | public BytesReference slice(int from, int length) { 74 | if (from < 0 || (from + length) > this.length) { 75 | throw new ElasticsearchIllegalArgumentException("can't slice a buffer with length [" + this.length + "], with slice parameters from [" + from + "], length [" + length + "]"); 76 | } 77 | return new BytesArray(bytes, offset + from, length); 78 | } 79 | 80 | @Override 81 | public StreamInput streamInput() { 82 | return new BytesStreamInput(bytes, offset, length, false); 83 | } 84 | 85 | @Override 86 | public void writeTo(OutputStream os) throws IOException { 87 | os.write(bytes, offset, length); 88 | } 89 | 90 | @Override 91 | public void writeTo(GatheringByteChannel channel) throws IOException { 92 | channel.write(ByteBuffer.wrap(bytes, offset, length())); 93 | } 94 | 95 | @Override 96 | public byte[] toBytes() { 97 | if (offset == 0 && bytes.length == length) { 98 | return bytes; 99 | } 100 | return Arrays.copyOfRange(bytes, offset, offset + length); 101 | } 102 | 103 | @Override 104 | public BytesArray toBytesArray() { 105 | return this; 106 | } 107 | 108 | @Override 109 | public BytesArray copyBytesArray() { 110 | return new BytesArray(Arrays.copyOfRange(bytes, offset, offset + length)); 111 | } 112 | 113 | @Override 114 | public ChannelBuffer toChannelBuffer() { 115 | return ChannelBuffers.wrappedBuffer(bytes, offset, length); 116 | } 117 | 118 | @Override 119 | public boolean hasArray() { 120 | return true; 121 | } 122 | 123 | @Override 124 | public byte[] array() { 125 | return bytes; 126 | } 127 | 128 | @Override 129 | public int arrayOffset() { 130 | return offset; 131 | } 132 | 133 | @Override 134 | public String toUtf8() { 135 | if (length == 0) { 136 | return ""; 137 | } 138 | return new String(bytes, offset, length, Charsets.UTF_8); 139 | } 140 | 141 | @Override 142 | public BytesRef toBytesRef() { 143 | return new BytesRef(bytes, offset, length); 144 | } 145 | 146 | @Override 147 | public BytesRef copyBytesRef() { 148 | return new BytesRef(Arrays.copyOfRange(bytes, offset, offset + length)); 149 | } 150 | 151 | @Override 152 | public int hashCode() { 153 | return Helper.bytesHashCode(this); 154 | } 155 | 156 | @Override 157 | public boolean equals(Object obj) { 158 | return Helper.bytesEqual(this, (BytesReference) obj); 159 | } 160 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/bytes/BytesReference.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.common.bytes; 2 | 3 | import org.apache.lucene.util.BytesRef; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.jboss.netty.buffer.ChannelBuffer; 6 | 7 | import java.io.IOException; 8 | import java.io.OutputStream; 9 | import java.nio.channels.GatheringByteChannel; 10 | 11 | /** 12 | * A reference to bytes for our Netty. 13 | */ 14 | public interface BytesReference { 15 | 16 | public static class Helper { 17 | 18 | public static boolean bytesEqual(BytesReference a, BytesReference b) { 19 | if (a == b) { 20 | return true; 21 | } 22 | if (a.length() != b.length()) { 23 | return false; 24 | } 25 | 26 | return bytesEquals(a, b); 27 | } 28 | 29 | // pkg-private for testing 30 | static boolean bytesEquals(BytesReference a, BytesReference b) { 31 | assert a.length() == b.length(); 32 | for (int i = 0, end = a.length(); i < end; ++i) { 33 | if (a.get(i) != b.get(i)) { 34 | return false; 35 | } 36 | } 37 | 38 | return true; 39 | } 40 | 41 | // pkg-private for testing 42 | static boolean slowBytesEquals(BytesReference a, BytesReference b) { 43 | assert a.length() == b.length(); 44 | for (int i = 0, end = a.length(); i < end; ++i) { 45 | if (a.get(i) != b.get(i)) { 46 | return false; 47 | } 48 | } 49 | 50 | return true; 51 | } 52 | 53 | public static int bytesHashCode(BytesReference a) { 54 | if (a.hasArray()) { 55 | return hashCode(a.array(), a.arrayOffset(), a.length()); 56 | } else { 57 | return slowHashCode(a); 58 | } 59 | } 60 | 61 | // pkg-private for testing 62 | static int hashCode(byte[] array, int offset, int length) { 63 | int result = 1; 64 | for (int i = offset, end = offset + length; i < end; ++i) { 65 | result = 31 * result + array[i]; 66 | } 67 | return result; 68 | } 69 | 70 | // pkg-private for testing 71 | static int slowHashCode(BytesReference a) { 72 | int result = 1; 73 | for (int i = 0, end = a.length(); i < end; ++i) { 74 | result = 31 * result + a.get(i); 75 | } 76 | return result; 77 | } 78 | } 79 | 80 | /** 81 | * Returns the byte at the specified index. Need to be between 0 and length. 82 | * @param index index 83 | * @return byte 84 | */ 85 | byte get(int index); 86 | 87 | /** 88 | * The length. 89 | * @return length 90 | */ 91 | int length(); 92 | 93 | /** 94 | * Slice the bytes from the from index up to length. 95 | * @param from from 96 | * @param length length 97 | * @return byte reference 98 | */ 99 | BytesReference slice(int from, int length); 100 | 101 | /** 102 | * A stream input of the bytes. 103 | * @return byte reference 104 | */ 105 | StreamInput streamInput(); 106 | 107 | /** 108 | * Writes the bytes directly to the output stream. 109 | * @param os output stream 110 | * @throws java.io.IOException if this method fails 111 | */ 112 | void writeTo(OutputStream os) throws IOException; 113 | 114 | /** 115 | * Writes the bytes directly to the channel. 116 | * @param channel channel 117 | * @throws java.io.IOException if this method fails 118 | */ 119 | void writeTo(GatheringByteChannel channel) throws IOException; 120 | 121 | /** 122 | * Returns the bytes as a single byte array. 123 | * @return bytes 124 | */ 125 | byte[] toBytes(); 126 | 127 | /** 128 | * Returns the bytes as a byte array, possibly sharing the underlying byte buffer. 129 | * @return byt earray 130 | */ 131 | BytesArray toBytesArray(); 132 | 133 | /** 134 | * Returns the bytes copied over as a byte array. 135 | * @return byte array 136 | */ 137 | BytesArray copyBytesArray(); 138 | 139 | /** 140 | * Returns the bytes as a channel buffer. 141 | * @return channel buffer 142 | */ 143 | ChannelBuffer toChannelBuffer(); 144 | 145 | /** 146 | * Is there an underlying byte array for this bytes reference. 147 | * @return true or false 148 | */ 149 | boolean hasArray(); 150 | 151 | /** 152 | * The underlying byte array (if exists). 153 | * @return byte array 154 | */ 155 | byte[] array(); 156 | 157 | /** 158 | * The offset into the underlying byte array. 159 | * @return offset 160 | */ 161 | int arrayOffset(); 162 | 163 | /** 164 | * Converts to a string based on utf8. 165 | * @return string 166 | */ 167 | String toUtf8(); 168 | 169 | /** 170 | * Converts to Lucene BytesRef. 171 | * @return bytes reference 172 | */ 173 | BytesRef toBytesRef(); 174 | 175 | /** 176 | * Converts to a copied Lucene BytesRef. 177 | * @return bytes reference 178 | */ 179 | BytesRef copyBytesRef(); 180 | } 181 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/bytes/ChannelBufferBytesReference.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.common.bytes; 2 | 3 | import org.apache.lucene.util.BytesRef; 4 | import org.elasticsearch.common.base.Charsets; 5 | import org.elasticsearch.common.bytes.BytesArray; 6 | import org.elasticsearch.common.bytes.BytesReference; 7 | import org.elasticsearch.common.io.stream.StreamInput; 8 | import org.jboss.netty.buffer.ChannelBuffer; 9 | import org.xbib.elasticsearch.transport.netty.ChannelBufferStreamInputFactory; 10 | 11 | import java.io.IOException; 12 | import java.io.OutputStream; 13 | import java.nio.channels.GatheringByteChannel; 14 | 15 | /** 16 | * ChannelBufferBytesReference for un-relocated netty 17 | */ 18 | public class ChannelBufferBytesReference implements BytesReference { 19 | 20 | private final ChannelBuffer buffer; 21 | 22 | public ChannelBufferBytesReference(ChannelBuffer buffer) { 23 | this.buffer = buffer; 24 | } 25 | 26 | @Override 27 | public byte get(int index) { 28 | return buffer.getByte(buffer.readerIndex() + index); 29 | } 30 | 31 | @Override 32 | public int length() { 33 | return buffer.readableBytes(); 34 | } 35 | 36 | @Override 37 | public BytesReference slice(int from, int length) { 38 | return new ChannelBufferBytesReference(buffer.slice(from, length)); 39 | } 40 | 41 | @Override 42 | public StreamInput streamInput() { 43 | return ChannelBufferStreamInputFactory.create(buffer.duplicate()); 44 | } 45 | 46 | @Override 47 | public void writeTo(OutputStream os) throws IOException { 48 | buffer.getBytes(buffer.readerIndex(), os, length()); 49 | } 50 | 51 | @Override 52 | public void writeTo(GatheringByteChannel channel) throws IOException { 53 | buffer.getBytes(buffer.readerIndex(), channel, length()); 54 | } 55 | 56 | @Override 57 | public byte[] toBytes() { 58 | return copyBytesArray().toBytes(); 59 | } 60 | 61 | @Override 62 | public BytesArray toBytesArray() { 63 | if (buffer.hasArray()) { 64 | return new BytesArray(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), buffer.readableBytes()); 65 | } 66 | return copyBytesArray(); 67 | } 68 | 69 | @Override 70 | public BytesArray copyBytesArray() { 71 | byte[] copy = new byte[buffer.readableBytes()]; 72 | buffer.getBytes(buffer.readerIndex(), copy); 73 | return new BytesArray(copy); 74 | } 75 | 76 | 77 | @Override 78 | public boolean hasArray() { 79 | return buffer.hasArray(); 80 | } 81 | 82 | @Override 83 | public byte[] array() { 84 | return buffer.array(); 85 | } 86 | 87 | @Override 88 | public int arrayOffset() { 89 | return buffer.arrayOffset() + buffer.readerIndex(); 90 | } 91 | 92 | @Override 93 | public String toUtf8() { 94 | return buffer.toString(Charsets.UTF_8); 95 | } 96 | 97 | @Override 98 | public BytesRef toBytesRef() { 99 | throw new UnsupportedOperationException("Not supported"); 100 | } 101 | 102 | @Override 103 | public BytesRef copyBytesRef() { 104 | throw new UnsupportedOperationException("Not supported"); 105 | } 106 | 107 | @Override 108 | public org.elasticsearch.common.netty.buffer.ChannelBuffer toChannelBuffer() { 109 | // return buffer.duplicate(); 110 | throw new UnsupportedOperationException("Not supported"); 111 | } 112 | 113 | @Override 114 | public int hashCode() { 115 | return Helper.bytesHashCode(this); 116 | } 117 | 118 | @Override 119 | public boolean equals(Object obj) { 120 | return Helper.bytesEqual(this, (BytesReference) obj); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/bytes/ReleasableBytesReference.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.common.bytes; 2 | 3 | import org.elasticsearch.common.lease.Releasable; 4 | 5 | public interface ReleasableBytesReference extends BytesReference, Releasable { 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/bytes/ReleasablePagedBytesReference.java: -------------------------------------------------------------------------------- 1 | 2 | package org.xbib.elasticsearch.common.bytes; 3 | 4 | import org.elasticsearch.ElasticsearchException; 5 | import org.elasticsearch.common.lease.Releasables; 6 | import org.elasticsearch.common.util.BigArrays; 7 | import org.elasticsearch.common.util.ByteArray; 8 | 9 | /** 10 | * An extension to {@link org.elasticsearch.common.bytes.PagedBytesReference} that requires releasing its content. This 11 | * class exists to make it explicit when a bytes reference needs to be released, and when not. 12 | */ 13 | public class ReleasablePagedBytesReference extends PagedBytesReference implements ReleasableBytesReference { 14 | 15 | public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int length) { 16 | super(bigarrays, bytearray, length); 17 | } 18 | 19 | public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int from, int length) { 20 | super(bigarrays, bytearray, from, length); 21 | } 22 | 23 | @Override 24 | public void close() throws ElasticsearchException { 25 | Releasables.close(bytearray); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/io/ReleasableBytesStream.java: -------------------------------------------------------------------------------- 1 | 2 | package org.xbib.elasticsearch.common.io; 3 | 4 | import org.xbib.elasticsearch.common.bytes.ReleasableBytesReference; 5 | import org.elasticsearch.common.io.BytesStream; 6 | 7 | /** 8 | * A bytes stream that requires its bytes to be released once no longer used. 9 | */ 10 | public interface ReleasableBytesStream extends BytesStream { 11 | 12 | ReleasableBytesReference ourBytes(); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/io/stream/BytesStreamOutput.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.common.io.stream; 2 | 3 | import org.xbib.elasticsearch.common.bytes.BytesReference; 4 | import org.xbib.elasticsearch.common.bytes.PagedBytesReference; 5 | import org.elasticsearch.common.io.BytesStream; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | import org.elasticsearch.common.util.BigArrays; 8 | import org.elasticsearch.common.util.ByteArray; 9 | 10 | import java.io.IOException; 11 | 12 | public class BytesStreamOutput extends StreamOutput implements BytesStream { 13 | 14 | protected final BigArrays bigarrays; 15 | 16 | protected ByteArray bytes; 17 | 18 | protected int count; 19 | 20 | /** 21 | * Create a non recycling {@link BytesStreamOutput} with 1 initial page acquired. 22 | */ 23 | public BytesStreamOutput() { 24 | this(BigArrays.PAGE_SIZE_IN_BYTES); 25 | } 26 | 27 | /** 28 | * Create a non recycling {@link BytesStreamOutput} with enough initial pages acquired 29 | * to satisfy the capacity given by expected size. 30 | * 31 | * @param expectedSize the expected maximum size of the stream in bytes. 32 | */ 33 | public BytesStreamOutput(int expectedSize) { 34 | this(expectedSize, BigArrays.NON_RECYCLING_INSTANCE); 35 | } 36 | 37 | protected BytesStreamOutput(int expectedSize, BigArrays bigarrays) { 38 | this.bigarrays = bigarrays; 39 | this.bytes = bigarrays.newByteArray(expectedSize); 40 | } 41 | 42 | @Override 43 | public boolean seekPositionSupported() { 44 | return true; 45 | } 46 | 47 | @Override 48 | public long position() throws IOException { 49 | return count; 50 | } 51 | 52 | @Override 53 | public void writeByte(byte b) throws IOException { 54 | ensureCapacity(count+1); 55 | bytes.set(count, b); 56 | count++; 57 | } 58 | 59 | @Override 60 | public void writeBytes(byte[] b, int offset, int length) throws IOException { 61 | // nothing to copy 62 | if (length == 0) { 63 | return; 64 | } 65 | 66 | // illegal args: offset and/or length exceed array size 67 | if (b.length < (offset + length)) { 68 | throw new IllegalArgumentException("Illegal offset " + offset + "/length " + length + " for byte[] of length " + b.length); 69 | } 70 | 71 | // get enough pages for new size 72 | ensureCapacity(count+length); 73 | 74 | // bulk copy 75 | bytes.set(count, b, offset, length); 76 | 77 | // advance 78 | count += length; 79 | } 80 | 81 | public void reset() { 82 | // shrink list of pages 83 | if (bytes.size() > BigArrays.PAGE_SIZE_IN_BYTES) { 84 | bytes = bigarrays.resize(bytes, BigArrays.PAGE_SIZE_IN_BYTES); 85 | } 86 | 87 | // go back to start 88 | count = 0; 89 | } 90 | 91 | @Override 92 | public void flush() throws IOException { 93 | // nothing to do 94 | } 95 | 96 | @Override 97 | public void seek(long position) throws IOException { 98 | if (position > Integer.MAX_VALUE) { 99 | throw new IllegalArgumentException("position " + position + " > Integer.MAX_VALUE"); 100 | } 101 | 102 | count = (int)position; 103 | ensureCapacity(count); 104 | } 105 | 106 | public void skip(int length) { 107 | count += length; 108 | ensureCapacity(count); 109 | } 110 | 111 | @Override 112 | public void close() throws IOException { 113 | // empty for now. 114 | } 115 | 116 | /** 117 | * Returns the current size of the buffer. 118 | * 119 | * @return the value of the count field, which is the number of valid 120 | * bytes in this output stream. 121 | * @see java.io.ByteArrayOutputStream#count 122 | */ 123 | public int size() { 124 | return count; 125 | } 126 | 127 | @Override 128 | public org.elasticsearch.common.bytes.BytesReference bytes() { 129 | return new org.elasticsearch.common.bytes.PagedBytesReference(bigarrays, bytes, count); 130 | } 131 | 132 | public BytesReference ourBytes() { 133 | return new PagedBytesReference(bigarrays, bytes, count); 134 | } 135 | 136 | private void ensureCapacity(int offset) { 137 | bytes = bigarrays.grow(bytes, offset); 138 | } 139 | 140 | } 141 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/io/stream/CachedStreamOutput.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.common.io.stream; 2 | 3 | import org.elasticsearch.common.compress.Compressor; 4 | import org.elasticsearch.common.io.UTF8StreamWriter; 5 | import org.elasticsearch.common.io.stream.HandlesStreamOutput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | import org.elasticsearch.common.unit.ByteSizeValue; 8 | import org.elasticsearch.common.util.concurrent.ConcurrentCollections; 9 | import org.elasticsearch.monitor.jvm.JvmInfo; 10 | 11 | import java.io.IOException; 12 | import java.lang.ref.SoftReference; 13 | import java.util.Queue; 14 | import java.util.concurrent.atomic.AtomicInteger; 15 | 16 | /** 17 | * 18 | */ 19 | public class CachedStreamOutput { 20 | 21 | private static Entry newEntry() { 22 | BytesStreamOutput bytes = new BytesStreamOutput(); 23 | HandlesStreamOutput handles = new HandlesStreamOutput(bytes); 24 | return new Entry(bytes, handles); 25 | } 26 | 27 | public static class Entry { 28 | private final BytesStreamOutput bytes; 29 | private final HandlesStreamOutput handles; 30 | 31 | Entry(BytesStreamOutput bytes, HandlesStreamOutput handles) { 32 | this.bytes = bytes; 33 | this.handles = handles; 34 | } 35 | 36 | public void reset() { 37 | bytes.reset(); 38 | handles.setOut(bytes); 39 | handles.clear(); 40 | } 41 | 42 | public BytesStreamOutput bytes() { 43 | return bytes; 44 | } 45 | 46 | public StreamOutput handles() throws IOException { 47 | return handles; 48 | } 49 | 50 | public StreamOutput bytes(Compressor compressor) throws IOException { 51 | return compressor.streamOutput(bytes); 52 | } 53 | 54 | public StreamOutput handles(Compressor compressor) throws IOException { 55 | StreamOutput compressed = compressor.streamOutput(bytes); 56 | handles.clear(); 57 | handles.setOut(compressed); 58 | return handles; 59 | } 60 | } 61 | 62 | static class SoftWrapper { 63 | private SoftReference ref; 64 | 65 | public SoftWrapper() { 66 | } 67 | 68 | public void set(T ref) { 69 | this.ref = new SoftReference(ref); 70 | } 71 | 72 | public T get() { 73 | return ref == null ? null : ref.get(); 74 | } 75 | 76 | public void clear() { 77 | ref = null; 78 | } 79 | } 80 | 81 | private static final SoftWrapper> cache = new SoftWrapper>(); 82 | private static final AtomicInteger counter = new AtomicInteger(); 83 | public static int BYTES_LIMIT = 1 * 1024 * 1024; // don't cache entries that are bigger than that... 84 | public static int COUNT_LIMIT = 100; // number of concurrent entries cached 85 | 86 | static { 87 | // guess the maximum size per entry and the maximum number of entries based on the heap size 88 | long maxHeap = JvmInfo.jvmInfo().mem().heapMax().bytes(); 89 | if (maxHeap < ByteSizeValue.parseBytesSizeValue("500mb").bytes()) { 90 | BYTES_LIMIT = (int) ByteSizeValue.parseBytesSizeValue("500kb").bytes(); 91 | COUNT_LIMIT = 10; 92 | } else if (maxHeap < ByteSizeValue.parseBytesSizeValue("1gb").bytes()) { 93 | BYTES_LIMIT = (int) ByteSizeValue.parseBytesSizeValue("1mb").bytes(); 94 | COUNT_LIMIT = 20; 95 | } else if (maxHeap < ByteSizeValue.parseBytesSizeValue("4gb").bytes()) { 96 | BYTES_LIMIT = (int) ByteSizeValue.parseBytesSizeValue("2mb").bytes(); 97 | COUNT_LIMIT = 50; 98 | } else if (maxHeap < ByteSizeValue.parseBytesSizeValue("10gb").bytes()) { 99 | BYTES_LIMIT = (int) ByteSizeValue.parseBytesSizeValue("5mb").bytes(); 100 | COUNT_LIMIT = 50; 101 | } else { 102 | BYTES_LIMIT = (int) ByteSizeValue.parseBytesSizeValue("10mb").bytes(); 103 | COUNT_LIMIT = 100; 104 | } 105 | } 106 | 107 | public static void clear() { 108 | cache.clear(); 109 | } 110 | 111 | public static Entry popEntry() { 112 | Queue ref = cache.get(); 113 | if (ref == null) { 114 | return newEntry(); 115 | } 116 | Entry entry = ref.poll(); 117 | if (entry == null) { 118 | return newEntry(); 119 | } 120 | counter.decrementAndGet(); 121 | entry.reset(); 122 | return entry; 123 | } 124 | 125 | public static void pushEntry(Entry entry) { 126 | entry.reset(); 127 | if (entry.bytes().bytes().length() > BYTES_LIMIT) { 128 | return; 129 | } 130 | Queue ref = cache.get(); 131 | if (ref == null) { 132 | ref = ConcurrentCollections.newQueue(); 133 | counter.set(0); 134 | cache.set(ref); 135 | } 136 | if (counter.incrementAndGet() > COUNT_LIMIT) { 137 | counter.decrementAndGet(); 138 | } else { 139 | ref.add(entry); 140 | } 141 | } 142 | 143 | private static ThreadLocal> utf8StreamWriter = new ThreadLocal>(); 144 | 145 | public static UTF8StreamWriter utf8StreamWriter() { 146 | SoftReference ref = utf8StreamWriter.get(); 147 | UTF8StreamWriter writer = (ref == null) ? null : ref.get(); 148 | if (writer == null) { 149 | writer = new UTF8StreamWriter(1024 * 4); 150 | utf8StreamWriter.set(new SoftReference(writer)); 151 | } 152 | writer.reset(); 153 | return writer; 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/io/stream/ReleasableBytesStreamOutput.java: -------------------------------------------------------------------------------- 1 | 2 | package org.xbib.elasticsearch.common.io.stream; 3 | 4 | import org.xbib.elasticsearch.common.bytes.ReleasableBytesReference; 5 | import org.xbib.elasticsearch.common.bytes.ReleasablePagedBytesReference; 6 | import org.xbib.elasticsearch.common.io.ReleasableBytesStream; 7 | import org.elasticsearch.common.util.BigArrays; 8 | 9 | public class ReleasableBytesStreamOutput extends BytesStreamOutput implements ReleasableBytesStream { 10 | 11 | public ReleasableBytesStreamOutput(BigArrays bigarrays) { 12 | super(BigArrays.PAGE_SIZE_IN_BYTES, bigarrays); 13 | } 14 | 15 | public ReleasableBytesStreamOutput(int expectedSize, BigArrays bigarrays) { 16 | super(expectedSize, bigarrays); 17 | } 18 | 19 | @Override 20 | public ReleasableBytesReference ourBytes() { 21 | return new ReleasablePagedBytesReference(bigarrays, bytes, count); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/netty/KeepFrameDecoder.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.common.netty; 2 | 3 | import org.jboss.netty.buffer.ChannelBuffer; 4 | import org.jboss.netty.channel.Channel; 5 | import org.jboss.netty.channel.ChannelHandlerContext; 6 | import org.jboss.netty.handler.codec.frame.FrameDecoder; 7 | 8 | /** 9 | * A marker to not remove frame decoder from the resulting jar, so plugins can use it. 10 | */ 11 | public class KeepFrameDecoder extends FrameDecoder { 12 | 13 | public static final KeepFrameDecoder decoder = new KeepFrameDecoder(); 14 | 15 | @Override 16 | protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer) throws Exception { 17 | return null; 18 | } 19 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/netty/NettyStaticSetup.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.common.netty; 2 | 3 | import org.jboss.netty.util.ThreadNameDeterminer; 4 | import org.jboss.netty.util.ThreadRenamingRunnable; 5 | 6 | /** 7 | */ 8 | public class NettyStaticSetup { 9 | 10 | private static EsThreadNameDeterminer ES_THREAD_NAME_DETERMINER = new EsThreadNameDeterminer(); 11 | 12 | public static class EsThreadNameDeterminer implements ThreadNameDeterminer { 13 | @Override 14 | public String determineThreadName(String currentThreadName, String proposedThreadName) throws Exception { 15 | // we control the thread name with a context, so use both 16 | return currentThreadName + "{" + proposedThreadName + "}"; 17 | } 18 | } 19 | 20 | static { 21 | ThreadRenamingRunnable.setThreadNameDeterminer(ES_THREAD_NAME_DETERMINER); 22 | } 23 | 24 | public static void setup() { 25 | 26 | } 27 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/netty/OpenChannelsHandler.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.common.netty; 2 | 3 | import org.elasticsearch.common.logging.ESLogger; 4 | import org.elasticsearch.common.metrics.CounterMetric; 5 | import org.elasticsearch.common.util.concurrent.ConcurrentCollections; 6 | import org.jboss.netty.channel.Channel; 7 | import org.jboss.netty.channel.ChannelEvent; 8 | import org.jboss.netty.channel.ChannelFuture; 9 | import org.jboss.netty.channel.ChannelFutureListener; 10 | import org.jboss.netty.channel.ChannelHandler; 11 | import org.jboss.netty.channel.ChannelHandlerContext; 12 | import org.jboss.netty.channel.ChannelState; 13 | import org.jboss.netty.channel.ChannelStateEvent; 14 | import org.jboss.netty.channel.SimpleChannelUpstreamHandler; 15 | 16 | import java.util.Map; 17 | 18 | /** 19 | * 20 | */ 21 | @ChannelHandler.Sharable 22 | public class OpenChannelsHandler extends SimpleChannelUpstreamHandler { 23 | 24 | final Map openChannels = ConcurrentCollections.newConcurrentMap(); 25 | 26 | final CounterMetric openChannelsMetric = new CounterMetric(); 27 | 28 | final CounterMetric totalChannelsMetric = new CounterMetric(); 29 | 30 | final ESLogger logger; 31 | 32 | public OpenChannelsHandler(ESLogger logger) { 33 | this.logger = logger; 34 | } 35 | 36 | final ChannelFutureListener remover = new ChannelFutureListener() { 37 | @Override 38 | public void operationComplete(ChannelFuture future) throws Exception { 39 | Channel channel = openChannels.remove(future.getChannel().getId()); 40 | if (channel != null) { 41 | openChannelsMetric.dec(); 42 | } 43 | if (logger.isTraceEnabled()) { 44 | logger.trace("channel closed: {}", future.getChannel()); 45 | } 46 | } 47 | }; 48 | 49 | @Override 50 | public void handleUpstream(ChannelHandlerContext ctx, ChannelEvent e) throws Exception { 51 | if (e instanceof ChannelStateEvent) { 52 | ChannelStateEvent evt = (ChannelStateEvent) e; 53 | // OPEN is also sent to when closing channel, but with FALSE on it to indicate it closes 54 | if (evt.getState() == ChannelState.OPEN && Boolean.TRUE.equals(evt.getValue())) { 55 | if (logger.isTraceEnabled()) { 56 | logger.trace("channel opened: {}", ctx.getChannel()); 57 | } 58 | Channel channel = openChannels.put(ctx.getChannel().getId(), ctx.getChannel()); 59 | if (channel == null) { 60 | openChannelsMetric.inc(); 61 | totalChannelsMetric.inc(); 62 | ctx.getChannel().getCloseFuture().addListener(remover); 63 | } 64 | } 65 | } 66 | ctx.sendUpstream(e); 67 | } 68 | 69 | public Channel channel(Integer id) { 70 | return openChannels.get(id); 71 | } 72 | 73 | public long numberOfOpenChannels() { 74 | return openChannelsMetric.count(); 75 | } 76 | 77 | public long totalChannels() { 78 | return totalChannelsMetric.count(); 79 | } 80 | 81 | public void close() { 82 | for (Channel channel : openChannels.values()) { 83 | channel.close().awaitUninterruptibly(); 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/common/netty/ReleaseChannelFutureListener.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.common.netty; 2 | 3 | import org.elasticsearch.common.lease.Releasable; 4 | import org.jboss.netty.channel.ChannelFuture; 5 | import org.jboss.netty.channel.ChannelFutureListener; 6 | 7 | public class ReleaseChannelFutureListener implements ChannelFutureListener { 8 | 9 | private final Releasable releasable; 10 | 11 | public ReleaseChannelFutureListener(Releasable releasable) { 12 | this.releasable = releasable; 13 | } 14 | 15 | @Override 16 | public void operationComplete(ChannelFuture future) throws Exception { 17 | releasable.close(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/BindHttpException.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http; 2 | 3 | public class BindHttpException extends HttpException { 4 | 5 | public BindHttpException(String message) { 6 | super(message); 7 | } 8 | 9 | public BindHttpException(String message, Throwable cause) { 10 | super(message, cause); 11 | } 12 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/HttpChannel.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http; 2 | 3 | import org.elasticsearch.rest.RestChannel; 4 | import org.elasticsearch.rest.RestRequest; 5 | 6 | public abstract class HttpChannel extends RestChannel { 7 | 8 | protected HttpChannel(RestRequest request) { 9 | super(request); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/HttpException.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http; 2 | 3 | import org.elasticsearch.ElasticsearchException; 4 | 5 | public class HttpException extends ElasticsearchException { 6 | 7 | public HttpException(String message) { 8 | super(message); 9 | } 10 | 11 | public HttpException(String message, Throwable cause) { 12 | super(message, cause); 13 | } 14 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/HttpInfo.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http; 2 | 3 | import org.elasticsearch.common.io.stream.StreamInput; 4 | import org.elasticsearch.common.io.stream.StreamOutput; 5 | import org.elasticsearch.common.io.stream.Streamable; 6 | import org.elasticsearch.common.transport.BoundTransportAddress; 7 | import org.elasticsearch.common.unit.ByteSizeValue; 8 | import org.elasticsearch.common.xcontent.ToXContent; 9 | import org.elasticsearch.common.xcontent.XContentBuilder; 10 | import org.elasticsearch.common.xcontent.XContentBuilderString; 11 | 12 | import java.io.IOException; 13 | import java.io.Serializable; 14 | 15 | 16 | public class HttpInfo implements Streamable, Serializable, ToXContent { 17 | 18 | private BoundTransportAddress address; 19 | private long maxContentLength; 20 | 21 | HttpInfo() { 22 | } 23 | 24 | public HttpInfo(BoundTransportAddress address, long maxContentLength) { 25 | this.address = address; 26 | this.maxContentLength = maxContentLength; 27 | } 28 | 29 | static final class Fields { 30 | static final XContentBuilderString HTTP = new XContentBuilderString("http"); 31 | static final XContentBuilderString BOUND_ADDRESS = new XContentBuilderString("bound_address"); 32 | static final XContentBuilderString PUBLISH_ADDRESS = new XContentBuilderString("publish_address"); 33 | static final XContentBuilderString MAX_CONTENT_LENGTH = new XContentBuilderString("max_content_length"); 34 | static final XContentBuilderString MAX_CONTENT_LENGTH_IN_BYTES = new XContentBuilderString("max_content_length_in_bytes"); 35 | } 36 | 37 | @Override 38 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 39 | builder.startObject(Fields.HTTP); 40 | builder.field(Fields.BOUND_ADDRESS, address.boundAddress().toString()); 41 | builder.field(Fields.PUBLISH_ADDRESS, address.publishAddress().toString()); 42 | builder.byteSizeField(Fields.MAX_CONTENT_LENGTH_IN_BYTES, Fields.MAX_CONTENT_LENGTH, maxContentLength); 43 | builder.endObject(); 44 | return builder; 45 | } 46 | 47 | public static HttpInfo readHttpInfo(StreamInput in) throws IOException { 48 | HttpInfo info = new HttpInfo(); 49 | info.readFrom(in); 50 | return info; 51 | } 52 | 53 | @Override 54 | public void readFrom(StreamInput in) throws IOException { 55 | address = BoundTransportAddress.readBoundTransportAddress(in); 56 | maxContentLength = in.readLong(); 57 | } 58 | 59 | @Override 60 | public void writeTo(StreamOutput out) throws IOException { 61 | address.writeTo(out); 62 | out.writeLong(maxContentLength); 63 | } 64 | 65 | public BoundTransportAddress address() { 66 | return address; 67 | } 68 | 69 | public BoundTransportAddress getAddress() { 70 | return address(); 71 | } 72 | 73 | public ByteSizeValue maxContentLength() { 74 | return new ByteSizeValue(maxContentLength); 75 | } 76 | 77 | public ByteSizeValue getMaxContentLength() { 78 | return maxContentLength(); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/HttpRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http; 2 | 3 | import org.elasticsearch.rest.RestRequest; 4 | 5 | public abstract class HttpRequest extends RestRequest { 6 | 7 | public abstract String getMethod(); 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/HttpServerAdapter.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http; 2 | 3 | public interface HttpServerAdapter { 4 | 5 | void dispatchRequest(HttpRequest request, HttpChannel channel); 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/HttpServerModule.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http; 2 | 3 | import org.elasticsearch.common.collect.ImmutableList; 4 | import org.elasticsearch.common.inject.AbstractModule; 5 | import org.elasticsearch.common.inject.Module; 6 | import org.elasticsearch.common.inject.Modules; 7 | import org.elasticsearch.common.inject.SpawnModules; 8 | import org.elasticsearch.common.settings.Settings; 9 | import org.xbib.elasticsearch.http.netty.NettyWebSocketServerTransportModule; 10 | 11 | public class HttpServerModule extends AbstractModule implements SpawnModules { 12 | 13 | private final Settings settings; 14 | 15 | public HttpServerModule(Settings settings) { 16 | this.settings = settings; 17 | } 18 | 19 | @Override 20 | public Iterable spawnModules() { 21 | return ImmutableList.of(Modules.createModule(settings.getAsClass("websocket.type", NettyWebSocketServerTransportModule.class, "org.xbib.elasticsearch.websocket.http.", "HttpServerTransportModule"), settings)); 22 | } 23 | 24 | @SuppressWarnings({"unchecked"}) 25 | @Override 26 | protected void configure() { 27 | bind(HttpServer.class).asEagerSingleton(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/HttpServerTransport.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http; 2 | 3 | import org.elasticsearch.common.component.LifecycleComponent; 4 | import org.elasticsearch.common.transport.BoundTransportAddress; 5 | import org.elasticsearch.common.xcontent.XContentBuilder; 6 | import org.jboss.netty.channel.Channel; 7 | 8 | /** 9 | * HttpServerTransport extended by Websocket services 10 | */ 11 | public interface HttpServerTransport extends LifecycleComponent { 12 | 13 | /** 14 | * Return the bound transport addess 15 | * 16 | * @return the bound transport addess 17 | */ 18 | BoundTransportAddress boundAddress(); 19 | 20 | HttpInfo info(); 21 | 22 | /** 23 | * Get HTTP statistics 24 | * 25 | * @return a statistics object 26 | */ 27 | HttpStats stats(); 28 | 29 | /** 30 | * Set HTTP server adapter 31 | * 32 | * @param httpServerAdapter HTTP server adapter 33 | */ 34 | void httpServerAdapter(HttpServerAdapter httpServerAdapter); 35 | 36 | /** 37 | * Set WebSocket server adapter 38 | * 39 | * @param webSocketServerAdapter web socket server adapter 40 | */ 41 | void webSocketServerAdapter(WebSocketServerAdapter webSocketServerAdapter); 42 | 43 | /** 44 | * Find channel for a given channel ID 45 | * 46 | * @param id id 47 | * @return channel or null 48 | */ 49 | Channel channel(Integer id); 50 | 51 | /** 52 | * Forward a message to a node for a given channel ID 53 | * 54 | * @param nodeAdress node address 55 | * @param channelId channel ID 56 | * @param message message 57 | */ 58 | void forward(String nodeAdress, Integer channelId, XContentBuilder message); 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/HttpStats.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http; 2 | 3 | import org.elasticsearch.common.io.stream.StreamInput; 4 | import org.elasticsearch.common.io.stream.StreamOutput; 5 | import org.elasticsearch.common.io.stream.Streamable; 6 | import org.elasticsearch.common.xcontent.ToXContent; 7 | import org.elasticsearch.common.xcontent.XContentBuilder; 8 | import org.elasticsearch.common.xcontent.XContentBuilderString; 9 | 10 | import java.io.IOException; 11 | 12 | public class HttpStats implements Streamable, ToXContent { 13 | 14 | private long serverOpen; 15 | private long totalOpen; 16 | 17 | HttpStats() { 18 | } 19 | 20 | public HttpStats(long serverOpen, long totalOpen) { 21 | this.serverOpen = serverOpen; 22 | this.totalOpen = totalOpen; 23 | } 24 | 25 | public long getServerOpen() { 26 | return this.serverOpen; 27 | } 28 | 29 | public long getTotalOpen() { 30 | return this.totalOpen; 31 | } 32 | 33 | public static HttpStats readHttpStats(StreamInput in) throws IOException { 34 | HttpStats stats = new HttpStats(); 35 | stats.readFrom(in); 36 | return stats; 37 | } 38 | 39 | @Override 40 | public void readFrom(StreamInput in) throws IOException { 41 | serverOpen = in.readVLong(); 42 | totalOpen = in.readVLong(); 43 | } 44 | 45 | @Override 46 | public void writeTo(StreamOutput out) throws IOException { 47 | out.writeVLong(serverOpen); 48 | out.writeVLong(totalOpen); 49 | } 50 | 51 | static final class Fields { 52 | static final XContentBuilderString WEBSOCKET = new XContentBuilderString("websocket"); 53 | static final XContentBuilderString CURRENT_OPEN = new XContentBuilderString("current_open"); 54 | static final XContentBuilderString TOTAL_OPENED = new XContentBuilderString("total_opened"); 55 | } 56 | 57 | @Override 58 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 59 | builder.startObject(Fields.WEBSOCKET); 60 | builder.field(Fields.CURRENT_OPEN, serverOpen); 61 | builder.field(Fields.TOTAL_OPENED, totalOpen); 62 | builder.endObject(); 63 | return builder; 64 | } 65 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/WebSocketServerAdapter.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http; 2 | 3 | import org.jboss.netty.channel.Channel; 4 | import org.jboss.netty.channel.ChannelHandlerContext; 5 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 6 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketServerHandshaker; 7 | import org.xbib.elasticsearch.websocket.Presence; 8 | 9 | /** 10 | * WebSocket server adapter 11 | */ 12 | public interface WebSocketServerAdapter { 13 | 14 | /** 15 | * Emit a presence event. 16 | * 17 | * @param presence presence 18 | * @param topic topic 19 | * @param channel channel 20 | */ 21 | void presence(Presence presence, String topic, Channel channel); 22 | 23 | /** 24 | * Emit a frame. 25 | * 26 | * @param handshaker handshaker 27 | * @param frame frame 28 | * @param context context 29 | */ 30 | void frame(WebSocketServerHandshaker handshaker, WebSocketFrame frame, ChannelHandlerContext context); 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/NettyHttpRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty; 2 | 3 | import org.elasticsearch.ElasticsearchIllegalArgumentException; 4 | import org.elasticsearch.common.bytes.BytesArray; 5 | import org.elasticsearch.common.bytes.BytesReference; 6 | import org.elasticsearch.rest.support.RestUtils; 7 | import org.jboss.netty.handler.codec.http.HttpMethod; 8 | import org.xbib.elasticsearch.common.bytes.ChannelBufferBytesReference; 9 | import org.xbib.elasticsearch.http.HttpRequest; 10 | 11 | import java.util.Map; 12 | 13 | import static org.elasticsearch.common.collect.Maps.newHashMap; 14 | 15 | public class NettyHttpRequest extends HttpRequest { 16 | 17 | private final org.jboss.netty.handler.codec.http.HttpRequest request; 18 | 19 | private final Map params; 20 | 21 | private final String rawPath; 22 | 23 | private final BytesReference content; 24 | 25 | private final String uri; 26 | 27 | public NettyHttpRequest(org.jboss.netty.handler.codec.http.HttpRequest request) { 28 | this.request = request; 29 | this.params = newHashMap(); 30 | this.content = request.getContent().readable() ? 31 | new ChannelBufferBytesReference(request.getContent()) : 32 | BytesArray.EMPTY; 33 | this.uri = request.getUri(); 34 | int pathEndPos = uri.indexOf('?'); 35 | if (pathEndPos < 0) { 36 | this.rawPath = uri; 37 | } else { 38 | this.rawPath = uri.substring(0, pathEndPos); 39 | RestUtils.decodeQueryString(uri, pathEndPos + 1, params); 40 | } 41 | } 42 | 43 | public org.jboss.netty.handler.codec.http.HttpRequest request() { 44 | return this.request; 45 | } 46 | 47 | @Override 48 | public Method method() { 49 | HttpMethod httpMethod = request.getMethod(); 50 | if (httpMethod == HttpMethod.GET) 51 | return Method.GET; 52 | 53 | if (httpMethod == HttpMethod.POST) 54 | return Method.POST; 55 | 56 | if (httpMethod == HttpMethod.PUT) 57 | return Method.PUT; 58 | 59 | if (httpMethod == HttpMethod.DELETE) 60 | return Method.DELETE; 61 | 62 | if (httpMethod == HttpMethod.HEAD) { 63 | return Method.HEAD; 64 | } 65 | 66 | if (httpMethod == HttpMethod.OPTIONS) { 67 | return Method.OPTIONS; 68 | } 69 | 70 | throw new ElasticsearchIllegalArgumentException("unsupported method " + httpMethod.getName()); 71 | } 72 | 73 | @Override 74 | public String getMethod() { 75 | return request.getMethod().getName(); 76 | } 77 | 78 | @Override 79 | public String uri() { 80 | return uri; 81 | } 82 | 83 | @Override 84 | public String rawPath() { 85 | return rawPath; 86 | } 87 | 88 | @Override 89 | public Map params() { 90 | return params; 91 | } 92 | 93 | @Override 94 | public boolean hasContent() { 95 | return content.length() > 0; 96 | } 97 | 98 | @Override 99 | public boolean contentUnsafe() { 100 | // Netty http decoder always copies over the http content 101 | return false; 102 | } 103 | 104 | @Override 105 | public BytesReference content() { 106 | return content; 107 | } 108 | 109 | @Override 110 | public String header(String name) { 111 | return request.headers().get(name); 112 | } 113 | 114 | @Override 115 | public Iterable> headers() { 116 | return request.headers().entries(); 117 | } 118 | 119 | @Override 120 | public boolean hasParam(String key) { 121 | return params.containsKey(key); 122 | } 123 | 124 | @Override 125 | public String param(String key) { 126 | return params.get(key); 127 | } 128 | 129 | @Override 130 | public String param(String key, String defaultValue) { 131 | String value = params.get(key); 132 | if (value == null) { 133 | return defaultValue; 134 | } 135 | return value; 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/NettyHttpRequestHandler.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty; 2 | 3 | import org.jboss.netty.channel.ChannelHandler; 4 | import org.jboss.netty.channel.ChannelHandlerContext; 5 | import org.jboss.netty.channel.ExceptionEvent; 6 | import org.jboss.netty.channel.MessageEvent; 7 | import org.jboss.netty.channel.SimpleChannelUpstreamHandler; 8 | import org.jboss.netty.handler.codec.http.HttpHeaders; 9 | import org.jboss.netty.handler.codec.http.HttpRequest; 10 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketServerHandshaker; 11 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketServerHandshakerFactory; 12 | import org.xbib.elasticsearch.websocket.Presence; 13 | 14 | /** 15 | * Handles HTTP request and upgrades HTTP to WebSocket if appropriate. 16 | */ 17 | @ChannelHandler.Sharable 18 | public class NettyHttpRequestHandler extends SimpleChannelUpstreamHandler { 19 | 20 | protected final NettyWebSocketServerTransport serverTransport; 21 | 22 | protected WebSocketServerHandshaker handshaker; 23 | 24 | private static String getWebSocketLocation(HttpRequest req) { 25 | return "ws://" + req.headers().get(HttpHeaders.Names.HOST) + WEBSOCKET_PATH; 26 | } 27 | 28 | private static final String WEBSOCKET_PATH = "/websocket"; 29 | 30 | public NettyHttpRequestHandler(NettyWebSocketServerTransport serverTransport) { 31 | this.serverTransport = serverTransport; 32 | } 33 | 34 | @Override 35 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { 36 | HttpRequest request = (HttpRequest) e.getMessage(); 37 | if (request.getUri().startsWith(WEBSOCKET_PATH)) { 38 | // Websocket handshake 39 | WebSocketServerHandshakerFactory wsFactory = new WebSocketServerHandshakerFactory(getWebSocketLocation(request), null, false); 40 | handshaker = wsFactory.newHandshaker(request); 41 | if (handshaker == null) { 42 | wsFactory.sendUnsupportedWebSocketVersionResponse(ctx.getChannel()); 43 | } else { 44 | handshaker.handshake(ctx.getChannel(), request).addListener(WebSocketServerHandshaker.HANDSHAKE_LISTENER); 45 | } 46 | // extract topic from request URI 47 | String topic = request.getUri(); 48 | if (topic.length() > WEBSOCKET_PATH.length() + 1) { 49 | topic = topic.substring(WEBSOCKET_PATH.length() + 1); 50 | serverTransport.presence(Presence.CONNECTED, topic, e.getChannel()); 51 | } 52 | } else { 53 | NettyHttpRequest nettyHttpRequest =new NettyHttpRequest(request); 54 | serverTransport.dispatchRequest(nettyHttpRequest, new NettyHttpChannel(serverTransport, e.getChannel(), nettyHttpRequest)); 55 | super.messageReceived(ctx, e); 56 | } 57 | } 58 | 59 | @Override 60 | public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception { 61 | serverTransport.exceptionCaught(ctx, e); 62 | e.getChannel().close(); 63 | } 64 | 65 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/NettyHttpServerPipelineFactory.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty; 2 | 3 | import org.jboss.netty.channel.ChannelPipeline; 4 | import org.jboss.netty.channel.ChannelPipelineFactory; 5 | import org.jboss.netty.channel.Channels; 6 | import org.jboss.netty.handler.codec.http.HttpChunkAggregator; 7 | import org.jboss.netty.handler.codec.http.HttpContentCompressor; 8 | import org.jboss.netty.handler.codec.http.HttpContentDecompressor; 9 | import org.jboss.netty.handler.codec.http.HttpRequestDecoder; 10 | import org.jboss.netty.handler.codec.http.HttpResponseEncoder; 11 | 12 | public class NettyHttpServerPipelineFactory implements ChannelPipelineFactory { 13 | 14 | protected final NettyWebSocketServerTransport transport; 15 | 16 | private final NettyHttpRequestHandler requestHandler; 17 | 18 | NettyHttpServerPipelineFactory(NettyWebSocketServerTransport transport) { 19 | this.transport = transport; 20 | this.requestHandler = new NettyHttpRequestHandler(transport); 21 | } 22 | 23 | @Override 24 | public ChannelPipeline getPipeline() throws Exception { 25 | ChannelPipeline pipeline = Channels.pipeline(); 26 | pipeline.addLast("openChannels", transport.serverOpenChannels); 27 | HttpRequestDecoder requestDecoder = new HttpRequestDecoder( 28 | (int) transport.maxInitialLineLength.bytes(), 29 | (int) transport.maxHeaderSize.bytes(), 30 | (int) transport.maxChunkSize.bytes()); 31 | if (transport.maxCumulationBufferCapacity != null) { 32 | if (transport.maxCumulationBufferCapacity.bytes() > Integer.MAX_VALUE) { 33 | requestDecoder.setMaxCumulationBufferCapacity(Integer.MAX_VALUE); 34 | } else { 35 | requestDecoder.setMaxCumulationBufferCapacity((int) transport.maxCumulationBufferCapacity.bytes()); 36 | } 37 | } 38 | if (transport.maxCompositeBufferComponents != -1) { 39 | requestDecoder.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); 40 | } 41 | pipeline.addLast("decoder", requestDecoder); 42 | if (transport.compression) { 43 | pipeline.addLast("decoder_compress", new HttpContentDecompressor()); 44 | } 45 | HttpChunkAggregator httpChunkAggregator = new HttpChunkAggregator((int) transport.maxContentLength.bytes()); 46 | if (transport.maxCompositeBufferComponents != -1) { 47 | httpChunkAggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents); 48 | } 49 | pipeline.addLast("aggregator", httpChunkAggregator); 50 | pipeline.addLast("encoder", new HttpResponseEncoder()); 51 | if (transport.compression) { 52 | pipeline.addLast("encoder_compress", new HttpContentCompressor(transport.compressionLevel)); 53 | } 54 | pipeline.addLast("handler", requestHandler); 55 | return pipeline; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/NettyInteractiveChannel.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty; 2 | 3 | import org.elasticsearch.common.xcontent.XContentBuilder; 4 | import org.jboss.netty.channel.Channel; 5 | import org.xbib.elasticsearch.websocket.InteractiveChannel; 6 | import org.xbib.elasticsearch.websocket.InteractiveResponse; 7 | 8 | import java.io.IOException; 9 | 10 | /** 11 | * Netty implementation for an interactive channel 12 | */ 13 | public class NettyInteractiveChannel implements InteractiveChannel { 14 | 15 | private final Channel channel; 16 | 17 | public NettyInteractiveChannel(Channel channel) { 18 | this.channel = channel; 19 | } 20 | 21 | @Override 22 | public Channel getChannel() { 23 | return channel; 24 | } 25 | 26 | @Override 27 | public void sendResponse(InteractiveResponse response) throws IOException { 28 | channel.write(response.response()); 29 | } 30 | 31 | @Override 32 | public void sendResponse(String type, Throwable t) throws IOException { 33 | channel.write(new NettyInteractiveResponse(type, t).response()); 34 | } 35 | 36 | @Override 37 | public void sendResponse(String type, XContentBuilder builder) throws IOException { 38 | channel.write(new NettyInteractiveResponse(type, builder).response()); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/NettyInteractiveRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty; 2 | 3 | import org.elasticsearch.common.unit.TimeValue; 4 | import org.elasticsearch.common.xcontent.XContentBuilder; 5 | import org.xbib.elasticsearch.websocket.InteractiveRequest; 6 | import org.xbib.elasticsearch.websocket.client.WebSocketClient; 7 | import org.xbib.elasticsearch.websocket.client.WebSocketClientRequest; 8 | 9 | import java.io.IOException; 10 | import java.util.Map; 11 | 12 | /** 13 | * Netty implemenation for an interactive request. 14 | * At the same time, this class serves as an implementation 15 | * for a WebSocket client request. 16 | */ 17 | public class NettyInteractiveRequest implements InteractiveRequest, WebSocketClientRequest { 18 | 19 | protected String type; 20 | 21 | protected Map data; 22 | 23 | protected XContentBuilder builder; 24 | 25 | public NettyInteractiveRequest() { 26 | } 27 | 28 | public NettyInteractiveRequest(Map data) { 29 | this.data = data; 30 | } 31 | 32 | @Override 33 | public NettyInteractiveRequest type(String type) { 34 | this.type = type; 35 | return this; 36 | } 37 | 38 | @Override 39 | public NettyInteractiveRequest data(XContentBuilder builder) { 40 | this.builder = builder; 41 | return this; 42 | } 43 | 44 | @Override 45 | public void send(WebSocketClient client) throws IOException { 46 | client.send(new NettyInteractiveResponse(type, builder).response()); 47 | } 48 | 49 | @Override 50 | public Map asMap() { 51 | return data; 52 | } 53 | 54 | @Override 55 | public boolean hasParam(String key) { 56 | return data.containsKey(key); 57 | } 58 | 59 | @Override 60 | public Object param(String key) { 61 | return data.get(key); 62 | } 63 | 64 | @Override 65 | public String paramAsString(String key) { 66 | Object o = param(key); 67 | return o != null ? o.toString() : null; 68 | } 69 | 70 | @Override 71 | public String paramAsString(String key, String defaultValue) { 72 | Object o = param(key); 73 | return o != null ? o.toString() : defaultValue; 74 | } 75 | 76 | @Override 77 | public long paramAsLong(String key) { 78 | Object o = param(key); 79 | try { 80 | return o != null ? Long.parseLong(o.toString()) : null; 81 | } catch (NumberFormatException e) { 82 | return 0L; 83 | } 84 | } 85 | 86 | @Override 87 | public long paramAsLong(String key, long defaultValue) { 88 | Object o = param(key); 89 | try { 90 | return o != null ? Long.parseLong(o.toString()) : defaultValue; 91 | } catch (NumberFormatException e) { 92 | return defaultValue; 93 | } 94 | } 95 | 96 | @Override 97 | public boolean paramAsBoolean(String key) { 98 | Object o = param(key); 99 | return o != null ? Boolean.getBoolean(o.toString()) : null; 100 | } 101 | 102 | @Override 103 | public boolean paramAsBoolean(String key, boolean defaultValue) { 104 | Object o = param(key); 105 | return o != null ? Boolean.getBoolean(o.toString()) : defaultValue; 106 | } 107 | 108 | @Override 109 | public TimeValue paramAsTime(String key) { 110 | Object o = param(key); 111 | return o != null ? TimeValue.parseTimeValue(o.toString(), null) : null; 112 | } 113 | 114 | @Override 115 | public TimeValue paramAsTime(String key, TimeValue defaultValue) { 116 | Object o = param(key); 117 | return o != null ? TimeValue.parseTimeValue(o.toString(), defaultValue) : defaultValue; 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/NettyInteractiveResponse.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty; 2 | 3 | import org.elasticsearch.common.xcontent.XContentBuilder; 4 | import org.jboss.netty.handler.codec.http.websocketx.TextWebSocketFrame; 5 | import org.xbib.elasticsearch.websocket.InteractiveResponse; 6 | 7 | import java.io.IOException; 8 | import java.util.Map; 9 | 10 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 11 | 12 | /** 13 | * Netty implementation of an interactive response 14 | */ 15 | public class NettyInteractiveResponse implements InteractiveResponse { 16 | 17 | private final String type; 18 | 19 | private final TextWebSocketFrame response; 20 | 21 | public NettyInteractiveResponse(String type, XContentBuilder builder) throws IOException { 22 | this.type = type; 23 | XContentBuilder responseBuilder = jsonBuilder() 24 | .startObject().field("success", true).field("type", type); 25 | if (builder != null) { 26 | responseBuilder.rawField("data", builder.bytes()); 27 | } 28 | responseBuilder.endObject(); 29 | this.response = new TextWebSocketFrame(responseBuilder.string()); 30 | } 31 | 32 | public NettyInteractiveResponse(String type, Map map) throws IOException { 33 | this.type = type; 34 | XContentBuilder responseBuilder = jsonBuilder(); 35 | responseBuilder.startObject().field("success", true).field("type", type).field("data", map).endObject(); 36 | this.response = new TextWebSocketFrame(responseBuilder.string()); 37 | } 38 | 39 | public NettyInteractiveResponse(String type, Throwable t) { 40 | this.type = type; 41 | this.response = new TextWebSocketFrame("{\"success\":false,\"type\":\"" + type + "\",\"error\":\"" + t.getMessage() + "\""); 42 | } 43 | 44 | @Override 45 | public String type() { 46 | return type; 47 | } 48 | 49 | /** 50 | * The response frame with content, ready for writing to a Channel 51 | * 52 | * @return a TextWebSocketFrame 53 | * @throws IOException if response fails 54 | */ 55 | @Override 56 | public TextWebSocketFrame response() throws IOException { 57 | return response; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/NettyWebSocketRequestHandler.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty; 2 | 3 | import org.jboss.netty.channel.ChannelHandlerContext; 4 | import org.jboss.netty.channel.ExceptionEvent; 5 | import org.jboss.netty.channel.MessageEvent; 6 | import org.jboss.netty.handler.codec.http.HttpRequest; 7 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 8 | 9 | /** 10 | * Netty implementation for a WebSocket request handler. 11 | * It is based on the HTTP request handler. 12 | */ 13 | public class NettyWebSocketRequestHandler extends NettyHttpRequestHandler { 14 | 15 | public NettyWebSocketRequestHandler(NettyWebSocketServerTransport serverTransport) { 16 | super(serverTransport); 17 | } 18 | 19 | @Override 20 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception { 21 | Object msg = e.getMessage(); 22 | if (msg instanceof HttpRequest) { 23 | super.messageReceived(ctx, e); 24 | } else if (msg instanceof WebSocketFrame) { 25 | serverTransport.frame(handshaker, (WebSocketFrame) msg, ctx); 26 | } 27 | } 28 | 29 | @Override 30 | public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception { 31 | serverTransport.exceptionCaught(ctx, e); 32 | e.getChannel().close(); 33 | } 34 | 35 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/NettyWebSocketServerPipelineFactory.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty; 2 | 3 | import org.jboss.netty.channel.ChannelPipeline; 4 | 5 | /** 6 | * Netty implementation for a WebSocket server pipeline factory. 7 | * It is based on the HTTP server pipeline factory. 8 | */ 9 | public class NettyWebSocketServerPipelineFactory extends NettyHttpServerPipelineFactory { 10 | 11 | private final NettyWebSocketRequestHandler handler; 12 | 13 | public NettyWebSocketServerPipelineFactory(NettyWebSocketServerTransport transport) { 14 | super(transport); 15 | this.handler = new NettyWebSocketRequestHandler(transport); 16 | } 17 | 18 | @Override 19 | public ChannelPipeline getPipeline() throws Exception { 20 | ChannelPipeline pipeline = super.getPipeline(); 21 | pipeline.replace("handler", "handler", handler); 22 | return pipeline; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/NettyWebSocketServerTransportModule.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty; 2 | 3 | import org.elasticsearch.common.inject.AbstractModule; 4 | import org.xbib.elasticsearch.http.HttpServerTransport; 5 | 6 | /** 7 | * Module for HttpServerModule. 8 | */ 9 | public class NettyWebSocketServerTransportModule extends AbstractModule { 10 | 11 | @Override 12 | protected void configure() { 13 | bind(HttpServerTransport.class).to(NettyWebSocketServerTransport.class).asEagerSingleton(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/client/NettyWebSocketBulkRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty.client; 2 | 3 | import org.elasticsearch.common.xcontent.XContentBuilder; 4 | import org.xbib.elasticsearch.websocket.client.WebSocketClient; 5 | import org.xbib.elasticsearch.websocket.client.WebSocketClientBulkRequest; 6 | import org.xbib.elasticsearch.http.netty.NettyInteractiveRequest; 7 | import org.xbib.elasticsearch.http.netty.NettyInteractiveResponse; 8 | 9 | import java.io.IOException; 10 | 11 | /** 12 | * Netty bulk request convenience class. 13 | */ 14 | public class NettyWebSocketBulkRequest 15 | extends NettyInteractiveRequest 16 | implements WebSocketClientBulkRequest { 17 | 18 | public NettyWebSocketBulkRequest(String type) { 19 | super.type(type); 20 | } 21 | 22 | @Override 23 | public NettyWebSocketBulkRequest data(XContentBuilder builder) { 24 | super.data(builder); 25 | return this; 26 | } 27 | 28 | @Override 29 | public void send(WebSocketClient client) throws IOException { 30 | client.send(new NettyInteractiveResponse(super.type, super.builder).response()); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/client/NettyWebSocketClient.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty.client; 2 | 3 | import org.jboss.netty.channel.ChannelFuture; 4 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 5 | import org.xbib.elasticsearch.websocket.client.WebSocketClient; 6 | 7 | /** 8 | * A Websocket client. 9 | */ 10 | public interface NettyWebSocketClient extends WebSocketClient { 11 | 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/client/NettyWebSocketClientFactory.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty.client; 2 | 3 | import org.jboss.netty.bootstrap.ClientBootstrap; 4 | import org.jboss.netty.channel.ChannelPipeline; 5 | import org.jboss.netty.channel.ChannelPipelineFactory; 6 | import org.jboss.netty.channel.Channels; 7 | import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; 8 | import org.jboss.netty.handler.codec.http.HttpRequestEncoder; 9 | import org.jboss.netty.handler.codec.http.HttpResponseDecoder; 10 | import org.xbib.elasticsearch.websocket.client.WebSocketActionListener; 11 | import org.xbib.elasticsearch.websocket.client.WebSocketClient; 12 | import org.xbib.elasticsearch.websocket.client.WebSocketClientBulkRequest; 13 | import org.xbib.elasticsearch.websocket.client.WebSocketClientFactory; 14 | import org.xbib.elasticsearch.websocket.client.WebSocketClientRequest; 15 | import org.xbib.elasticsearch.http.netty.NettyInteractiveRequest; 16 | 17 | import java.net.URI; 18 | import java.util.concurrent.Executors; 19 | 20 | /** 21 | * A factory for creating Websocket clients. The entry point for creating and 22 | * connecting a client. Can and should be used to create multiple instances. 23 | * Extended for Websocket client request methods. 24 | */ 25 | public class NettyWebSocketClientFactory implements WebSocketClientFactory { 26 | 27 | private NioClientSocketChannelFactory socketChannelFactory = new NioClientSocketChannelFactory( 28 | Executors.newCachedThreadPool(), 29 | Executors.newCachedThreadPool()); 30 | 31 | /** 32 | * Create a new WebSocket client 33 | * 34 | * @param url URL to connect to. 35 | * @param listener Callback interface to receive events 36 | * @return A WebSocket client. Call {@link NettyWebSocketClient#connect()} to 37 | * connect. 38 | */ 39 | @Override 40 | public WebSocketClient newClient(final URI url, final WebSocketActionListener listener) { 41 | ClientBootstrap bootstrap = new ClientBootstrap(socketChannelFactory); 42 | 43 | String protocol = url.getScheme(); 44 | if (!protocol.equals("ws") && !protocol.equals("wss")) { 45 | throw new IllegalArgumentException("unsupported protocol: " + protocol); 46 | } 47 | 48 | final NettyWebSocketClientHandler clientHandler = new NettyWebSocketClientHandler(bootstrap, url, listener); 49 | 50 | bootstrap.setPipelineFactory(new ChannelPipelineFactory() { 51 | @Override 52 | public ChannelPipeline getPipeline() throws Exception { 53 | ChannelPipeline pipeline = Channels.pipeline(); 54 | pipeline.addLast("decoder", new HttpResponseDecoder()); 55 | pipeline.addLast("encoder", new HttpRequestEncoder()); 56 | pipeline.addLast("ws-handler", clientHandler); 57 | return pipeline; 58 | } 59 | }); 60 | 61 | return clientHandler; 62 | } 63 | 64 | @Override 65 | public void shutdown() { 66 | socketChannelFactory.releaseExternalResources(); 67 | } 68 | 69 | @Override 70 | public WebSocketClientRequest newRequest() { 71 | return new NettyInteractiveRequest(); 72 | } 73 | 74 | @Override 75 | public WebSocketClientBulkRequest indexRequest() { 76 | return new NettyWebSocketBulkRequest("index"); 77 | } 78 | 79 | @Override 80 | public WebSocketClientBulkRequest deleteRequest() { 81 | return new NettyWebSocketBulkRequest("delete"); 82 | } 83 | 84 | @Override 85 | public WebSocketClientRequest flushRequest() { 86 | return new NettyInteractiveRequest().type("flush"); 87 | } 88 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/client/NettyWebSocketClientHandler.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty.client; 2 | 3 | import org.jboss.netty.bootstrap.ClientBootstrap; 4 | import org.jboss.netty.channel.Channel; 5 | import org.jboss.netty.channel.ChannelFuture; 6 | import org.jboss.netty.channel.ChannelHandlerContext; 7 | import org.jboss.netty.channel.ChannelStateEvent; 8 | import org.jboss.netty.channel.ExceptionEvent; 9 | import org.jboss.netty.channel.MessageEvent; 10 | import org.jboss.netty.channel.SimpleChannelUpstreamHandler; 11 | import org.jboss.netty.handler.codec.http.DefaultHttpRequest; 12 | import org.jboss.netty.handler.codec.http.HttpHeaders.Names; 13 | import org.jboss.netty.handler.codec.http.HttpHeaders.Values; 14 | import org.jboss.netty.handler.codec.http.HttpMethod; 15 | import org.jboss.netty.handler.codec.http.HttpRequest; 16 | import org.jboss.netty.handler.codec.http.HttpResponse; 17 | import org.jboss.netty.handler.codec.http.HttpResponseStatus; 18 | import org.jboss.netty.handler.codec.http.HttpVersion; 19 | import org.jboss.netty.handler.codec.http.websocketx.WebSocket00FrameDecoder; 20 | import org.jboss.netty.handler.codec.http.websocketx.WebSocket00FrameEncoder; 21 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 22 | import org.jboss.netty.util.CharsetUtil; 23 | import org.xbib.elasticsearch.websocket.client.WebSocketActionListener; 24 | import org.xbib.elasticsearch.websocket.client.WebSocketClient; 25 | 26 | import java.net.InetSocketAddress; 27 | import java.net.URI; 28 | 29 | /** 30 | * Handles socket communication for a connected WebSocket client. Not intended 31 | * for end-users. Please use {@link NettyWebSocketClient} for controlling your client. 32 | */ 33 | public class NettyWebSocketClientHandler extends SimpleChannelUpstreamHandler 34 | implements WebSocketClient { 35 | 36 | private final ClientBootstrap bootstrap; 37 | private final URI url; 38 | private final WebSocketActionListener listener; 39 | private boolean handshakeCompleted = false; 40 | private Channel channel; 41 | 42 | public NettyWebSocketClientHandler(ClientBootstrap bootstrap, URI url, WebSocketActionListener listener) { 43 | this.bootstrap = bootstrap; 44 | this.url = url; 45 | this.listener = listener; 46 | } 47 | 48 | @Override 49 | public Channel channel() { 50 | return channel; 51 | } 52 | 53 | @Override 54 | public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent event) throws Exception { 55 | String path = url.getPath(); 56 | if (url.getQuery() != null && url.getQuery().length() > 0) { 57 | path = url.getPath() + "?" + url.getQuery(); 58 | } 59 | HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, path); 60 | request.headers().add(Names.UPGRADE, Values.WEBSOCKET); 61 | request.headers().add(Names.CONNECTION, Values.UPGRADE); 62 | request.headers().add(Names.HOST, url.getHost()); 63 | request.headers().add(Names.ORIGIN, "http://" + url.getHost()); 64 | event.getChannel().write(request); 65 | ctx.getPipeline().replace("encoder", "ws-encoder", new WebSocket00FrameEncoder()); 66 | this.channel = event.getChannel(); 67 | } 68 | 69 | @Override 70 | public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent event) throws Exception { 71 | listener.onDisconnect(this); 72 | handshakeCompleted = false; 73 | channel = null; 74 | } 75 | 76 | @Override 77 | public void messageReceived(ChannelHandlerContext ctx, MessageEvent event) throws Exception { 78 | if (!handshakeCompleted) { 79 | HttpResponse response = (HttpResponse) event.getMessage(); 80 | final HttpResponseStatus status = new HttpResponseStatus(101, "Web Socket Protocol Handshake"); 81 | final boolean validStatus = response.getStatus().equals(status); 82 | final boolean validUpgrade = response.headers().get(Names.UPGRADE).equals(Values.WEBSOCKET); 83 | final boolean validConnection = response.headers().get(Names.CONNECTION).equals(Values.UPGRADE); 84 | if (!validStatus || !validUpgrade || !validConnection) { 85 | throw new NettyWebSocketException("Invalid handshake response"); 86 | } 87 | handshakeCompleted = true; 88 | ctx.getPipeline().replace("decoder", "ws-decoder", new WebSocket00FrameDecoder()); 89 | listener.onConnect(this); 90 | return; 91 | } 92 | if (event.getMessage() instanceof HttpResponse) { 93 | HttpResponse response = (HttpResponse) event.getMessage(); 94 | throw new NettyWebSocketException("Unexpected HttpResponse (status=" + response.getStatus() + ", content=" + response.getContent().toString(CharsetUtil.UTF_8) + ")"); 95 | } 96 | WebSocketFrame frame = (WebSocketFrame) event.getMessage(); 97 | listener.onMessage(this, frame); 98 | } 99 | 100 | @Override 101 | public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception { 102 | final Throwable t = e.getCause(); 103 | listener.onError(t); 104 | e.getChannel().close(); 105 | } 106 | 107 | @Override 108 | public ChannelFuture connect() { 109 | return bootstrap.connect(new InetSocketAddress(url.getHost(), url.getPort())); 110 | } 111 | 112 | @Override 113 | public ChannelFuture disconnect() { 114 | if (channel == null) { 115 | return null; 116 | } 117 | return channel.close(); 118 | } 119 | 120 | @Override 121 | public ChannelFuture send(WebSocketFrame frame) { 122 | if (channel == null) { 123 | return null; 124 | } 125 | return channel.write(frame); 126 | } 127 | 128 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/http/netty/client/NettyWebSocketException.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.http.netty.client; 2 | 3 | import java.io.IOException; 4 | 5 | /** 6 | * A WebSocket related exception 7 | */ 8 | public class NettyWebSocketException extends IOException { 9 | 10 | public NettyWebSocketException(String s) { 11 | super(s); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/plugin/websocket/Build.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.websocket; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.InputStream; 5 | import java.io.StringReader; 6 | import java.net.URL; 7 | import java.util.Enumeration; 8 | import java.util.Properties; 9 | 10 | public class Build { 11 | 12 | private static final Build INSTANCE; 13 | 14 | static { 15 | String version = "NA"; 16 | String hash = "NA"; 17 | String hashShort = "NA"; 18 | String timestamp = "NA"; 19 | String date = "NA"; 20 | 21 | try { 22 | String pluginName = WebSocketPlugin.class.getName(); 23 | Enumeration e = WebSocketPlugin.class.getClassLoader().getResources("es-plugin.properties"); 24 | while (e.hasMoreElements()) { 25 | URL url = e.nextElement(); 26 | InputStream in = url.openStream(); 27 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 28 | byte[] buffer = new byte[1024]; 29 | int len; 30 | while ((len = in.read(buffer)) != -1) { 31 | out.write(buffer, 0, len); 32 | } 33 | in.close(); 34 | Properties props = new Properties(); 35 | props.load(new StringReader(new String(out.toByteArray()))); 36 | String plugin = props.getProperty("plugin"); 37 | if (pluginName.equals(plugin)) { 38 | version = props.getProperty("version"); 39 | hash = props.getProperty("hash"); 40 | if (!"NA".equals(hash)) { 41 | hashShort = hash.substring(0, 7); 42 | } 43 | timestamp = props.getProperty("timestamp"); 44 | date = props.getProperty("date"); 45 | } 46 | } 47 | } catch (Throwable e) { 48 | // just ignore... 49 | } 50 | INSTANCE = new Build(version, hash, hashShort, timestamp, date); 51 | } 52 | 53 | private String version; 54 | 55 | private String hash; 56 | 57 | private String hashShort; 58 | 59 | private String timestamp; 60 | 61 | private String date; 62 | 63 | Build(String version, String hash, String hashShort, String timestamp, String date) { 64 | this.version = version; 65 | this.hash = hash; 66 | this.hashShort = hashShort; 67 | this.timestamp = timestamp; 68 | this.date = date; 69 | } 70 | 71 | public static Build getInstance() { 72 | return INSTANCE; 73 | } 74 | 75 | public String getVersion() { 76 | return version; 77 | } 78 | 79 | public String getHash() { 80 | return hash; 81 | } 82 | 83 | public String getShortHash() { 84 | return hashShort; 85 | } 86 | 87 | public String getTimestamp() { 88 | return timestamp; 89 | } 90 | 91 | public String getDate() { 92 | return date; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/plugin/websocket/WebSocketModule.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.websocket; 2 | 3 | import org.elasticsearch.common.collect.Lists; 4 | import org.elasticsearch.common.inject.AbstractModule; 5 | import org.elasticsearch.common.settings.Settings; 6 | import org.xbib.elasticsearch.rest.HttpPatchRestController; 7 | import org.xbib.elasticsearch.websocket.BaseInteractiveHandler; 8 | import org.xbib.elasticsearch.websocket.InteractiveActionModule; 9 | import org.xbib.elasticsearch.websocket.InteractiveController; 10 | 11 | import java.util.List; 12 | 13 | public class WebSocketModule extends AbstractModule { 14 | 15 | private final Settings settings; 16 | 17 | private List> webSocketActions = Lists.newArrayList(); 18 | 19 | public WebSocketModule(Settings settings) { 20 | this.settings = settings; 21 | } 22 | 23 | public void addInteractiveAction(Class action) { 24 | webSocketActions.add(action); 25 | } 26 | 27 | @Override 28 | protected void configure() { 29 | bind(InteractiveController.class).asEagerSingleton(); 30 | new InteractiveActionModule(webSocketActions).configure(binder()); 31 | bind(HttpPatchRestController.class).asEagerSingleton(); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/plugin/websocket/WebSocketPlugin.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.websocket; 2 | 3 | import org.elasticsearch.action.ActionModule; 4 | import org.elasticsearch.common.component.LifecycleComponent; 5 | import org.elasticsearch.common.inject.Module; 6 | import org.elasticsearch.common.settings.Settings; 7 | import org.elasticsearch.plugins.AbstractPlugin; 8 | import org.elasticsearch.rest.RestModule; 9 | import org.xbib.elasticsearch.action.cluster.admin.websocket.TransportWebsocketInfoAction; 10 | import org.xbib.elasticsearch.action.cluster.admin.websocket.WebsocketInfoAction; 11 | import org.xbib.elasticsearch.action.websocket.pubsub.Checkpointer; 12 | import org.xbib.elasticsearch.http.HttpServer; 13 | import org.xbib.elasticsearch.http.HttpServerModule; 14 | import org.xbib.elasticsearch.rest.action.websocket.RestPublishAction; 15 | import org.xbib.elasticsearch.rest.action.websocket.RestUnsubscribeAction; 16 | 17 | import java.util.Collection; 18 | 19 | import static org.elasticsearch.common.collect.Lists.newArrayList; 20 | 21 | /** 22 | * Websocket plugin 23 | */ 24 | public class WebSocketPlugin extends AbstractPlugin { 25 | 26 | private final Settings settings; 27 | 28 | public WebSocketPlugin(Settings settings) { 29 | this.settings = settings; 30 | } 31 | 32 | @Override 33 | public String name() { 34 | return "transport-websocket-" 35 | + Build.getInstance().getVersion() + "-" 36 | + Build.getInstance().getShortHash(); 37 | } 38 | 39 | @Override 40 | public String description() { 41 | return "Websocket transport plugin"; 42 | } 43 | 44 | @Override 45 | public Collection> modules() { 46 | Collection> modules = newArrayList(); 47 | if (settings.getAsBoolean("websocket.enabled", true)) { 48 | modules.add(HttpServerModule.class); 49 | modules.add(WebSocketModule.class); 50 | } 51 | return modules; 52 | } 53 | 54 | @Override 55 | public Collection> services() { 56 | Collection> services = newArrayList(); 57 | if (settings.getAsBoolean("websocket.enabled", true)) { 58 | services.add(HttpServer.class); 59 | services.add(Checkpointer.class); 60 | } 61 | return services; 62 | } 63 | 64 | public void onModule(ActionModule module) { 65 | module.registerAction(WebsocketInfoAction.INSTANCE, TransportWebsocketInfoAction.class); 66 | } 67 | 68 | public void onModule(RestModule module) { 69 | module.addRestAction(RestPublishAction.class); 70 | module.addRestAction(RestUnsubscribeAction.class); 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/rest/HttpPatchRestController.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.rest; 2 | 3 | import org.elasticsearch.ElasticsearchException; 4 | import org.elasticsearch.common.component.AbstractLifecycleComponent; 5 | import org.elasticsearch.common.inject.Inject; 6 | import org.elasticsearch.common.path.PathTrie; 7 | import org.elasticsearch.common.settings.Settings; 8 | import org.elasticsearch.rest.BytesRestResponse; 9 | import org.elasticsearch.rest.RestChannel; 10 | import org.elasticsearch.rest.RestHandler; 11 | import org.elasticsearch.rest.RestRequest; 12 | import org.elasticsearch.rest.support.RestUtils; 13 | import org.xbib.elasticsearch.http.netty.NettyHttpRequest; 14 | 15 | public class HttpPatchRestController extends AbstractLifecycleComponent { 16 | 17 | private final PathTrie patchHandlers = new PathTrie<>(RestUtils.REST_DECODER); 18 | 19 | @Inject 20 | public HttpPatchRestController(Settings settings) { 21 | super(settings); 22 | } 23 | 24 | @Override 25 | protected void doStart() throws ElasticsearchException { 26 | 27 | } 28 | 29 | @Override 30 | protected void doStop() throws ElasticsearchException { 31 | 32 | } 33 | 34 | @Override 35 | protected void doClose() throws ElasticsearchException { 36 | 37 | } 38 | 39 | public void registerHandler(String method, String path, RestHandler handler) { 40 | if ("PATCH".equalsIgnoreCase(method)) { 41 | patchHandlers.insert(path, handler); 42 | } 43 | } 44 | 45 | public void dispatchRequest(final RestRequest request, final RestChannel channel) { 46 | try { 47 | if (request instanceof NettyHttpRequest) { 48 | NettyHttpRequest nettyHttpRequest = (NettyHttpRequest)request; 49 | if ("PATCH".equalsIgnoreCase(nettyHttpRequest.getMethod())) { 50 | RestHandler handler = patchHandlers.retrieve(request.rawPath()); 51 | if (handler != null) { 52 | handler.handleRequest(request, channel); 53 | } 54 | } 55 | } 56 | } catch (Throwable e) { 57 | try { 58 | channel.sendResponse(new BytesRestResponse(channel, e)); 59 | } catch (Throwable e1) { 60 | logger.error("failed to send failure response for uri [" + request.uri() + "]", e1); 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/rest/RestXContentBuilder.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.rest; 2 | 3 | import org.elasticsearch.common.io.stream.BytesStreamOutput; 4 | import org.elasticsearch.common.xcontent.XContentBuilder; 5 | import org.elasticsearch.common.xcontent.XContentFactory; 6 | import org.elasticsearch.common.xcontent.XContentType; 7 | import org.elasticsearch.rest.RestRequest; 8 | 9 | import java.io.IOException; 10 | 11 | public class RestXContentBuilder { 12 | 13 | public static XContentBuilder restContentBuilder(RestRequest request) throws IOException { 14 | XContentType contentType = XContentType.fromRestContentType(request.param("format", request.header("Content-Type"))); 15 | if (contentType == null) { 16 | // default to JSON 17 | contentType = XContentType.JSON; 18 | } 19 | XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), 20 | new BytesStreamOutput()); 21 | if (request.paramAsBoolean("pretty", false)) { 22 | builder.prettyPrint().lfAtEnd(); 23 | } 24 | String casing = request.param("case"); 25 | if (casing != null && "camelCase".equals(casing)) { 26 | builder.fieldCaseConversion(XContentBuilder.FieldCaseConversion.CAMELCASE); 27 | } else { 28 | // we expect all REST interfaces to write results in underscore casing, so 29 | // no need for double casing 30 | builder.fieldCaseConversion(XContentBuilder.FieldCaseConversion.NONE); 31 | } 32 | return builder; 33 | } 34 | 35 | public static XContentBuilder emptyBuilder(RestRequest request) throws IOException { 36 | return restContentBuilder(request).startObject().endObject(); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/rest/XContentRestResponse.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.rest; 2 | 3 | import org.elasticsearch.common.xcontent.XContentBuilder; 4 | import org.elasticsearch.rest.BytesRestResponse; 5 | import org.elasticsearch.rest.RestRequest; 6 | import org.elasticsearch.rest.RestStatus; 7 | 8 | import java.io.IOException; 9 | 10 | public class XContentRestResponse extends BytesRestResponse { 11 | 12 | public XContentRestResponse(RestRequest request, RestStatus status, XContentBuilder builder) throws IOException { 13 | super(status, builder); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/rest/XContentThrowableRestResponse.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.rest; 2 | 3 | import org.elasticsearch.ElasticsearchException; 4 | import org.elasticsearch.common.xcontent.XContentBuilder; 5 | import org.elasticsearch.rest.RestRequest; 6 | import org.elasticsearch.rest.RestStatus; 7 | 8 | import java.io.IOException; 9 | 10 | import static org.elasticsearch.ExceptionsHelper.detailedMessage; 11 | import static org.xbib.elasticsearch.rest.RestXContentBuilder.restContentBuilder; 12 | 13 | public class XContentThrowableRestResponse extends XContentRestResponse { 14 | 15 | public XContentThrowableRestResponse(RestRequest request, Throwable t) throws IOException { 16 | this(request, ((t instanceof ElasticsearchException) ? ((ElasticsearchException) t).status() : RestStatus.INTERNAL_SERVER_ERROR), t); 17 | } 18 | 19 | public XContentThrowableRestResponse(RestRequest request, RestStatus status, Throwable t) throws IOException { 20 | super(request, status, convert(request, status, t)); 21 | } 22 | 23 | private static XContentBuilder convert(RestRequest request, RestStatus status, Throwable t) throws IOException { 24 | XContentBuilder builder = restContentBuilder(request).startObject() 25 | .field("error", detailedMessage(t)) 26 | .field("status", status.getStatus()); 27 | if (t != null && request.paramAsBoolean("error_trace", false)) { 28 | builder.startObject("error_trace"); 29 | boolean first = true; 30 | while (t != null) { 31 | if (!first) { 32 | builder.startObject("cause"); 33 | } 34 | buildThrowable(t, builder); 35 | if (!first) { 36 | builder.endObject(); 37 | } 38 | t = t.getCause(); 39 | first = false; 40 | } 41 | builder.endObject(); 42 | } 43 | builder.endObject(); 44 | return builder; 45 | } 46 | 47 | private static void buildThrowable(Throwable t, XContentBuilder builder) throws IOException { 48 | builder.field("message", t.getMessage()); 49 | for (StackTraceElement stElement : t.getStackTrace()) { 50 | builder.startObject("at") 51 | .field("class", stElement.getClassName()) 52 | .field("method", stElement.getMethodName()); 53 | if (stElement.getFileName() != null) { 54 | builder.field("file", stElement.getFileName()); 55 | } 56 | if (stElement.getLineNumber() >= 0) { 57 | builder.field("line", stElement.getLineNumber()); 58 | } 59 | builder.endObject(); 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/rest/action/websocket/RestPublishAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.rest.action.websocket; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.index.IndexResponse; 5 | import org.elasticsearch.action.search.SearchResponse; 6 | import org.elasticsearch.action.search.SearchType; 7 | import org.elasticsearch.client.Client; 8 | import org.elasticsearch.common.inject.Inject; 9 | import org.elasticsearch.common.settings.Settings; 10 | import org.elasticsearch.common.unit.TimeValue; 11 | import org.elasticsearch.common.xcontent.XContentBuilder; 12 | import org.elasticsearch.common.xcontent.XContentFactory; 13 | import org.elasticsearch.common.xcontent.XContentParser; 14 | import org.elasticsearch.index.query.QueryBuilder; 15 | import org.elasticsearch.rest.BaseRestHandler; 16 | import org.elasticsearch.rest.RestChannel; 17 | import org.elasticsearch.rest.RestController; 18 | import org.elasticsearch.rest.RestRequest; 19 | import org.elasticsearch.search.SearchHit; 20 | import org.elasticsearch.search.SearchHitField; 21 | import org.jboss.netty.channel.Channel; 22 | import org.xbib.elasticsearch.action.websocket.pubsub.Checkpointer; 23 | import org.xbib.elasticsearch.action.websocket.pubsub.PubSubIndexName; 24 | import org.xbib.elasticsearch.http.HttpServerTransport; 25 | import org.xbib.elasticsearch.http.netty.NettyInteractiveResponse; 26 | import org.xbib.elasticsearch.rest.XContentRestResponse; 27 | import org.xbib.elasticsearch.rest.XContentThrowableRestResponse; 28 | 29 | import java.io.IOException; 30 | import java.util.Map; 31 | 32 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 33 | import static org.elasticsearch.index.query.QueryBuilders.termQuery; 34 | import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; 35 | import static org.elasticsearch.rest.RestStatus.OK; 36 | import static org.xbib.elasticsearch.rest.RestXContentBuilder.restContentBuilder; 37 | 38 | /** 39 | * Publish action for REST 40 | */ 41 | public class RestPublishAction extends BaseRestHandler { 42 | 43 | private final static String TYPE = "publish"; 44 | 45 | private final String pubSubIndexName; 46 | 47 | private final HttpServerTransport transport; 48 | 49 | private final Checkpointer service; 50 | 51 | @Inject 52 | public RestPublishAction(Settings settings, Client client, 53 | RestController restController, 54 | HttpServerTransport transport, 55 | Checkpointer service) { 56 | super(settings, restController, client); 57 | this.pubSubIndexName = PubSubIndexName.Conf.indexName(settings); 58 | this.transport = transport; 59 | this.service = service; 60 | restController.registerHandler(RestRequest.Method.GET, "/_publish", this); 61 | restController.registerHandler(RestRequest.Method.POST, "/_publish", this); 62 | } 63 | 64 | @Override 65 | public void handleRequest(final RestRequest request, final RestChannel channel, Client client) { 66 | String topic = request.hasParam("topic") ? request.param("topic") : "*"; 67 | try { 68 | final XContentBuilder messageBuilder = createPublishMessage(request); 69 | client.prepareIndex() 70 | .setIndex(pubSubIndexName) 71 | .setType(TYPE) 72 | .setSource(messageBuilder) 73 | .setRefresh(request.paramAsBoolean("refresh", true)) 74 | .execute(new ActionListener() { 75 | @Override 76 | public void onResponse(IndexResponse response) { 77 | try { 78 | XContentBuilder builder = restContentBuilder(request); 79 | builder.startObject().field("ok", true).field("id", response.getId()).endObject(); 80 | channel.sendResponse(new XContentRestResponse(request, OK, builder)); 81 | } catch (Exception e) { 82 | onFailure(e); 83 | } 84 | } 85 | 86 | @Override 87 | public void onFailure(Throwable e) { 88 | try { 89 | logger.error("Error processing publish request", e); 90 | channel.sendResponse(new XContentThrowableRestResponse(request, e)); 91 | } catch (IOException e1) { 92 | logger.error("Failed to send failure response", e1); 93 | } 94 | } 95 | }); 96 | // push phase - scroll over subscribers for this topic currently connected 97 | QueryBuilder queryBuilder = termQuery("topic", topic); 98 | SearchResponse searchResponse = client.prepareSearch() 99 | .setIndices(pubSubIndexName) 100 | .setTypes("subscribe") 101 | .setSearchType(SearchType.SCAN) 102 | .setScroll(new TimeValue(60000)) 103 | .setQuery(queryBuilder) 104 | .addField("subscriber.channel") 105 | .setSize(100) 106 | .execute().actionGet(); 107 | messageBuilder.close(); 108 | service.checkpoint(topic); 109 | // push phase - write the message to the subscribers. We have 60 seconds per 100 subscribers. 110 | while (true) { 111 | searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()) 112 | .setScroll(new TimeValue(60000)) 113 | .execute().actionGet(); 114 | for (SearchHit hit : searchResponse.getHits()) { 115 | service.checkpoint(hit.getId()); 116 | SearchHitField channelField = hit.field("subscriber.channel"); 117 | Map channelfieldMap = channelField.getValue(); 118 | Integer id = (Integer) channelfieldMap.get("id"); 119 | Channel ch = transport.channel(id); 120 | if (ch != null) { 121 | ch.write(new NettyInteractiveResponse("message", messageBuilder).response()); 122 | } 123 | } 124 | if (searchResponse.getHits().hits().length == 0) { 125 | break; 126 | } 127 | } 128 | service.flushCheckpoint(); 129 | } catch (Exception e) { 130 | try { 131 | XContentBuilder builder = restContentBuilder(request); 132 | channel.sendResponse(new XContentRestResponse(request, BAD_REQUEST, builder.startObject().field("error", e.getMessage()).endObject())); 133 | } catch (IOException e1) { 134 | logger.error("Failed to send failure response", e1); 135 | } 136 | } 137 | } 138 | 139 | private XContentBuilder createPublishMessage(RestRequest request) { 140 | try { 141 | Map map = null; 142 | String message = request.content().toUtf8(); 143 | XContentParser parser = null; 144 | try { 145 | parser = XContentFactory.xContent(message).createParser(message); 146 | map = parser.map(); 147 | } catch (Exception e) { 148 | logger.warn("unable to parse {}", message); 149 | } finally { 150 | parser = null; 151 | } 152 | return jsonBuilder().startObject() 153 | .field("timestamp", request.param("timestamp", Long.toString(System.currentTimeMillis()))) 154 | .field("message", map) 155 | .endObject(); 156 | } catch (IOException e) { 157 | return null; 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/rest/action/websocket/RestUnsubscribeAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.rest.action.websocket; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.delete.DeleteResponse; 5 | import org.elasticsearch.client.Client; 6 | import org.elasticsearch.common.inject.Inject; 7 | import org.elasticsearch.common.settings.Settings; 8 | import org.elasticsearch.common.xcontent.XContentBuilder; 9 | import org.elasticsearch.rest.BaseRestHandler; 10 | import org.elasticsearch.rest.RestChannel; 11 | import org.elasticsearch.rest.RestController; 12 | import org.elasticsearch.rest.RestRequest; 13 | import org.xbib.elasticsearch.rest.XContentRestResponse; 14 | import org.xbib.elasticsearch.rest.XContentThrowableRestResponse; 15 | import org.xbib.elasticsearch.action.websocket.pubsub.PubSubIndexName; 16 | 17 | import java.io.IOException; 18 | 19 | import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; 20 | import static org.elasticsearch.rest.RestStatus.OK; 21 | import static org.xbib.elasticsearch.rest.RestXContentBuilder.restContentBuilder; 22 | 23 | /** 24 | * Unsubscribe action for REST 25 | */ 26 | public class RestUnsubscribeAction extends BaseRestHandler { 27 | 28 | private final String pubSubIndexName; 29 | 30 | @Inject 31 | public RestUnsubscribeAction(Settings settings, Client client, 32 | RestController restController) { 33 | super(settings, restController, client); 34 | this.pubSubIndexName = PubSubIndexName.Conf.indexName(settings); 35 | restController.registerHandler(RestRequest.Method.GET, "/_unsubscribe", this); 36 | restController.registerHandler(RestRequest.Method.POST, "/_unsubscribe", this); 37 | } 38 | 39 | @Override 40 | public void handleRequest(final RestRequest request, final RestChannel channel, Client client) { 41 | String subscriberId = request.hasParam("subscriber") ? request.param("subscriber") : null; 42 | if (subscriberId == null) { 43 | try { 44 | channel.sendResponse(new XContentThrowableRestResponse(request, new IllegalArgumentException("no subscriber"))); 45 | } catch (IOException e) { 46 | logger.error("error while sending failure response", e); 47 | } 48 | return; 49 | } 50 | try { 51 | client.prepareDelete(pubSubIndexName, "subscribe", subscriberId) 52 | .execute(new ActionListener() { 53 | @Override 54 | public void onResponse(DeleteResponse response) { 55 | try { 56 | XContentBuilder builder = restContentBuilder(request); 57 | builder.startObject().field("ok", true).field("id", response.getId()).endObject(); 58 | channel.sendResponse(new XContentRestResponse(request, OK, builder)); 59 | } catch (Exception e) { 60 | onFailure(e); 61 | } 62 | } 63 | 64 | @Override 65 | public void onFailure(Throwable e) { 66 | try { 67 | logger.error("error while processing unsubscribe request", e); 68 | channel.sendResponse(new XContentThrowableRestResponse(request, e)); 69 | } catch (IOException e1) { 70 | logger.error("error while sending error response", e1); 71 | } 72 | } 73 | }); 74 | } catch (Exception e) { 75 | try { 76 | XContentBuilder builder = restContentBuilder(request); 77 | channel.sendResponse(new XContentRestResponse(request, BAD_REQUEST, builder.startObject().field("error", e.getMessage()).endObject())); 78 | } catch (IOException e1) { 79 | logger.error("exception while sending exception response", e1); 80 | } 81 | } 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/transport/netty/ChannelBufferStreamInput.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.transport.netty; 2 | 3 | import org.elasticsearch.common.bytes.BytesReference; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.jboss.netty.buffer.ChannelBuffer; 6 | import org.xbib.elasticsearch.common.bytes.ChannelBufferBytesReference; 7 | 8 | import java.io.EOFException; 9 | import java.io.IOException; 10 | 11 | /** 12 | * A Netty {@link org.jboss.netty.buffer.ChannelBuffer} based {@link org.elasticsearch.common.io.stream.StreamInput}. 13 | */ 14 | public class ChannelBufferStreamInput extends StreamInput { 15 | 16 | private final ChannelBuffer buffer; 17 | 18 | private final int startIndex; 19 | 20 | private final int endIndex; 21 | 22 | public ChannelBufferStreamInput(ChannelBuffer buffer) { 23 | this(buffer, buffer.readableBytes()); 24 | } 25 | 26 | public ChannelBufferStreamInput(ChannelBuffer buffer, int length) { 27 | if (length > buffer.readableBytes()) { 28 | throw new IndexOutOfBoundsException(); 29 | } 30 | this.buffer = buffer; 31 | startIndex = buffer.readerIndex(); 32 | endIndex = startIndex + length; 33 | buffer.markReaderIndex(); 34 | } 35 | 36 | @Override 37 | public BytesReference readBytesReference(int length) throws IOException { 38 | ChannelBufferBytesReference ref = new ChannelBufferBytesReference(buffer.slice(buffer.readerIndex(), length)); 39 | buffer.skipBytes(length); 40 | return ref; 41 | } 42 | 43 | @Override 44 | public int available() throws IOException { 45 | return endIndex - buffer.readerIndex(); 46 | } 47 | 48 | @Override 49 | public void mark(int readlimit) { 50 | buffer.markReaderIndex(); 51 | } 52 | 53 | @Override 54 | public boolean markSupported() { 55 | return true; 56 | } 57 | 58 | @Override 59 | public int read() throws IOException { 60 | if (available() == 0) { 61 | return -1; 62 | } 63 | return buffer.readByte() & 0xff; 64 | } 65 | 66 | @Override 67 | public int read(byte[] b, int off, int len) throws IOException { 68 | if (len == 0) { 69 | return 0; 70 | } 71 | int available = available(); 72 | if (available == 0) { 73 | return -1; 74 | } 75 | 76 | len = Math.min(available, len); 77 | buffer.readBytes(b, off, len); 78 | return len; 79 | } 80 | 81 | @Override 82 | public void reset() throws IOException { 83 | buffer.resetReaderIndex(); 84 | } 85 | 86 | @Override 87 | public long skip(long n) throws IOException { 88 | if (n > Integer.MAX_VALUE) { 89 | return skipBytes(Integer.MAX_VALUE); 90 | } else { 91 | return skipBytes((int) n); 92 | } 93 | } 94 | 95 | public int skipBytes(int n) throws IOException { 96 | int nBytes = Math.min(available(), n); 97 | buffer.skipBytes(nBytes); 98 | return nBytes; 99 | } 100 | 101 | 102 | @Override 103 | public byte readByte() throws IOException { 104 | return buffer.readByte(); 105 | } 106 | 107 | @Override 108 | public void readBytes(byte[] b, int offset, int len) throws IOException { 109 | int read = read(b, offset, len); 110 | if (read < len) { 111 | throw new EOFException(); 112 | } 113 | } 114 | 115 | @Override 116 | public void close() throws IOException { 117 | // nothing to do here 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/transport/netty/ChannelBufferStreamInputFactory.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.transport.netty; 2 | 3 | import org.elasticsearch.common.io.stream.StreamInput; 4 | import org.jboss.netty.buffer.ChannelBuffer; 5 | 6 | /** 7 | */ 8 | public class ChannelBufferStreamInputFactory { 9 | 10 | public static StreamInput create(ChannelBuffer buffer) { 11 | return new ChannelBufferStreamInput(buffer, buffer.readableBytes()); 12 | } 13 | 14 | public static StreamInput create(ChannelBuffer buffer, int size) { 15 | return new ChannelBufferStreamInput(buffer, size); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/transport/netty/NettyHeader.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.transport.netty; 2 | 3 | import org.elasticsearch.Version; 4 | import org.jboss.netty.buffer.ChannelBuffer; 5 | 6 | /** 7 | */ 8 | public class NettyHeader { 9 | 10 | public static final int HEADER_SIZE = 2 + 4 + 8 + 1 + 4; 11 | 12 | public static void writeHeader(ChannelBuffer buffer, long requestId, byte status, Version version) { 13 | int index = buffer.readerIndex(); 14 | buffer.setByte(index, 'E'); 15 | index += 1; 16 | buffer.setByte(index, 'S'); 17 | index += 1; 18 | // write the size, the size indicates the remaining message size, not including the size int 19 | buffer.setInt(index, buffer.readableBytes() - 6); 20 | index += 4; 21 | buffer.setLong(index, requestId); 22 | index += 8; 23 | buffer.setByte(index, status); 24 | index += 1; 25 | buffer.setInt(index, version.id); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/transport/netty/NettyTransportChannel.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.transport.netty; 2 | 3 | import org.elasticsearch.Version; 4 | import org.elasticsearch.common.compress.CompressorFactory; 5 | import org.elasticsearch.common.io.stream.HandlesStreamOutput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | import org.elasticsearch.common.io.ThrowableObjectOutputStream; 8 | import org.elasticsearch.common.lease.Releasables; 9 | import org.elasticsearch.transport.NotSerializableTransportException; 10 | import org.elasticsearch.transport.RemoteTransportException; 11 | import org.elasticsearch.transport.TransportChannel; 12 | import org.elasticsearch.transport.TransportResponse; 13 | import org.elasticsearch.transport.TransportResponseOptions; 14 | import org.elasticsearch.transport.support.TransportStatus; 15 | import org.jboss.netty.buffer.ChannelBuffer; 16 | import org.jboss.netty.channel.Channel; 17 | import org.jboss.netty.channel.ChannelFuture; 18 | import org.xbib.elasticsearch.common.bytes.ReleasableBytesReference; 19 | import org.xbib.elasticsearch.common.io.stream.BytesStreamOutput; 20 | import org.xbib.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; 21 | import org.xbib.elasticsearch.common.netty.ReleaseChannelFutureListener; 22 | 23 | import java.io.IOException; 24 | import java.io.NotSerializableException; 25 | 26 | /** 27 | * 28 | */ 29 | public class NettyTransportChannel implements TransportChannel { 30 | 31 | private final NettyTransport transport; 32 | private final Version version; 33 | private final String action; 34 | private final Channel channel; 35 | private final long requestId; 36 | 37 | public NettyTransportChannel(NettyTransport transport, String action, Channel channel, long requestId, Version version) { 38 | this.version = version; 39 | this.transport = transport; 40 | this.action = action; 41 | this.channel = channel; 42 | this.requestId = requestId; 43 | } 44 | 45 | @Override 46 | public String action() { 47 | return this.action; 48 | } 49 | 50 | @Override 51 | public void sendResponse(TransportResponse response) throws IOException { 52 | sendResponse(response, TransportResponseOptions.EMPTY); 53 | } 54 | 55 | @Override 56 | public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException { 57 | if (transport.compress) { 58 | options.withCompress(true); 59 | } 60 | byte status = 0; 61 | status = TransportStatus.setResponse(status); 62 | 63 | ReleasableBytesStreamOutput bStream = new ReleasableBytesStreamOutput(transport.bigArrays); 64 | boolean addedReleaseListener = false; 65 | try { 66 | bStream.skip(NettyHeader.HEADER_SIZE); 67 | StreamOutput stream = bStream; 68 | if (options.compress()) { 69 | status = TransportStatus.setCompress(status); 70 | stream = CompressorFactory.defaultCompressor().streamOutput(stream); 71 | } 72 | stream = new HandlesStreamOutput(stream); 73 | stream.setVersion(version); 74 | response.writeTo(stream); 75 | stream.close(); 76 | 77 | ReleasableBytesReference bytes = bStream.ourBytes(); 78 | ChannelBuffer buffer = bytes.toChannelBuffer(); 79 | NettyHeader.writeHeader(buffer, requestId, status, version); 80 | ChannelFuture future = channel.write(buffer); 81 | ReleaseChannelFutureListener listener = new ReleaseChannelFutureListener(bytes); 82 | future.addListener(listener); 83 | addedReleaseListener = true; 84 | } finally { 85 | if (!addedReleaseListener) { 86 | Releasables.close(bStream.ourBytes()); 87 | } 88 | } 89 | } 90 | 91 | @Override 92 | public void sendResponse(Throwable error) throws IOException { 93 | BytesStreamOutput stream = new BytesStreamOutput(); 94 | try { 95 | stream.skip(NettyHeader.HEADER_SIZE); 96 | RemoteTransportException tx = new RemoteTransportException(transport.nodeName(), transport.wrapAddress(channel.getLocalAddress()), action, error); 97 | ThrowableObjectOutputStream too = new ThrowableObjectOutputStream(stream); 98 | too.writeObject(tx); 99 | too.close(); 100 | } catch (NotSerializableException e) { 101 | stream.reset(); 102 | stream.skip(org.elasticsearch.transport.netty.NettyHeader.HEADER_SIZE); 103 | RemoteTransportException tx = new RemoteTransportException(transport.nodeName(), transport.wrapAddress(channel.getLocalAddress()), action, new NotSerializableTransportException(error)); 104 | ThrowableObjectOutputStream too = new ThrowableObjectOutputStream(stream); 105 | too.writeObject(tx); 106 | too.close(); 107 | } 108 | 109 | byte status = 0; 110 | status = TransportStatus.setResponse(status); 111 | status = TransportStatus.setError(status); 112 | 113 | ChannelBuffer buffer = stream.ourBytes().toChannelBuffer(); 114 | NettyHeader.writeHeader(buffer, requestId, status, version); 115 | channel.write(buffer); 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/transport/netty/SizeHeaderFrameDecoder.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.transport.netty; 2 | 3 | import org.elasticsearch.common.unit.ByteSizeValue; 4 | import org.elasticsearch.monitor.jvm.JvmInfo; 5 | import org.jboss.netty.buffer.ChannelBuffer; 6 | import org.jboss.netty.channel.Channel; 7 | import org.jboss.netty.channel.ChannelHandlerContext; 8 | import org.jboss.netty.handler.codec.frame.FrameDecoder; 9 | import org.jboss.netty.handler.codec.frame.TooLongFrameException; 10 | 11 | import java.io.StreamCorruptedException; 12 | 13 | /** 14 | */ 15 | public class SizeHeaderFrameDecoder extends FrameDecoder { 16 | 17 | private static final long NINETY_PER_HEAP_SIZE = (long) (JvmInfo.jvmInfo().mem().heapMax().bytes() * 0.9); 18 | 19 | @Override 20 | protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer) throws Exception { 21 | if (buffer.readableBytes() < 4) { 22 | return null; 23 | } 24 | 25 | int dataLen = buffer.getInt(buffer.readerIndex()); 26 | if (dataLen <= 0) { 27 | throw new StreamCorruptedException("invalid data length: " + dataLen); 28 | } 29 | // safety against too large frames being sent 30 | if (dataLen > NINETY_PER_HEAP_SIZE) { 31 | throw new TooLongFrameException( 32 | "transport content length received [" + new ByteSizeValue(dataLen) + "] exceeded [" + new ByteSizeValue(NINETY_PER_HEAP_SIZE) + "]"); 33 | } 34 | 35 | if (buffer.readableBytes() < dataLen + 4) { 36 | return null; 37 | } 38 | buffer.skipBytes(4); 39 | return buffer; 40 | } 41 | } -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/BaseInteractiveHandler.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket; 2 | 3 | import org.elasticsearch.client.Client; 4 | import org.elasticsearch.common.component.AbstractComponent; 5 | import org.elasticsearch.common.settings.Settings; 6 | 7 | /** 8 | * The BaseInteractiveHandler is the base class for interactive actions. 9 | */ 10 | public abstract class BaseInteractiveHandler extends AbstractComponent 11 | implements InteractiveHandler { 12 | 13 | protected final Client client; 14 | 15 | protected BaseInteractiveHandler(Settings settings, Client client) { 16 | super(settings); 17 | this.client = client; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/InteractiveActionModule.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket; 2 | 3 | import org.elasticsearch.common.collect.Lists; 4 | import org.elasticsearch.common.inject.AbstractModule; 5 | import org.xbib.elasticsearch.action.websocket.bulk.BulkDeleteAction; 6 | import org.xbib.elasticsearch.action.websocket.bulk.BulkFlushAction; 7 | import org.xbib.elasticsearch.action.websocket.bulk.BulkIndexAction; 8 | import org.xbib.elasticsearch.action.websocket.pubsub.ForwardAction; 9 | import org.xbib.elasticsearch.action.websocket.pubsub.PublishAction; 10 | import org.xbib.elasticsearch.action.websocket.pubsub.SubscribeAction; 11 | import org.xbib.elasticsearch.action.websocket.pubsub.UnsubscribeAction; 12 | 13 | import java.util.List; 14 | 15 | /** 16 | * The InteractiveActionModule binds all WebSocket actions. 17 | */ 18 | public class InteractiveActionModule extends AbstractModule { 19 | 20 | private List> websocketActions = Lists.newArrayList(); 21 | 22 | public InteractiveActionModule(List> websocketActions) { 23 | this.websocketActions = websocketActions; 24 | } 25 | 26 | @Override 27 | protected void configure() { 28 | for (Class websocketAction : websocketActions) { 29 | bind(websocketAction).asEagerSingleton(); 30 | } 31 | bind(PublishAction.class).asEagerSingleton(); 32 | bind(SubscribeAction.class).asEagerSingleton(); 33 | bind(UnsubscribeAction.class).asEagerSingleton(); 34 | bind(ForwardAction.class).asEagerSingleton(); 35 | bind(BulkDeleteAction.class).asEagerSingleton(); 36 | bind(BulkIndexAction.class).asEagerSingleton(); 37 | bind(BulkFlushAction.class).asEagerSingleton(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/InteractiveChannel.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket; 2 | 3 | import org.elasticsearch.common.xcontent.XContentBuilder; 4 | import org.jboss.netty.channel.Channel; 5 | 6 | import java.io.IOException; 7 | 8 | /** 9 | * InteractiveChannel writes responses to a a channel 10 | */ 11 | public interface InteractiveChannel { 12 | 13 | Channel getChannel(); 14 | 15 | void sendResponse(InteractiveResponse response) throws IOException; 16 | 17 | void sendResponse(String type, Throwable t) throws IOException; 18 | 19 | void sendResponse(String type, XContentBuilder builder) throws IOException; 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/InteractiveController.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket; 2 | 3 | import org.elasticsearch.ElasticsearchException; 4 | import org.elasticsearch.common.component.AbstractLifecycleComponent; 5 | import org.elasticsearch.common.inject.Inject; 6 | import org.elasticsearch.common.logging.ESLogger; 7 | import org.elasticsearch.common.logging.ESLoggerFactory; 8 | import org.elasticsearch.common.settings.Settings; 9 | import org.elasticsearch.common.xcontent.XContentFactory; 10 | import org.elasticsearch.common.xcontent.XContentParser; 11 | import org.jboss.netty.channel.Channel; 12 | import org.jboss.netty.channel.ChannelHandlerContext; 13 | import org.jboss.netty.handler.codec.http.websocketx.CloseWebSocketFrame; 14 | import org.jboss.netty.handler.codec.http.websocketx.PingWebSocketFrame; 15 | import org.jboss.netty.handler.codec.http.websocketx.PongWebSocketFrame; 16 | import org.jboss.netty.handler.codec.http.websocketx.TextWebSocketFrame; 17 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 18 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketServerHandshaker; 19 | import org.xbib.elasticsearch.http.netty.NettyInteractiveChannel; 20 | import org.xbib.elasticsearch.http.netty.NettyInteractiveRequest; 21 | 22 | import java.nio.charset.Charset; 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | 26 | /** 27 | * The InteractiveController controls the presence of websocket connections 28 | * and the flow of websocket frames for interactive use. 29 | */ 30 | public class InteractiveController extends AbstractLifecycleComponent { 31 | 32 | private final ESLogger logger = ESLoggerFactory.getLogger(InteractiveController.class.getSimpleName()); 33 | 34 | private final HashMap handlers = new HashMap(); 35 | 36 | @Inject 37 | public InteractiveController(Settings settings) { 38 | super(settings); 39 | } 40 | 41 | @Override 42 | protected void doStart() throws ElasticsearchException { 43 | } 44 | 45 | @Override 46 | protected void doStop() throws ElasticsearchException { 47 | } 48 | 49 | @Override 50 | protected void doClose() throws ElasticsearchException { 51 | } 52 | 53 | public void registerHandler(String type, InteractiveHandler handler) { 54 | handlers.put(type, handler); 55 | } 56 | 57 | public void presence(Presence presence, String topic, Channel channel) { 58 | if (logger.isDebugEnabled()) { 59 | logger.debug("presence: " + presence.name() 60 | + " topic =" + topic 61 | + " channel =" + channel); 62 | } 63 | } 64 | 65 | public void frame(WebSocketServerHandshaker handshaker, WebSocketFrame frame, ChannelHandlerContext context) { 66 | Channel channel = context.getChannel(); 67 | if (frame instanceof TextWebSocketFrame) { 68 | text((TextWebSocketFrame) frame, channel); 69 | } else if (handshaker != null && frame instanceof CloseWebSocketFrame) { 70 | handshaker.close(context.getChannel(), (CloseWebSocketFrame) frame); 71 | presence(Presence.DISCONNECTED, null, channel); 72 | } else if (frame instanceof PingWebSocketFrame) { 73 | channel.write(new PongWebSocketFrame(frame.getBinaryData())); 74 | } 75 | } 76 | 77 | private void text(TextWebSocketFrame frame, Channel channel) { 78 | Map map = parse(frame.getBinaryData().toString(Charset.forName("UTF-8"))); 79 | if (map == null) { 80 | error("invalid request", channel); 81 | return; 82 | } 83 | String type = (String) map.get("type"); 84 | if (type == null) { 85 | error("no type found", channel); 86 | return; 87 | } 88 | if (!handlers.containsKey(type)) { 89 | error("missing handler for type: " + type, channel); 90 | return; 91 | } 92 | Map data = (Map) map.get("data"); 93 | handlers.get(type).handleRequest(new NettyInteractiveRequest(data), 94 | new NettyInteractiveChannel(channel)); 95 | } 96 | 97 | private Map parse(String source) { 98 | Map map = null; 99 | XContentParser parser = null; 100 | try { 101 | parser = XContentFactory.xContent(source).createParser(source); 102 | map = parser.map(); 103 | } catch (Exception e) { 104 | logger.error("unable to parse: {}", source); 105 | } finally { 106 | if (parser != null) { 107 | parser.close(); 108 | } 109 | } 110 | return map; 111 | } 112 | 113 | private void error(String message, Channel channel) { 114 | String text = "{\"ok\":false,\"error\":\"" + message + "\"}"; 115 | TextWebSocketFrame frame = new TextWebSocketFrame(text); 116 | channel.write(frame); 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/InteractiveHandler.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket; 2 | 3 | /** 4 | * A request handler for interactive requests 5 | */ 6 | public interface InteractiveHandler { 7 | 8 | void handleRequest(InteractiveRequest request, InteractiveChannel channel); 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/InteractiveRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket; 2 | 3 | import org.elasticsearch.common.unit.TimeValue; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * The InteractiveRequest manages parameters in an interaction. 9 | */ 10 | public interface InteractiveRequest { 11 | 12 | Map asMap(); 13 | 14 | boolean hasParam(String key); 15 | 16 | Object param(String key); 17 | 18 | String paramAsString(String key); 19 | 20 | String paramAsString(String key, String defaultValue); 21 | 22 | long paramAsLong(String key); 23 | 24 | long paramAsLong(String key, long defaultValue); 25 | 26 | boolean paramAsBoolean(String key); 27 | 28 | boolean paramAsBoolean(String key, boolean defaultValue); 29 | 30 | TimeValue paramAsTime(String key); 31 | 32 | TimeValue paramAsTime(String key, TimeValue defaultValue); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/InteractiveResponse.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket; 2 | 3 | import org.jboss.netty.handler.codec.http.websocketx.TextWebSocketFrame; 4 | 5 | import java.io.IOException; 6 | 7 | /** 8 | * The InteractiveResponse can serve frames to websocket connections. 9 | */ 10 | public interface InteractiveResponse { 11 | 12 | String type(); 13 | 14 | TextWebSocketFrame response() throws IOException; 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/Presence.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket; 2 | 3 | /** 4 | * The presence states of a websocket client 5 | */ 6 | public enum Presence { 7 | 8 | CONNECTED, DISCONNECTED 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/client/WebSocketActionListener.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket.client; 2 | 3 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 4 | 5 | import java.io.IOException; 6 | 7 | /** 8 | * Listening to WebSocket actions. 9 | * 10 | * @param the WebSocketClient class parameter 11 | * @param the WebSocketFrame class parameter 12 | */ 13 | public interface WebSocketActionListener { 14 | 15 | /** 16 | * Called when the client is connected to the server 17 | * 18 | * @param client Current client used to connect 19 | * @throws java.io.IOException if this method fails 20 | */ 21 | void onConnect(C client) throws IOException; 22 | 23 | /** 24 | * Called when the client got disconnected from the server 25 | * 26 | * @param client Current client that was disconnected 27 | * @throws java.io.IOException if this method fails 28 | */ 29 | void onDisconnect(C client) throws IOException; 30 | 31 | /** 32 | * Called when a message arrives from the server 33 | * 34 | * @param client the connected client 35 | * @param frame the data received from server 36 | * @throws java.io.IOException if this method fails 37 | */ 38 | void onMessage(C client, F frame) throws IOException; 39 | 40 | /** 41 | * Called when an unhandled errors occurs. 42 | * 43 | * @param t The causing error 44 | * @throws java.io.IOException if this method fails 45 | */ 46 | void onError(Throwable t) throws IOException; 47 | 48 | class Adapter implements WebSocketActionListener { 49 | 50 | @Override 51 | public void onConnect(WebSocketClient client) throws IOException { 52 | } 53 | 54 | @Override 55 | public void onDisconnect(WebSocketClient client) throws IOException { 56 | } 57 | 58 | @Override 59 | public void onMessage(WebSocketClient client, WebSocketFrame frame) throws IOException { 60 | } 61 | 62 | @Override 63 | public void onError(Throwable t) throws IOException { 64 | } 65 | 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/client/WebSocketClient.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket.client; 2 | 3 | import org.jboss.netty.channel.Channel; 4 | import org.jboss.netty.channel.ChannelFuture; 5 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 6 | 7 | /** 8 | * A WebSocket client. 9 | */ 10 | public interface WebSocketClient { 11 | 12 | /** 13 | * The channel this client has opened. 14 | * 15 | * @return the channel 16 | */ 17 | Channel channel(); 18 | 19 | /** 20 | * Connect to host and port. 21 | * 22 | * @return Connect future. Fires when connected. 23 | */ 24 | F connect(); 25 | 26 | /** 27 | * Disconnect from the server 28 | * 29 | * @return Disconnect future. Fires when disconnected. 30 | */ 31 | F disconnect(); 32 | 33 | /** 34 | * Send data to server 35 | * 36 | * @param frame Data for sending 37 | * @return Write future. Will fire when the data is sent. 38 | */ 39 | F send(Frame frame); 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/client/WebSocketClientBulkRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket.client; 2 | 3 | import org.elasticsearch.common.xcontent.XContentBuilder; 4 | 5 | import java.io.IOException; 6 | 7 | public interface WebSocketClientBulkRequest extends WebSocketClientRequest { 8 | 9 | @Override 10 | WebSocketClientBulkRequest data(XContentBuilder builder); 11 | 12 | @Override 13 | void send(WebSocketClient client) throws IOException; 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/client/WebSocketClientFactory.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket.client; 2 | 3 | import java.net.URI; 4 | 5 | /** 6 | * A WebSocketClientFactory has methods for creating WebSocket clients 7 | * and for creating WebSocket requests. 8 | */ 9 | public interface WebSocketClientFactory { 10 | 11 | WebSocketClient newClient(URI resourceIdentifier, WebSocketActionListener listener); 12 | 13 | WebSocketClientRequest newRequest(); 14 | 15 | WebSocketClientBulkRequest indexRequest(); 16 | 17 | WebSocketClientBulkRequest deleteRequest(); 18 | 19 | WebSocketClientRequest flushRequest(); 20 | 21 | void shutdown(); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/xbib/elasticsearch/websocket/client/WebSocketClientRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket.client; 2 | 3 | import org.elasticsearch.common.xcontent.XContentBuilder; 4 | 5 | import java.io.IOException; 6 | 7 | public interface WebSocketClientRequest { 8 | 9 | WebSocketClientRequest type(String type); 10 | 11 | WebSocketClientRequest data(XContentBuilder builder); 12 | 13 | void send(WebSocketClient client) throws IOException; 14 | } 15 | -------------------------------------------------------------------------------- /src/main/resources/es-plugin.properties: -------------------------------------------------------------------------------- 1 | plugin=org.xbib.elasticsearch.plugin.websocket.WebSocketPlugin 2 | version=${project.version} 3 | hash=${buildNumber} 4 | timestamp=${timestamp} 5 | date=${tstamp} 6 | -------------------------------------------------------------------------------- /src/site/resources/elasticsearch-websocket.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jprante/elasticsearch-transport-websocket/b6c75660487ffd17db6d2c861448611741db2d9a/src/site/resources/elasticsearch-websocket.png -------------------------------------------------------------------------------- /src/site/resources/publish-subscribe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jprante/elasticsearch-transport-websocket/b6c75660487ffd17db6d2c861448611741db2d9a/src/site/resources/publish-subscribe.png -------------------------------------------------------------------------------- /src/site/resources/transport-modules.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jprante/elasticsearch-transport-websocket/b6c75660487ffd17db6d2c861448611741db2d9a/src/site/resources/transport-modules.png -------------------------------------------------------------------------------- /src/site/site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | org.apache.maven.skins 5 | maven-fluido-skin 6 | 1.2.1 7 | 8 | 9 | 10 | true 11 | true 12 | 13 | jprante/elasticsearch-transport-websocket 14 | right 15 | black 16 | 17 | 18 | xbib 19 | true 20 | true 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/test/java/org/xbib/elasticsearch/websocket/BulkTest.java: -------------------------------------------------------------------------------- 1 | 2 | package org.xbib.elasticsearch.websocket; 3 | 4 | import java.util.concurrent.TimeUnit; 5 | 6 | import org.jboss.netty.handler.codec.http.websocketx.CloseWebSocketFrame; 7 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 8 | 9 | import org.elasticsearch.common.logging.ESLogger; 10 | import org.elasticsearch.common.logging.ESLoggerFactory; 11 | 12 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 13 | 14 | import org.junit.Test; 15 | import org.xbib.elasticsearch.websocket.client.WebSocketActionListener; 16 | import org.xbib.elasticsearch.websocket.client.WebSocketClient; 17 | import org.xbib.elasticsearch.websocket.client.WebSocketClientFactory; 18 | import org.xbib.elasticsearch.websocket.helper.AbstractNodeTestHelper; 19 | import org.xbib.elasticsearch.http.netty.client.NettyWebSocketClientFactory; 20 | 21 | public class BulkTest extends AbstractNodeTestHelper { 22 | 23 | private final static ESLogger logger = ESLoggerFactory.getLogger("test"); 24 | 25 | @Test 26 | public void testBulk() { 27 | try { 28 | final WebSocketClientFactory clientFactory = new NettyWebSocketClientFactory(); 29 | WebSocketClient client = clientFactory.newClient( 30 | getAddressOfNode("1"), 31 | new WebSocketActionListener.Adapter() { 32 | @Override 33 | public void onConnect(WebSocketClient client) { 34 | try { 35 | logger.info("sending some index requests (longer than a single bulk size)"); 36 | for (int i = 0; i < 250; i++) { 37 | clientFactory.indexRequest() 38 | .data(jsonBuilder() 39 | .startObject() 40 | .field("index", "test") 41 | .field("type", "test") 42 | .field("id", Integer.toString(i)) 43 | .startObject("data") 44 | .field("field1", "value" + i) 45 | .field("field2", "value" + i) 46 | .endObject() 47 | .endObject()) 48 | .send(client); 49 | } 50 | // more bulks could be added here ... 51 | logger.info("at the end, let us flush the bulk"); 52 | clientFactory.flushRequest().send(client); 53 | } catch (Exception e) { 54 | onError(e); 55 | } 56 | } 57 | 58 | @Override 59 | public void onDisconnect(WebSocketClient client) { 60 | logger.info("disconnected"); 61 | } 62 | 63 | @Override 64 | public void onMessage(WebSocketClient client, WebSocketFrame frame) { 65 | logger.info("frame received: {}", frame); 66 | } 67 | 68 | @Override 69 | public void onError(Throwable t) { 70 | logger.error(t.getMessage(), t); 71 | } 72 | }); 73 | client.connect().await(1000, TimeUnit.MILLISECONDS); 74 | Thread.sleep(1000); 75 | logger.info("closing bulk client"); 76 | client.send(new CloseWebSocketFrame()); 77 | Thread.sleep(1000); 78 | client.disconnect(); 79 | clientFactory.shutdown(); 80 | } catch (Exception e) { 81 | logger.error(e.getMessage(), e); 82 | } 83 | 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/test/java/org/xbib/elasticsearch/websocket/HelloWorldWebSocketTest.java: -------------------------------------------------------------------------------- 1 | 2 | package org.xbib.elasticsearch.websocket; 3 | 4 | import java.util.concurrent.TimeUnit; 5 | 6 | import org.jboss.netty.handler.codec.http.websocketx.CloseWebSocketFrame; 7 | import org.jboss.netty.handler.codec.http.websocketx.TextWebSocketFrame; 8 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 9 | 10 | import org.elasticsearch.common.logging.ESLogger; 11 | import org.elasticsearch.common.logging.ESLoggerFactory; 12 | import org.junit.Test; 13 | import org.xbib.elasticsearch.websocket.client.WebSocketActionListener; 14 | import org.xbib.elasticsearch.websocket.client.WebSocketClient; 15 | import org.xbib.elasticsearch.websocket.client.WebSocketClientFactory; 16 | import org.xbib.elasticsearch.websocket.helper.AbstractNodeTestHelper; 17 | import org.xbib.elasticsearch.http.netty.client.NettyWebSocketClientFactory; 18 | 19 | 20 | public class HelloWorldWebSocketTest extends AbstractNodeTestHelper { 21 | 22 | private final static ESLogger logger = ESLoggerFactory.getLogger("test"); 23 | 24 | /** 25 | * Primitive websocket client, just send "Hello World" 26 | */ 27 | @Test 28 | public void helloWorld() { 29 | try { 30 | WebSocketClientFactory clientFactory = new NettyWebSocketClientFactory(); 31 | WebSocketClient client = clientFactory.newClient(getAddressOfNode("1"), 32 | new WebSocketActionListener() { 33 | @Override 34 | public void onConnect(WebSocketClient client) { 35 | logger.info("web socket connected"); 36 | String s = "{\"Hello\":\"World\"}"; 37 | client.send(new TextWebSocketFrame(s)); 38 | logger.info("sent " + s); 39 | } 40 | 41 | @Override 42 | public void onDisconnect(WebSocketClient client) { 43 | logger.info("web socket disconnected"); 44 | } 45 | 46 | @Override 47 | public void onMessage(WebSocketClient client, WebSocketFrame frame) { 48 | logger.info("frame received: " + frame); 49 | } 50 | 51 | @Override 52 | public void onError(Throwable t) { 53 | logger.error(t.getMessage(), t); 54 | } 55 | }); 56 | client.connect().await(1000, TimeUnit.MILLISECONDS); 57 | Thread.sleep(1000); 58 | client.send(new CloseWebSocketFrame()); 59 | Thread.sleep(1000); 60 | client.disconnect(); 61 | } catch (Exception e) { 62 | logger.error(e.getMessage(), e); 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/org/xbib/elasticsearch/websocket/PublishSubscribeRequestTest.java: -------------------------------------------------------------------------------- 1 | 2 | package org.xbib.elasticsearch.websocket; 3 | 4 | import java.io.IOException; 5 | import java.util.concurrent.TimeUnit; 6 | 7 | import org.jboss.netty.handler.codec.http.websocketx.CloseWebSocketFrame; 8 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 9 | 10 | import org.elasticsearch.common.logging.ESLogger; 11 | import org.elasticsearch.common.logging.ESLoggerFactory; 12 | 13 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 14 | 15 | import org.junit.Test; 16 | import org.xbib.elasticsearch.websocket.client.WebSocketActionListener; 17 | import org.xbib.elasticsearch.websocket.client.WebSocketClient; 18 | import org.xbib.elasticsearch.websocket.client.WebSocketClientFactory; 19 | import org.xbib.elasticsearch.websocket.client.WebSocketClientRequest; 20 | import org.xbib.elasticsearch.websocket.helper.AbstractNodeTestHelper; 21 | import org.xbib.elasticsearch.http.netty.client.NettyWebSocketClientFactory; 22 | 23 | public class PublishSubscribeRequestTest extends AbstractNodeTestHelper { 24 | 25 | private final static ESLogger logger = ESLoggerFactory.getLogger("test"); 26 | 27 | public void testOneClient() { 28 | try { 29 | final String subscriberId = "oneclient"; 30 | final String topic = "oneclienttest"; 31 | final WebSocketClientFactory clientFactory = new NettyWebSocketClientFactory(); 32 | 33 | final WebSocketClientRequest subscribe = clientFactory.newRequest() 34 | .type("subscribe").data(jsonBuilder().startObject() 35 | .field("subscriber", "singleclient") 36 | .field("topic", topic).endObject()); 37 | 38 | final WebSocketClientRequest publish = clientFactory.newRequest() 39 | .type("publish").data(jsonBuilder().startObject() 40 | .field("message", "Hello World") 41 | .field("topic", topic).endObject()); 42 | 43 | WebSocketClient client = clientFactory.newClient(getAddressOfNode("1"), 44 | new WebSocketActionListener.Adapter() { 45 | @Override 46 | public void onConnect(WebSocketClient client) throws IOException { 47 | logger.info("sending subscribe command, channel = {}", client.channel()); 48 | subscribe.send(client); 49 | logger.info("sending publish command (to ourselves), channel = {}", client.channel()); 50 | publish.send(client); 51 | } 52 | 53 | @Override 54 | public void onMessage(WebSocketClient client, WebSocketFrame frame) { 55 | logger.info("frame received: " + frame); 56 | } 57 | }); 58 | client.connect().await(1000, TimeUnit.MILLISECONDS); 59 | Thread.sleep(1000); 60 | client.send(new CloseWebSocketFrame()); 61 | Thread.sleep(1000); 62 | client.disconnect(); 63 | 64 | clientFactory.shutdown(); 65 | 66 | } catch (Exception e) { 67 | logger.error(e.getMessage(), e); 68 | } 69 | } 70 | 71 | @Test 72 | public void testTwoClients() { 73 | try { 74 | final String subscriberId = "twoclients"; 75 | final String topic = "twoclienttest"; 76 | final WebSocketClientFactory clientFactory = new NettyWebSocketClientFactory(); 77 | 78 | final WebSocketClientRequest subscribe = clientFactory.newRequest() 79 | .type("subscribe").data(jsonBuilder().startObject() 80 | .field("subscriber", "doubleclient") 81 | .field("topic", topic).endObject()); 82 | 83 | final WebSocketClientRequest publish = clientFactory.newRequest() 84 | .type("publish").data(jsonBuilder().startObject() 85 | .field("message", "Hi there, I'm another client") 86 | .field("topic", topic).endObject()); 87 | 88 | // open first client 89 | WebSocketClient subscribingClient = clientFactory.newClient(getAddressOfNode("1"), 90 | new WebSocketActionListener.Adapter() { 91 | @Override 92 | public void onConnect(WebSocketClient client) { 93 | try { 94 | logger.info("sending subscribe command, channel {}", client.channel()); 95 | subscribe.send(client); 96 | } catch (Exception e) { 97 | } 98 | } 99 | 100 | @Override 101 | public void onMessage(WebSocketClient client, WebSocketFrame frame) { 102 | logger.info("frame {} received for subscribing client {}", frame, client.channel()); 103 | } 104 | 105 | }); 106 | 107 | // open two client 108 | WebSocketClient publishingClient = clientFactory.newClient(getAddressOfNode("1"), 109 | new WebSocketActionListener.Adapter() { 110 | @Override 111 | public void onConnect(WebSocketClient client) { 112 | try { 113 | logger.info("sending publish command, channel = {}", client.channel()); 114 | publish.send(client); 115 | } catch (Exception e) { 116 | } 117 | } 118 | 119 | @Override 120 | public void onMessage(WebSocketClient client, WebSocketFrame frame) { 121 | logger.info("frame {} received for publishing client {}", frame, client.channel()); 122 | } 123 | 124 | }); 125 | 126 | // connect both clients to node 127 | subscribingClient.connect().await(1000, TimeUnit.MILLISECONDS); 128 | publishingClient.connect().await(1000, TimeUnit.MILLISECONDS); 129 | 130 | // wait for publish/subscribe 131 | Thread.sleep(1000); 132 | 133 | // close first client 134 | publishingClient.send(new CloseWebSocketFrame()); 135 | publishingClient.disconnect(); 136 | 137 | // close second client 138 | subscribingClient.send(new CloseWebSocketFrame()); 139 | subscribingClient.disconnect(); 140 | 141 | Thread.sleep(1000); 142 | 143 | clientFactory.shutdown(); 144 | 145 | } catch (Exception e) { 146 | logger.error(e.getMessage(), e); 147 | } 148 | } 149 | 150 | } 151 | -------------------------------------------------------------------------------- /src/test/java/org/xbib/elasticsearch/websocket/SimplePublishSubscribeTest.java: -------------------------------------------------------------------------------- 1 | 2 | package org.xbib.elasticsearch.websocket; 3 | 4 | import java.util.concurrent.TimeUnit; 5 | 6 | import org.jboss.netty.handler.codec.http.websocketx.CloseWebSocketFrame; 7 | import org.jboss.netty.handler.codec.http.websocketx.TextWebSocketFrame; 8 | import org.jboss.netty.handler.codec.http.websocketx.WebSocketFrame; 9 | 10 | import org.elasticsearch.common.logging.ESLogger; 11 | import org.elasticsearch.common.logging.ESLoggerFactory; 12 | 13 | import org.junit.Test; 14 | import org.xbib.elasticsearch.websocket.client.WebSocketActionListener; 15 | import org.xbib.elasticsearch.websocket.client.WebSocketClient; 16 | import org.xbib.elasticsearch.websocket.client.WebSocketClientFactory; 17 | import org.xbib.elasticsearch.websocket.helper.AbstractNodeTestHelper; 18 | import org.xbib.elasticsearch.http.netty.client.NettyWebSocketClientFactory; 19 | 20 | public class SimplePublishSubscribeTest extends AbstractNodeTestHelper { 21 | 22 | private final static ESLogger logger = ESLoggerFactory.getLogger("test"); 23 | 24 | /** 25 | * In this test, we just use simple TextWebSocketFrame to send publish/subscribe requests. 26 | */ 27 | @Test 28 | public void subscribeToOurselves() { 29 | try { 30 | WebSocketClientFactory clientFactory = new NettyWebSocketClientFactory(); 31 | WebSocketClient client = clientFactory.newClient(getAddressOfNode("1"), 32 | new WebSocketActionListener() { 33 | @Override 34 | public void onConnect(WebSocketClient client) { 35 | try { 36 | logger.info("sending subscribe command"); 37 | client.send(new TextWebSocketFrame("{\"type\":\"subscribe\",\"data\":{\"subscriber\":\"mypubsubdemo\",\"topic\":\"demo\"}}")); 38 | Thread.sleep(500); 39 | logger.info("sending publish command (to ourselves)"); 40 | client.send(new TextWebSocketFrame("{\"type\":\"publish\",\"data\":{\"message\":\"Hello World\",\"topic\":\"demo\"}}")); 41 | } catch (Exception e) { 42 | logger.error(e.getMessage(), e); 43 | } 44 | } 45 | 46 | @Override 47 | public void onDisconnect(WebSocketClient client) { 48 | logger.info("web socket disconnected"); 49 | } 50 | 51 | @Override 52 | public void onMessage(WebSocketClient client, WebSocketFrame frame) { 53 | logger.info("frame received: {}", frame); 54 | } 55 | 56 | @Override 57 | public void onError(Throwable t) { 58 | logger.error(t.getMessage(), t); 59 | } 60 | }); 61 | client.connect().await(1000, TimeUnit.MILLISECONDS); 62 | Thread.sleep(1000); 63 | client.send(new CloseWebSocketFrame()); 64 | Thread.sleep(1000); 65 | client.disconnect(); 66 | } catch (Exception e) { 67 | logger.error(e.getMessage(), e); 68 | } 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/test/java/org/xbib/elasticsearch/websocket/helper/AbstractNodeRandomTestHelper.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.websocket.helper; 2 | 3 | import java.util.Random; 4 | 5 | public class AbstractNodeRandomTestHelper extends AbstractNodeTestHelper { 6 | 7 | private static Random random = new Random(); 8 | 9 | private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); 10 | 11 | protected String randomString(int len) { 12 | final char[] buf = new char[len]; 13 | final int n = numbersAndLetters.length - 1; 14 | for (int i = 0; i < buf.length; i++) { 15 | buf[i] = numbersAndLetters[random.nextInt(n)]; 16 | } 17 | return new String(buf); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/test/java/org/xbib/elasticsearch/websocket/helper/AbstractNodeTestHelper.java: -------------------------------------------------------------------------------- 1 | 2 | package org.xbib.elasticsearch.websocket.helper; 3 | 4 | import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; 5 | import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; 6 | import org.elasticsearch.client.Client; 7 | import org.elasticsearch.common.logging.ESLogger; 8 | import org.elasticsearch.common.logging.ESLoggerFactory; 9 | import org.elasticsearch.common.network.NetworkUtils; 10 | import org.elasticsearch.common.settings.ImmutableSettings; 11 | import org.elasticsearch.common.settings.Settings; 12 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 13 | import org.elasticsearch.indices.IndexMissingException; 14 | import org.elasticsearch.node.Node; 15 | import org.junit.After; 16 | import org.junit.Before; 17 | import org.xbib.elasticsearch.action.cluster.admin.websocket.WebsocketInfoAction; 18 | import org.xbib.elasticsearch.action.cluster.admin.websocket.WebsocketInfoRequest; 19 | import org.xbib.elasticsearch.action.cluster.admin.websocket.WebsocketInfoResponse; 20 | 21 | import java.net.URI; 22 | import java.util.Map; 23 | 24 | import static org.elasticsearch.common.collect.Maps.newHashMap; 25 | import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS; 26 | import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; 27 | import static org.elasticsearch.node.NodeBuilder.nodeBuilder; 28 | 29 | public abstract class AbstractNodeTestHelper { 30 | 31 | private final static ESLogger logger = ESLoggerFactory.getLogger("test"); 32 | 33 | protected final String CLUSTER = "test-cluster-" + NetworkUtils.getLocalAddress().getHostName(); 34 | 35 | protected final String INDEX = "test-" + NetworkUtils.getLocalAddress().getHostName().toLowerCase(); 36 | 37 | protected Settings defaultSettings = ImmutableSettings 38 | .settingsBuilder() 39 | .put("cluster.name", CLUSTER) 40 | .put("threadpool.bulk.queue_size", 200) 41 | .build(); 42 | 43 | private Map nodes = newHashMap(); 44 | 45 | private Map clients = newHashMap(); 46 | 47 | private Map addresses = newHashMap(); 48 | 49 | protected URI getAddressOfNode(String n) { 50 | InetSocketTransportAddress address = addresses.get(n); 51 | return URI.create("ws://" + address.address().getHostName() + ":" + address.address().getPort() + "/websocket"); 52 | } 53 | 54 | @Before 55 | public void setUp() throws Exception { 56 | startNode("1"); 57 | WebsocketInfoRequest request = new WebsocketInfoRequest("1"); 58 | WebsocketInfoResponse response = client("1").admin().cluster().execute(WebsocketInfoAction.INSTANCE, request).actionGet(); 59 | InetSocketTransportAddress address = response.getAt(0).getAddress(); 60 | addresses.put("1", address); 61 | 62 | logger.info("websocket address = {}", address); 63 | 64 | logger.info("creating index {}", INDEX); 65 | client("1").admin().indices().create(new CreateIndexRequest(INDEX)).actionGet(); 66 | logger.info("index {} created", INDEX); 67 | } 68 | 69 | @After 70 | public void deleteIndices() { 71 | logger.info("deleting index {}", INDEX); 72 | try { 73 | client("1").admin().indices().delete(new DeleteIndexRequest().indices(INDEX)).actionGet(); 74 | } catch (IndexMissingException e) { 75 | // ignore 76 | } 77 | logger.info("index {} deleted", INDEX); 78 | closeAllNodes(); 79 | } 80 | 81 | public Node startNode(String id) { 82 | return buildNode(id).start(); 83 | } 84 | 85 | public Node buildNode(String id) { 86 | return buildNode(id, EMPTY_SETTINGS); 87 | } 88 | 89 | public Node buildNode(String id, Settings settings) { 90 | String settingsSource = getClass().getName().replace('.', '/') + ".yml"; 91 | Settings finalSettings = settingsBuilder() 92 | .loadFromClasspath(settingsSource) 93 | .put(defaultSettings) 94 | .put(settings) 95 | .put("name", id) 96 | .put("gateway.type", "none") 97 | .put("cluster.routing.schedule", "50ms") 98 | .build(); 99 | Node node = nodeBuilder().settings(finalSettings).build(); 100 | Client client = node.client(); 101 | nodes.put(id, node); 102 | clients.put(id, client); 103 | return node; 104 | } 105 | 106 | public Client client(String id) { 107 | return clients.get(id); 108 | } 109 | 110 | public void closeAllNodes() { 111 | for (Client client : clients.values()) { 112 | client.close(); 113 | } 114 | clients.clear(); 115 | for (Node node : nodes.values()) { 116 | if (node != null) { 117 | node.close(); 118 | } 119 | } 120 | nodes.clear(); 121 | } 122 | 123 | } 124 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, out 2 | 3 | log4j.logger.test=TRACE 4 | 5 | #log4j.logger.transport=TRACE 6 | #log4j.logger.discovery=TRACE 7 | #log4j.logger.cluster.service=TRACE 8 | #log4j.logger.cluster.action.shard=DEBUG 9 | #log4j.logger.indices.cluster=DEBUG 10 | #log4j.logger.index=TRACE 11 | #log4j.logger.index.engine=DEBUG 12 | #log4j.logger.index.shard.service=DEBUG 13 | #log4j.logger.index.shard.recovery=DEBUG 14 | #log4j.logger.index.cache=DEBUG 15 | #log4j.logger.http=TRACE 16 | #log4j.logger.monitor.memory=TRACE 17 | #log4j.logger.monitor.memory=TRACE 18 | #log4j.logger.cluster.action.shard=TRACE 19 | #log4j.logger.index.gateway=TRACE 20 | 21 | log4j.appender.out=org.apache.log4j.ConsoleAppender 22 | log4j.appender.out.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.out.layout.ConversionPattern=[%d{ABSOLUTE}][%-5p][%-25c] %m%n 24 | --------------------------------------------------------------------------------