├── .gitignore ├── LICENSE ├── README.md ├── build.gradle ├── cluster ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── linkedin │ │ │ └── norbert │ │ │ └── protos │ │ │ ├── NorbertExampleProtos.java │ │ │ └── NorbertProtos.java │ ├── protobuf │ │ ├── norbert.proto │ │ └── norbert_example.proto │ └── scala │ │ └── com │ │ └── linkedin │ │ └── norbert │ │ ├── NorbertException.scala │ │ ├── cluster │ │ ├── ClusterClient.scala │ │ ├── ClusterClientComponent.scala │ │ ├── ClusterDefaults.scala │ │ ├── ClusterEvent.scala │ │ ├── ClusterException.scala │ │ ├── Node.scala │ │ ├── common │ │ │ ├── ClusterManagerComponent.scala │ │ │ ├── ClusterManagerHelper.scala │ │ │ └── ClusterNotificationManagerComponent.scala │ │ ├── memory │ │ │ ├── InMemoryClusterClient.scala │ │ │ └── InMemoryClusterManagerComponent.scala │ │ └── zookeeper │ │ │ ├── ZooKeeperClusterClient.scala │ │ │ └── ZooKeeperClusterManagerComponent.scala │ │ ├── jmx │ │ ├── AverageTimeTracker.scala │ │ └── JMX.scala │ │ ├── logging │ │ ├── Logger.scala │ │ └── Logging.scala │ │ └── norbertutils │ │ ├── Clock.scala │ │ ├── KeepAliveActor.scala │ │ ├── NamedPoolThreadFactory.scala │ │ └── package.scala │ └── test │ ├── resources │ └── log4j.properties │ └── scala │ └── com │ └── linkedin │ └── norbert │ ├── cluster │ ├── ClusterClientSpec.scala │ ├── NodeSpec.scala │ ├── common │ │ └── ClusterNotificationManagerComponentSpec.scala │ ├── memory │ │ └── InMemoryClusterClientSpec.scala │ └── zookeeper │ │ └── ZooKeeperClusterManagerComponentSpec.scala │ ├── jmx │ └── AverageTimeTrackerSpec.scala │ ├── logging │ └── LoggerSpec.scala │ └── norbertutils │ └── KeepAliveActorSpec.scala ├── defaultEnvironment.gradle ├── examples └── src │ └── main │ ├── java │ └── com │ │ └── linkedin │ │ └── norbert │ │ └── javacompat │ │ └── network │ │ ├── NorbertJavaNetworkClientMain.java │ │ ├── NorbertJavaNetworkServerMain.java │ │ ├── Ping.java │ │ └── RunNorbertSetup.java │ ├── resources │ └── log4j.properties │ └── scala │ └── com │ └── linkedin │ └── norbert │ ├── cluster │ └── NorbertClusterClientMain.scala │ └── network │ ├── NorbertNetworkClientMain.scala │ ├── NorbertNetworkServerMain.scala │ └── PingRequest.scala ├── gradle.properties ├── java-cluster ├── build.gradle └── src │ └── main │ ├── java │ └── com │ │ └── linkedin │ │ └── norbert │ │ └── javacompat │ │ └── cluster │ │ ├── ClusterClient.java │ │ ├── ClusterListener.java │ │ ├── ClusterListenerAdapter.java │ │ └── Node.java │ └── scala │ └── com │ └── linkedin │ └── norbert │ └── javacompat │ ├── cluster │ ├── BaseClusterClient.scala │ ├── InMemoryClusterClient.scala │ ├── JavaNode.scala │ └── ZooKeeperClusterClient.scala │ └── package.scala ├── java-network ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── linkedin │ │ │ └── norbert │ │ │ └── javacompat │ │ │ └── network │ │ │ ├── BaseNetworkClient.java │ │ │ ├── ConsistentHashPartitionedLoadBalancer.java │ │ │ ├── ConsistentHashPartitionedLoadBalancerFactory.java │ │ │ ├── Endpoint.java │ │ │ ├── HashFunction.java │ │ │ ├── LoadBalancer.java │ │ │ ├── LoadBalancerFactory.java │ │ │ ├── NetworkClient.java │ │ │ ├── NetworkClientConfig.java │ │ │ ├── NetworkServer.java │ │ │ ├── NetworkServerConfig.java │ │ │ ├── PartitionedLoadBalancer.java │ │ │ ├── PartitionedLoadBalancerFactory.java │ │ │ ├── PartitionedNetworkClient.java │ │ │ ├── RequestBuilder.java │ │ │ ├── RequestHandler.java │ │ │ ├── RingHashPartitionedLoadBalancer.java │ │ │ ├── RingHashPartitionedLoadBalancerFactory.java │ │ │ └── ScatterGatherHandler.java │ └── scala │ │ └── com │ │ └── linkedin │ │ └── norbert │ │ ├── javacompat │ │ └── network │ │ │ ├── BaseNettyNetworkClient.scala │ │ │ ├── DefaultPartitionedLoadBalancerFactory.scala │ │ │ ├── IntegerConsistentHashPartitionedLoadBalancerFactory.scala │ │ │ ├── JavaEndpoint.scala │ │ │ ├── JavaLbfToScalaLbf.scala │ │ │ ├── MultiRingConsistentHashPartitionedLoadBalancerFactory.scala │ │ │ ├── NettyNetworkServer.scala │ │ │ ├── PartitionedNetworkClientFactory.scala │ │ │ ├── RoundRobinLoadBalancerFactory.scala │ │ │ └── ScalaLbfToJavaLbf.scala │ │ └── package.scala │ └── test │ └── resources │ └── log4j.properties ├── lib └── sbt-launch.jar ├── network ├── build.gradle └── src │ ├── main │ └── scala │ │ └── com │ │ └── linkedin │ │ └── norbert │ │ └── network │ │ ├── JavaSerializer.scala │ │ ├── NetworkDefaults.scala │ │ ├── NetworkServerComponent.scala │ │ ├── NetworkingException.scala │ │ ├── Request.scala │ │ ├── ResponseIterator.scala │ │ ├── Serializer.scala │ │ ├── client │ │ ├── Filter.scala │ │ ├── NetworkClient.scala │ │ ├── NetworkClientComponent.scala │ │ ├── ResponseHandlerComponent.scala │ │ └── loadbalancer │ │ │ ├── LoadBalancerFactory.scala │ │ │ └── RoundRobinLoadBalancerFactory.scala │ │ ├── common │ │ ├── BaseNetworkClient.scala │ │ ├── CanServeRequestStrategy.scala │ │ ├── ClusterIoClientComponent.scala │ │ ├── Endpoint.scala │ │ ├── LocalMessageExecution.scala │ │ ├── MessageRegistryComponent.scala │ │ ├── NetworkStatisticsActor.scala │ │ └── NorbertFuture.scala │ │ ├── netty │ │ ├── ChannelPool.scala │ │ ├── ClientChannelHandler.scala │ │ ├── NettyClusterIoClientComponent.scala │ │ ├── NettyClusterIoServerComponent.scala │ │ ├── NettyNetworkClient.scala │ │ ├── NettyNetworkServer.scala │ │ ├── NettyServerFilter.scala │ │ ├── ServerChannelHandler.scala │ │ └── UrlParser.scala │ │ ├── partitioned │ │ ├── PartitionedNetworkClient.scala │ │ ├── PartitionedNetworkClientFactory.scala │ │ └── loadbalancer │ │ │ ├── DefaultLoadBalancerHelper.scala │ │ │ ├── DefaultPartitionedLoadBalancerFactory.scala │ │ │ ├── HashFunctions.scala │ │ │ ├── PartitionUtil.scala │ │ │ ├── PartitionedConsistentHashedLoadBalancerFactory.scala │ │ │ ├── PartitionedLoadBalancerFactory.scala │ │ │ └── SimpleConsistentHashedLoadBalancerFactory.scala │ │ ├── server │ │ ├── ClusterIoServerComponent.scala │ │ ├── Filter.scala │ │ ├── MessageExecutorComponent.scala │ │ ├── MessageHandlerRegistryComponent.scala │ │ ├── NetworkServer.scala │ │ └── RequestContext.scala │ │ └── util │ │ └── ProtoUtils.scala │ └── test │ ├── resources │ └── log4j.properties │ └── scala │ └── com │ └── linkedin │ └── norbert │ └── network │ ├── client │ ├── NetworkClientSpec.scala │ └── loadbalancer │ │ └── RoundRobinLoadBalancerFactorySpec.scala │ ├── common │ ├── BaseNetworkClientSpecification.scala │ ├── LocalMessageExecutionSpec.scala │ ├── MessageRegistrySpec.scala │ ├── NorbertFutureSpec.scala │ ├── NorbertResponseIteratorSpec.scala │ └── SampleMessage.scala │ ├── netty │ ├── ChannelPoolSpec.scala │ ├── ClientChannelHandlerSpec.scala │ ├── ClientStatisticsRequestStrategySpec.scala │ ├── NettyClusterIoClientComponentSpec.scala │ └── NettyClusterIoServerComponentSpec.scala │ ├── partitioned │ ├── PartitionedNetworkClientSpec.scala │ └── loadbalancer │ │ ├── ConsistentHashPartitionedLoadBalancerFactorySpec.scala │ │ ├── DefaultPartitionedLoadBalancerSpec.scala │ │ └── PartitionedConsistentHashedLoadBalancerSpec.scala │ └── server │ ├── MessageExecutorSpec.scala │ ├── MessageHandlerRegistrySpec.scala │ └── NetworkServerSpec.scala ├── norbert └── build.gradle ├── project ├── Build.scala └── build.properties ├── sbt └── settings.gradle /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | lib_managed/ 3 | src_managed/ 4 | project/boot/ 5 | project/plugins/project 6 | out/ 7 | build/ 8 | .gradle 9 | *.iml 10 | *.ipr 11 | *.iws 12 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | project.ext.isDefaultEnvironment = !project.hasProperty('overrideBuildEnvironment') 2 | 3 | File getEnvironmentScript() 4 | { 5 | final File env = file(isDefaultEnvironment ? 'defaultEnvironment.gradle' : project.overrideBuildEnvironment) 6 | assert env.isFile() : "The environment script [$env] does not exists or is not a file." 7 | return env 8 | } 9 | 10 | apply from: environmentScript 11 | 12 | project.ext.externalDependency = [ 13 | 'zookeeper':'org.apache.zookeeper:zookeeper:3.3.0', 14 | 'protobuf':'com.google.protobuf:protobuf-java:2.4.0a', 15 | 'log4j':'log4j:log4j:1.2.16', 16 | 'netty':'org.jboss.netty:netty:3.2.3.Final', 17 | 'slf4jApi':'org.slf4j:slf4j-api:1.5.6', 18 | 'slf4jLog4j':'org.slf4j:slf4j-log4j12:1.5.6', 19 | 'specs':'org.scala-tools.testing:specs_2.8.1:1.6.8', 20 | 'mockitoAll':'org.mockito:mockito-all:1.8.4', 21 | 'cglib':'cglib:cglib:2.1_3', 22 | 'objenesis':'org.objenesis:objenesis:1.2', 23 | 'scalaCompiler': 'org.scala-lang:scala-compiler:2.8.1', 24 | 'scalaLibrary': 'org.scala-lang:scala-library:2.8.1', 25 | 'scalatest': 'org.scalatest:scalatest:1.2' 26 | ]; 27 | 28 | -------------------------------------------------------------------------------- /cluster/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'scala' 3 | 4 | dependencies { 5 | compile externalDependency.scalaLibrary 6 | compile externalDependency.zookeeper 7 | compile externalDependency.protobuf 8 | compile externalDependency.log4j 9 | 10 | testCompile externalDependency.specs 11 | testCompile externalDependency.mockitoAll 12 | testCompile externalDependency.cglib 13 | testCompile externalDependency.objenesis 14 | 15 | scalaTools externalDependency.scalaCompiler 16 | scalaTools externalDependency.scalaLibrary 17 | } 18 | 19 | -------------------------------------------------------------------------------- /cluster/src/main/protobuf/norbert.proto: -------------------------------------------------------------------------------- 1 | package norbert; 2 | 3 | option optimize_for = SPEED; 4 | option java_package = "com.linkedin.norbert.protos"; 5 | option java_outer_classname = "NorbertProtos"; 6 | 7 | message NorbertMessage { 8 | required sfixed64 request_id_msb = 1; 9 | required sfixed64 request_id_lsb = 2; 10 | 11 | enum Status { 12 | OK = 0; 13 | ERROR = 1; 14 | HEAVYLOAD = 2; 15 | } 16 | optional Status status = 10 [default = OK]; 17 | 18 | required string message_name = 11; 19 | optional bytes message = 12; 20 | optional string error_message = 13; 21 | 22 | message Header { 23 | required string key = 1; 24 | optional string value = 2; 25 | } 26 | repeated Header header = 14; 27 | } 28 | 29 | message Node { 30 | required int32 id = 1; 31 | required string url = 2; 32 | repeated int32 partition = 3; 33 | optional int64 persistentCapability = 4; 34 | } 35 | -------------------------------------------------------------------------------- /cluster/src/main/protobuf/norbert_example.proto: -------------------------------------------------------------------------------- 1 | package norbert.example; 2 | 3 | option optimize_for = SPEED; 4 | option java_package = "com.linkedin.norbert.protos"; 5 | option java_outer_classname = "NorbertExampleProtos"; 6 | 7 | message Ping { 8 | required int64 timestamp = 1; 9 | } 10 | 11 | message PingResponse { 12 | required int64 timestamp = 1; 13 | } 14 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/NorbertException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | 18 | /** 19 | * Base exception class from which all other Norbert exceptions inherit. 20 | */ 21 | class NorbertException(message: String, cause: Throwable) extends RuntimeException(message, cause) { 22 | def this() = this(null, null) 23 | def this(message: String) = this(message, null) 24 | def this(cause: Throwable) = this(cause.getMessage, cause) 25 | } 26 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/cluster/ClusterClientComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | 19 | /** 20 | * A component which provides the client interface for interacting with a cluster. 21 | */ 22 | trait ClusterClientComponent { 23 | val clusterClient: ClusterClient 24 | } 25 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/cluster/ClusterDefaults.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | 19 | /** 20 | * A container for constants used in the ClusterClient. 21 | */ 22 | object ClusterDefaults { 23 | /** 24 | * The default ZooKeeper session timeout in milliseconds. 25 | */ 26 | val ZOOKEEPER_SESSION_TIMEOUT_MILLIS = 30000 27 | } 28 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/cluster/ClusterEvent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | 19 | sealed trait ClusterEvent 20 | 21 | object ClusterEvents { 22 | /** 23 | * ClusterEvent which indicates that you are now connected to the cluster. 24 | * 25 | * @param nodes the current list of Nodes stored in the cluster metadata 26 | * @param router a Router which is valid for the current state of the cluster 27 | */ 28 | case class Connected(nodes: Set[Node]) extends ClusterEvent 29 | 30 | /** 31 | * ClusterEvent which indicates that the cluster topology has changed. 32 | * 33 | * @param nodes the current list of Nodes stored in the cluster metadata 34 | * @param router a Router which is valid for the current state of the cluster 35 | */ 36 | case class NodesChanged(nodes: Set[Node]) extends ClusterEvent 37 | 38 | /** 39 | * ClusterEvent which indicates that the cluster is now disconnected. 40 | */ 41 | case object Disconnected extends ClusterEvent 42 | 43 | /** 44 | * ClusterEvent which indicates that the cluster is now shutdown. 45 | */ 46 | case object Shutdown extends ClusterEvent 47 | } 48 | 49 | /** 50 | * A trait to be implemented by classes which wish to receive cluster events. Register ClusterListeners 51 | * with ClusterClient#addListener(listener). 52 | */ 53 | trait ClusterListener { 54 | /** 55 | * Handle a cluster event. 56 | * 57 | * @param event the ClusterEvent to handle 58 | */ 59 | def handleClusterEvent(event: ClusterEvent): Unit 60 | } 61 | 62 | case class ClusterListenerKey(id: Long) 63 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/cluster/ClusterException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | 19 | /** 20 | * Base class for exceptions thrown by the ClusterClient. 21 | */ 22 | class ClusterException(message: String, cause: Throwable) extends NorbertException(message, cause) { 23 | def this() = this(null, null) 24 | def this(message: String) = this(message, null) 25 | def this(cause: Throwable) = this(cause.getMessage, cause) 26 | } 27 | 28 | /** 29 | * Exception that indicates that an operation was attempted when the current node was not connected to the cluster. 30 | */ 31 | class ClusterDisconnectedException(message: String) extends ClusterException(message) { 32 | def this() = this(null) 33 | } 34 | 35 | /** 36 | * Exception that indicates that an operation was attempted before start was called on the cluster. 37 | */ 38 | class ClusterNotStartedException extends ClusterException 39 | 40 | /** 41 | * Exception that indicates that an operation was attempted after shutdown was called on the cluster. 42 | */ 43 | class ClusterShutdownException extends ClusterException 44 | 45 | /** 46 | * Exception that indicates something was invalid on the node for which the operation was being performed. 47 | */ 48 | class InvalidNodeException(message: String, cause: Throwable) extends ClusterException(message) { 49 | def this(message: String) = this(message, null) 50 | } 51 | 52 | /** 53 | * Exception that indicates that something about the cluster was invalid when an operation was attempted. 54 | */ 55 | class InvalidClusterException(message: String, cause: Throwable) extends ClusterException(message) { 56 | def this(message: String) = this(message, null) 57 | } 58 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/cluster/common/ClusterManagerComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | package common 19 | 20 | import actors.Actor 21 | 22 | trait ClusterManagerComponent { 23 | val clusterManager: Actor 24 | 25 | sealed trait ClusterManagerMessage 26 | object ClusterManagerMessages { 27 | case class AddNode(node: Node) extends ClusterManagerMessage 28 | case class RemoveNode(nodeId: Int) extends ClusterManagerMessage 29 | case class MarkNodeAvailable(nodeId: Int, initialCapability: Long = 0L) extends ClusterManagerMessage 30 | case class MarkNodeUnavailable(nodeId: Int) extends ClusterManagerMessage 31 | case class SetNodeCapability(nodeId: Int, capability: Long) extends ClusterManagerMessage 32 | 33 | case object Shutdown extends ClusterManagerMessage 34 | 35 | case class ClusterManagerResponse(exception: Option[ClusterException]) extends ClusterManagerMessage 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/cluster/common/ClusterManagerHelper.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | package common 19 | 20 | trait ClusterManagerHelper { 21 | implicit def mapIntNodeToSetNode(map: collection.Map[Int, Node]): Set[Node] = map.foldLeft(Set[Node]()) { case (set, (key, node)) => set + node } 22 | } 23 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/cluster/memory/InMemoryClusterClient.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | package memory 19 | 20 | import common.ClusterNotificationManagerComponent 21 | 22 | class InMemoryClusterClient(val serviceName: String, override val clientName: Option[String] = None) extends ClusterClient with ClusterNotificationManagerComponent 23 | with InMemoryClusterManagerComponent{ 24 | val clusterNotificationManager = new ClusterNotificationManager 25 | val clusterManager = new InMemoryClusterManager 26 | } 27 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/cluster/memory/InMemoryClusterManagerComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | package memory 19 | 20 | import actors.Actor 21 | import Actor._ 22 | import common.{ClusterManagerComponent, ClusterNotificationManagerComponent, ClusterManagerHelper} 23 | 24 | trait InMemoryClusterManagerComponent extends ClusterManagerComponent with ClusterManagerHelper { 25 | this: ClusterNotificationManagerComponent => 26 | 27 | class InMemoryClusterManager extends Actor { 28 | private var currentNodes = scala.collection.mutable.Map[Int, Node]() 29 | private var available = scala.collection.mutable.Set[Int]() 30 | 31 | def act() = { 32 | actor { 33 | // Give the ClusterNotificationManager a chance to start 34 | Thread.sleep(100) 35 | clusterNotificationManager ! ClusterNotificationMessages.Connected(currentNodes) 36 | } 37 | 38 | while (true) { 39 | import ClusterManagerMessages._ 40 | 41 | receive { 42 | case AddNode(node) => if (currentNodes.contains(node.id)) { 43 | reply(ClusterManagerResponse(Some(new InvalidNodeException("A node with id %d already exists".format(node.id))))) 44 | } else { 45 | val n = if (available.contains(node.id)) node.copy(available = true) else node.copy(available = false) 46 | 47 | currentNodes += (n.id -> n) 48 | clusterNotificationManager ! ClusterNotificationMessages.NodesChanged(currentNodes) 49 | reply(ClusterManagerResponse(None)) 50 | } 51 | 52 | case RemoveNode(nodeId) => 53 | currentNodes -= nodeId 54 | clusterNotificationManager ! ClusterNotificationMessages.NodesChanged(currentNodes) 55 | reply(ClusterManagerResponse(None)) 56 | 57 | case MarkNodeAvailable(nodeId, initialCapability) => 58 | currentNodes.get(nodeId).foreach { node => currentNodes.update(nodeId, node.copy(available = true, capability = Some(initialCapability))) } 59 | available += nodeId 60 | clusterNotificationManager ! ClusterNotificationMessages.NodesChanged(currentNodes) 61 | reply(ClusterManagerResponse(None)) 62 | 63 | case MarkNodeUnavailable(nodeId) => 64 | currentNodes.get(nodeId).foreach { node => currentNodes.update(nodeId, node.copy(available = false, capability = None)) } 65 | available -= nodeId 66 | clusterNotificationManager ! ClusterNotificationMessages.NodesChanged(currentNodes) 67 | reply(ClusterManagerResponse(None)) 68 | 69 | case SetNodeCapability(nodeId, capability) => 70 | currentNodes.get(nodeId).foreach { node => currentNodes.update(nodeId, node.copy(capability = Some(capability)))} 71 | clusterManager ! ClusterNotificationMessages.NodesChanged(currentNodes) 72 | reply(ClusterManagerResponse(None)) 73 | 74 | case Shutdown => exit 75 | } 76 | } 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/cluster/zookeeper/ZooKeeperClusterClient.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | package zookeeper 19 | 20 | import common.ClusterNotificationManagerComponent 21 | 22 | class ZooKeeperClusterClient(override val clientName: Option[String], val serviceName: String, zooKeeperConnectString: String, zooKeeperSessionTimeoutMillis: Int) extends ClusterClient 23 | with ClusterNotificationManagerComponent with ZooKeeperClusterManagerComponent { 24 | val clusterNotificationManager = new ClusterNotificationManager 25 | val clusterManager = new ZooKeeperClusterManager(zooKeeperConnectString, zooKeeperSessionTimeoutMillis, serviceName) 26 | } 27 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/jmx/AverageTimeTracker.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package jmx 18 | 19 | import norbertutils._ 20 | import collection.JavaConversions 21 | import java.util.concurrent.atomic.AtomicInteger 22 | 23 | // Threadsafe. Writers should always complete more or less instantly. Readers work via copy-on-write. 24 | class FinishedRequestTimeTracker(clock: Clock, interval: Long) { 25 | private val q = new java.util.concurrent.ConcurrentLinkedQueue[(Long, Long)]() 26 | private val currentlyCleaning = new java.util.concurrent.atomic.AtomicBoolean 27 | 28 | private def clean { 29 | // Let only one thread clean at a time 30 | if(currentlyCleaning.compareAndSet(false, true)) { 31 | clean0 32 | currentlyCleaning.set(false) 33 | } 34 | } 35 | 36 | private def clean0 { 37 | while(!q.isEmpty) { 38 | val head = q.peek 39 | if(head == null) 40 | return 41 | 42 | val (completion, processingTime) = head 43 | if(clock.getCurrentTimeOffsetMicroseconds - completion > interval) { 44 | q.remove(head) 45 | } else { 46 | return 47 | } 48 | } 49 | } 50 | 51 | def addTime(processingTime: Long) { 52 | clean 53 | q.offer( (clock.getCurrentTimeOffsetMicroseconds, processingTime) ) 54 | } 55 | 56 | def getArray: Array[(Long, Long)] = { 57 | clean 58 | q.toArray(Array.empty[(Long, Long)]) 59 | } 60 | 61 | def getTimings: Array[Long] = { 62 | getArray.map(_._2).sorted 63 | } 64 | 65 | def total = { 66 | getTimings.sum 67 | } 68 | 69 | def reset { 70 | q.clear 71 | } 72 | } 73 | 74 | // Threadsafe 75 | class PendingRequestTimeTracker[KeyT](clock: Clock) { 76 | private val numRequests = new AtomicInteger() 77 | 78 | private val map : java.util.concurrent.ConcurrentMap[KeyT, Long] = 79 | new java.util.concurrent.ConcurrentHashMap[KeyT, Long] 80 | 81 | def getStartTime(key: KeyT) = Option(map.get(key)) 82 | 83 | def beginRequest(key: KeyT) { 84 | numRequests.incrementAndGet 85 | val now = clock.getCurrentTimeOffsetMicroseconds 86 | map.put(key, now) 87 | } 88 | 89 | def endRequest(key: KeyT) { 90 | map.remove(key) 91 | } 92 | 93 | def getTimings = { 94 | val now = clock.getCurrentTimeOffsetMicroseconds 95 | val timings = map.values.toArray(Array.empty[java.lang.Long]) 96 | timings.map(t => (now - t.longValue)).sorted 97 | } 98 | 99 | def reset { 100 | map.clear 101 | } 102 | 103 | def getTotalNumRequests = numRequests.get 104 | 105 | def total = getTimings.sum 106 | } 107 | 108 | class RequestTimeTracker[KeyT](clock: Clock, interval: Long) { 109 | val finishedRequestTimeTracker = new FinishedRequestTimeTracker(clock, interval) 110 | val pendingRequestTimeTracker = new PendingRequestTimeTracker[KeyT](clock) 111 | 112 | def beginRequest(key: KeyT) { 113 | pendingRequestTimeTracker.beginRequest(key) 114 | } 115 | 116 | def endRequest(key: KeyT) { 117 | pendingRequestTimeTracker.getStartTime(key).foreach { startTime => 118 | finishedRequestTimeTracker.addTime(clock.getCurrentTimeOffsetMicroseconds - startTime) 119 | } 120 | pendingRequestTimeTracker.endRequest(key) 121 | } 122 | 123 | def reset { 124 | finishedRequestTimeTracker.reset 125 | pendingRequestTimeTracker.reset 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/jmx/JMX.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package jmx 18 | 19 | import management.ManagementFactory 20 | import javax.management.{ObjectInstance, ObjectName, StandardMBean} 21 | import logging.Logging 22 | 23 | object JMX extends Logging { 24 | private val mbeanServer = ManagementFactory.getPlatformMBeanServer 25 | 26 | def register(mbean: AnyRef, name: String): Option[ObjectInstance] = if (System.getProperty("com.linkedin.norbert.disableJMX") == null) try { 27 | Some(mbeanServer.registerMBean(mbean, new ObjectName(getUniqueName(name)))) 28 | } catch { 29 | case ex: Exception => 30 | log.error(ex, "Error when registering mbean: %s".format(mbean)) 31 | None 32 | } else { 33 | None 34 | } 35 | 36 | val map = collection.mutable.Map.empty[String, Int] 37 | 38 | def getUniqueName(name: String): String = synchronized { 39 | val id = map.getOrElse(name, -1) 40 | val unique = if(id == -1) name else name + "-" + id 41 | map + (name -> (id + 1)) 42 | unique 43 | } 44 | 45 | 46 | def register(mbean: MBean): Option[ObjectInstance] = register(mbean, mbean.name) 47 | 48 | def register(mbean: Option[MBean]): Option[ObjectInstance] = mbean.flatMap(m => register(m, m.name)) 49 | 50 | def unregister(mbean: ObjectInstance) = try { 51 | mbeanServer.unregisterMBean(mbean.getObjectName) 52 | synchronized { map.remove(mbean.getObjectName.getCanonicalName) } 53 | } catch { 54 | case ex: Exception => log.error(ex, "Error while unregistering mbean: %s".format(mbean.getObjectName)) 55 | } 56 | 57 | def name(clientName: Option[String], serviceName: String) = 58 | if(clientName.isDefined) 59 | "client=%s,service=%s".format(clientName.get, serviceName) 60 | else 61 | "service=%s".format(serviceName) 62 | 63 | class MBean(klass: Class[_], namePropeties: String) extends StandardMBean(klass) { 64 | def this(klass: Class[_]) = this(klass, null) 65 | 66 | def name: String = { 67 | val simpleName = klass.getSimpleName 68 | val mbeanIndex = simpleName.lastIndexOf("MBean") 69 | 70 | val base = "com.linkedin.norbert:type=%s".format(if (mbeanIndex == -1) simpleName else simpleName.substring(0, mbeanIndex)) 71 | if (namePropeties != null) "%s,%s".format(base, namePropeties) else base 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/logging/Logging.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package logging 18 | 19 | /** 20 | * A mixin trait which provides a Logger instance. 21 | */ 22 | trait Logging { 23 | protected val log: Logger = Logger(this) 24 | } 25 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/norbertutils/Clock.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | 17 | package com.linkedin.norbert.norbertutils 18 | 19 | trait ClockComponent { 20 | val clock: Clock 21 | } 22 | 23 | trait Clock { 24 | def getCurrentTimeMilliseconds: Long 25 | //do not use this for absolute time 26 | //only for computing intervals 27 | def getCurrentTimeOffsetMicroseconds: Long 28 | } 29 | 30 | object MockClock extends Clock { 31 | var currentTime = 0L 32 | override def getCurrentTimeMilliseconds = currentTime 33 | override def getCurrentTimeOffsetMicroseconds = currentTime 34 | } 35 | 36 | object SystemClock extends Clock { 37 | def getCurrentTimeOffsetMicroseconds = System.nanoTime/1000 38 | def getCurrentTimeMilliseconds = System.currentTimeMillis 39 | } 40 | 41 | object SystemClockComponent extends ClockComponent { 42 | val clock = SystemClock 43 | } 44 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/norbertutils/KeepAliveActor.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | 17 | package com.linkedin.norbert.norbertutils 18 | 19 | import com.linkedin.norbert.logging.Logging 20 | import actors.{DaemonActor, Actor} 21 | 22 | object KeepAliveActor extends DaemonActor with Logging { 23 | def act() = loop { 24 | react { 25 | case scala.actors.Exit(actor: Actor, cause: Throwable) => 26 | log.error(cause, "Actor " + actor + " seems to have died. Restarting.") 27 | actor.restart 28 | actor ! LinkActor(this) 29 | 30 | case _ => 31 | } 32 | } 33 | 34 | this.start 35 | this.trapExit = true 36 | } 37 | 38 | case class LinkActor(actor: Actor) 39 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/norbertutils/NamedPoolThreadFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package norbertutils 18 | 19 | import java.util.concurrent.ThreadFactory 20 | import java.util.concurrent.atomic.AtomicInteger 21 | 22 | class NamedPoolThreadFactory(poolName: String) extends ThreadFactory { 23 | private val threadCount = new AtomicInteger(1) 24 | private val nameFormat = "%s-thread-%d" 25 | 26 | def newThread(r: Runnable) = new Thread(Thread.currentThread.getThreadGroup, r, nameFormat.format(poolName, threadCount.getAndIncrement)) 27 | } 28 | -------------------------------------------------------------------------------- /cluster/src/main/scala/com/linkedin/norbert/norbertutils/package.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert 2 | 3 | /* 4 | * Copyright 2009-2010 LinkedIn, Inc 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 7 | * use this file except in compliance with the License. You may obtain a copy of 8 | * the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | * License for the specific language governing permissions and limitations under 16 | * the License. 17 | */ 18 | 19 | import annotation.tailrec 20 | import java.util.concurrent.{ConcurrentMap} 21 | 22 | package object norbertutils { 23 | def binarySearch[T](array: Seq[T], value: T)(implicit ordering: Ordering[T]): Int = binarySearch(array, value, 0, array.length - 1) 24 | 25 | @tailrec 26 | private def binarySearch[T](array: Seq[T], value: T, lo: Int, hi: Int)(implicit ordering: Ordering[T]): Int = { 27 | if(lo > hi) -lo - 1 28 | else { 29 | val mid = lo + ((hi - lo) >> 2) 30 | val middleValue = array(mid) 31 | if(ordering.gt(value, middleValue)) 32 | binarySearch(array, value, mid + 1, hi) 33 | else if(ordering.lt(value, middleValue)) 34 | binarySearch(array, value, lo, mid - 1) 35 | else mid 36 | } 37 | } 38 | 39 | def getOrElse[T](seq: Seq[T], index: Int, other: T): T = { 40 | if(0 <= index && index < seq.size) seq(index) 41 | else other 42 | } 43 | 44 | // TODO: Put this into a utility somewhere? Scala's concurrent getOrElseUpdate is not atomic, unlike this guy 45 | def atomicCreateIfAbsent[K, V](map: ConcurrentMap[K, V], key: K)(fn: K => V): V = { 46 | val oldValue = map.get(key) 47 | if(oldValue == null) { 48 | map.synchronized { 49 | val oldValue2 = map.get(key) 50 | if(oldValue2 == null) { 51 | val newValue = fn(key) 52 | map.putIfAbsent(key, newValue) 53 | map.get(key) 54 | } else { 55 | oldValue2 56 | } 57 | } 58 | } else { 59 | oldValue 60 | } 61 | } 62 | 63 | def safeDivide(num: Double, den: Double)(orElse: Double): Double = { 64 | if(den == 0) orElse 65 | else num / den 66 | } 67 | 68 | // Apparently the map in JavaConversions isn't serializable... 69 | def toJMap[K, V](map: Map[K, V]): java.util.Map[K, V] = { 70 | val m = new java.util.HashMap[K, V](map.size) 71 | map.foreach { case (k, v) => m.put(k, v) } 72 | m 73 | } 74 | 75 | def toJMap[K, V](map: Option[Map[K, V]]): java.util.Map[K, V] = 76 | toJMap(map.getOrElse(Map.empty[K, V])) 77 | 78 | def calculatePercentile[T](values: Array[T], percentile: Double, default: Double = 0.0)(implicit n: Numeric[T]): Double = { 79 | import math._ 80 | 81 | if(values.isEmpty) 82 | return default 83 | 84 | val p = max(0.0, min(1.0, percentile)) 85 | 86 | var idx = p * (values.size - 1) 87 | idx = max(0.0, min(values.size - 1, idx)) 88 | 89 | val (lIdx, rIdx) = (idx.floor.toInt, idx.ceil.toInt) 90 | 91 | if(idx == lIdx) 92 | n.toDouble(values(lIdx)) 93 | else { 94 | // Linearly Interpolate between the two 95 | (idx - lIdx) * n.toDouble(values(rIdx)) + (rIdx - idx) * n.toDouble(values(lIdx)) 96 | } 97 | } 98 | 99 | def continueOnError(block: => Unit): Unit = { 100 | try { 101 | block 102 | } catch 103 | { 104 | case e : Exception => // nothing 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /cluster/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=FATAL, CONSOLE 2 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 3 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 4 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n 5 | -------------------------------------------------------------------------------- /cluster/src/test/scala/com/linkedin/norbert/cluster/NodeSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | 19 | import org.specs.Specification 20 | import protos.NorbertProtos 21 | 22 | class NodeSpec extends Specification { 23 | "Node" should { 24 | "serialize into the correct format" in { 25 | val builder = NorbertProtos.Node.newBuilder 26 | builder.setId(1) 27 | builder.setUrl("localhost:31313") 28 | builder.addPartition(0).addPartition(1) 29 | builder.setPersistentCapability(2L)//right not the only one we care about is 2 30 | val expectedBytes = builder.build.toByteArray.toList 31 | 32 | val nodeBytes = Node.nodeToByteArray(Node(1, "localhost:31313", false, Set(0, 1), Some(0L), Some(2L))) 33 | nodeBytes.toList must be_==(expectedBytes) 34 | } 35 | 36 | "deserialize into the corrent Node" in { 37 | val builder = NorbertProtos.Node.newBuilder 38 | builder.setId(1) 39 | builder.setUrl("localhost:31313") 40 | builder.addPartition(0).addPartition(1) 41 | builder.setPersistentCapability(2L) 42 | val bytes = builder.build.toByteArray 43 | 44 | val node = Node(1, "localhost:31313", true, Set(0, 1), Some(0L)) 45 | Node(1, bytes, true, Some(0L)) must be_==(node) 46 | } 47 | 48 | "have a sane equals method" in { 49 | val url = "localhost:31313" 50 | val node1 = Node(1, url, true, Set(0, 1)) 51 | val node2 = Node(1, url, false, Set(2, 3)) 52 | val node3 = Node(1, url, true, Set(4, 5)) 53 | 54 | // Reflexive 55 | node1 must be_==(node1) 56 | 57 | // Symmetric 58 | node1 must be_==(node2) 59 | node2 must be_==(node1) 60 | 61 | // Transitive 62 | node1 must be_==(node2) 63 | node2 must be_==(node3) 64 | node3 must be_==(node1) 65 | 66 | // Consistent already handled above 67 | 68 | // Handles null 69 | node1 must be_!=(null) 70 | 71 | // Hashcode 72 | node1.hashCode must be_==(node2.hashCode) 73 | } 74 | 75 | "be equal to another node if they have the same id and url" in { 76 | val url = "localhost:31313" 77 | val node1 = Node(1, url, true, Set(0, 1)) 78 | val node2 = Node(1, url, false, Set(1, 2)) 79 | node1 must be_==(node2) 80 | } 81 | 82 | "not be equal to another node if they have a different id" in { 83 | val url = "localhost:31313" 84 | val node1 = Node(1, url, true, Set(0, 1)) 85 | val node2 = Node(2, url, false, Set(1, 2)) 86 | node1 must be_!=(node2) 87 | } 88 | 89 | "not be equal to another node if they have a different url" in { 90 | val node1 = Node(1, "localhost:31313", true, Set(0, 1)) 91 | val node2 = Node(1, "localhost:16161", true, Set(0, 1)) 92 | node1 must be_!=(node2) 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /cluster/src/test/scala/com/linkedin/norbert/cluster/memory/InMemoryClusterClientSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | package memory 19 | 20 | import org.specs.Specification 21 | 22 | class InMemoryClusterClientSpec extends Specification { 23 | val clusterClient = new InMemoryClusterClient("test") 24 | clusterClient.start 25 | clusterClient.awaitConnectionUninterruptibly 26 | 27 | "InMemoryClusterClient" should { 28 | 29 | doAfter { 30 | clusterClient.shutdown 31 | } 32 | 33 | "start with no nodes" in { 34 | clusterClient.nodes.size must be_==(0) 35 | } 36 | 37 | "add the node" in { 38 | clusterClient.addNode(1, "test") must notBeNull 39 | val nodes = clusterClient.nodes 40 | nodes.size must be_==(1) 41 | nodes.foreach { node => 42 | node.id must be_==(1) 43 | node.url must be_==("test") 44 | node.available must beFalse 45 | } 46 | } 47 | 48 | "throw an InvalidNodeException if the node already exists" in { 49 | clusterClient.addNode(1, "test") must notBeNull 50 | clusterClient.addNode(1, "test") must throwA[InvalidNodeException] 51 | } 52 | 53 | "add the node as available" in { 54 | clusterClient.markNodeAvailable(1) 55 | clusterClient.addNode(1, "test") 56 | val nodes = clusterClient.nodes 57 | nodes.foreach { node => 58 | node.available must beTrue 59 | } 60 | } 61 | 62 | "remove the node" in { 63 | clusterClient.addNode(1, "test") 64 | clusterClient.nodes.size must be_==(1) 65 | clusterClient.removeNode(1) 66 | clusterClient.nodes.size must be_==(0) 67 | } 68 | 69 | "mark the node available" in { 70 | clusterClient.addNode(1, "test") 71 | var nodes = clusterClient.nodes 72 | nodes.foreach { node => 73 | node.available must beFalse 74 | } 75 | clusterClient.markNodeAvailable(1) 76 | nodes = clusterClient.nodes 77 | nodes.foreach { node => 78 | node.available must beTrue 79 | } 80 | } 81 | 82 | "mark the node unavailable" in { 83 | clusterClient.markNodeAvailable(1) 84 | clusterClient.addNode(1, "test") 85 | var nodes = clusterClient.nodes 86 | nodes.foreach { node => 87 | node.available must beTrue 88 | } 89 | clusterClient.markNodeUnavailable(1) 90 | nodes = clusterClient.nodes 91 | nodes.foreach { node => 92 | node.available must beFalse 93 | } 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /cluster/src/test/scala/com/linkedin/norbert/jmx/AverageTimeTrackerSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package jmx 18 | 19 | import org.specs.Specification 20 | import norbertutils.{SystemClock, MockClock, Clock, ClockComponent} 21 | 22 | class AverageTimeTrackerSpec extends Specification { 23 | "RequestTimeTracker" should { 24 | "correctly average the times provided" in { 25 | val a = new FinishedRequestTimeTracker(MockClock, 100) 26 | (1 to 100).foreach{ t => 27 | a.addTime(t) 28 | MockClock.currentTime = t 29 | } 30 | a.total must be_==(5050) 31 | 32 | a.addTime(101) 33 | MockClock.currentTime = 101 34 | 35 | a.total must be_==(5150) // first one gets knocked out 36 | } 37 | 38 | "Correctly calculate unfinished times" in { 39 | val tracker = new PendingRequestTimeTracker[Int](MockClock) 40 | 41 | (0 until 10).foreach { i => 42 | MockClock.currentTime = 1000L * i 43 | tracker.beginRequest(i) 44 | (tracker.total / (i + 1)) must be_==(1000L * i / 2) 45 | } 46 | } 47 | 48 | } 49 | } -------------------------------------------------------------------------------- /cluster/src/test/scala/com/linkedin/norbert/norbertutils/KeepAliveActorSpec.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.norbertutils 2 | 3 | /* 4 | * Copyright 2009-2010 LinkedIn, Inc 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 7 | * use this file except in compliance with the License. You may obtain a copy of 8 | * the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | * License for the specific language governing permissions and limitations under 16 | * the License. 17 | */ 18 | import org.specs.Specification 19 | import org.specs.mock.Mockito 20 | import org.specs.util.WaitFor 21 | import actors._ 22 | import actors.Actor._ 23 | 24 | class KeepAliveActorSpec extends Specification { 25 | case class Divide(n: Int, d: Int) 26 | 27 | class TestActor extends Actor { 28 | def act() = loop { 29 | react { 30 | case LinkActor(actor) => this.link(actor) 31 | case Divide(n, d) => reply(n / d) 32 | } 33 | } 34 | 35 | this ! LinkActor(KeepAliveActor) 36 | } 37 | "KeepAliveActor" should { 38 | "Keep an actor alive" in { 39 | val actor = new TestActor 40 | actor.start 41 | 42 | (actor !? (1000, Divide(10, 2))) must be_==(Some(5)) 43 | 44 | actor !? (1000, Divide(10, 0)) must be_==(None) 45 | 46 | actor !? (1000, Divide(10, 2)) must be_==(Some(5)) 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /defaultEnvironment.gradle: -------------------------------------------------------------------------------- 1 | subprojects { 2 | repositories { 3 | mavenCentral() 4 | } 5 | } 6 | 7 | -------------------------------------------------------------------------------- /examples/src/main/java/com/linkedin/norbert/javacompat/network/NorbertJavaNetworkClientMain.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | import com.linkedin.norbert.javacompat.cluster.ClusterClient; 19 | import com.linkedin.norbert.javacompat.cluster.Node; 20 | import com.linkedin.norbert.javacompat.cluster.ZooKeeperClusterClient; 21 | import org.jboss.netty.logging.InternalLoggerFactory; 22 | import org.jboss.netty.logging.Log4JLoggerFactory; 23 | 24 | import java.util.concurrent.ExecutionException; 25 | import java.util.concurrent.Future; 26 | import java.util.concurrent.TimeUnit; 27 | import java.util.concurrent.TimeoutException; 28 | 29 | public class NorbertJavaNetworkClientMain { 30 | public static void main(String[] args) { 31 | InternalLoggerFactory.setDefaultFactory(new Log4JLoggerFactory()); 32 | 33 | ClusterClient cc = new ZooKeeperClusterClient(null, args[0], args[1], 30000); 34 | NetworkClientConfig config = new NetworkClientConfig(); 35 | config.setClusterClient(cc); 36 | NetworkClient nc = new NettyNetworkClient(config, new RoundRobinLoadBalancerFactory()); 37 | // PartitionedNetworkClient nc = new NettyPartitionedNetworkClient(config, new IntegerConsistentHashPartitionedLoadBalancerFactory()); 38 | // nc.registerRequest(NorbertExampleProtos.Ping.getDefaultInstance(), NorbertExampleProtos.PingResponse.getDefaultInstance()); 39 | 40 | Node node = cc.getNodeWithId(1); 41 | 42 | Future f = nc.sendRequestToNode(new Ping(System.currentTimeMillis()), node, new PingSerializer()); 43 | try { 44 | Ping pong = f.get(750, TimeUnit.MILLISECONDS); 45 | System.out.println(String.format("Ping took %dms", System.currentTimeMillis() - pong.timestamp)); 46 | } catch (InterruptedException e) { 47 | e.printStackTrace(); 48 | } catch (ExecutionException e) { 49 | e.printStackTrace(); 50 | } catch (TimeoutException e) { 51 | e.printStackTrace(); 52 | } 53 | 54 | cc.shutdown(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /examples/src/main/java/com/linkedin/norbert/javacompat/network/NorbertJavaNetworkServerMain.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | import org.jboss.netty.logging.InternalLoggerFactory; 19 | import org.jboss.netty.logging.Log4JLoggerFactory; 20 | 21 | public class NorbertJavaNetworkServerMain { 22 | public static void main(String[] args) { 23 | InternalLoggerFactory.setDefaultFactory(new Log4JLoggerFactory()); 24 | 25 | NetworkServerConfig config = new NetworkServerConfig(); 26 | config.setServiceName(args[0]); 27 | config.setZooKeeperConnectString(args[1]); 28 | config.setZooKeeperSessionTimeoutMillis(30000); 29 | final NetworkServer ns = new NettyNetworkServer(config); 30 | ns.registerHandler(new PingHandler(), new PingSerializer()); 31 | ns.bind(Integer.parseInt(args[2])); 32 | 33 | Runtime.getRuntime().addShutdownHook(new Thread() { 34 | @Override 35 | public void run() { 36 | ns.shutdown(); 37 | } 38 | }); 39 | } 40 | 41 | private static class PingHandler implements RequestHandler { 42 | public Ping handleRequest(Ping ping) throws Exception { 43 | System.out.printf("Requested ping from client %d milliseconds ago (assuming synchronized clocks)", (ping.timestamp - System.currentTimeMillis())); 44 | return new Ping(System.currentTimeMillis()); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /examples/src/main/java/com/linkedin/norbert/javacompat/network/Ping.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | import com.google.protobuf.InvalidProtocolBufferException; 19 | import com.linkedin.norbert.network.Serializer; 20 | import com.linkedin.norbert.protos.NorbertExampleProtos; 21 | 22 | class Ping { 23 | public final long timestamp; 24 | 25 | public Ping(long timestamp) { 26 | this.timestamp = timestamp; 27 | } 28 | } 29 | 30 | class PingSerializer implements Serializer { 31 | public String requestName() { 32 | return "ping"; 33 | } 34 | 35 | public String responseName() { 36 | return "pong"; 37 | } 38 | 39 | public byte[] requestToBytes(Ping message) { 40 | return NorbertExampleProtos.Ping.newBuilder().setTimestamp(message.timestamp).build().toByteArray(); 41 | } 42 | 43 | public Ping requestFromBytes(byte[] bytes) { 44 | try { 45 | return new Ping(NorbertExampleProtos.Ping.newBuilder().mergeFrom(bytes).build().getTimestamp()); 46 | } catch (InvalidProtocolBufferException e) { 47 | System.out.println("Invalid protocol buffer exception " + e.getMessage()); 48 | throw new IllegalArgumentException(e); 49 | } 50 | } 51 | 52 | public byte[] responseToBytes(Ping message) { 53 | return requestToBytes(message); 54 | } 55 | 56 | public Ping responseFromBytes(byte[] bytes) { 57 | return requestFromBytes(bytes); 58 | } 59 | } 60 | 61 | -------------------------------------------------------------------------------- /examples/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=WARN, CONSOLE, R 2 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 3 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 4 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n 5 | 6 | #log4j.appender.R=org.apache.log4j.RollingFileAppender 7 | #log4j.appender.R.File=/tmp/norbert.log 8 | #log4j.appender.R.MaxFileSize=100KB 9 | #log4j.appender.R.MaxBackupIndex=1 10 | #log4j.appender.R.layout=org.apache.log4j.PatternLayout 11 | #log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n 12 | 13 | #log4j.category.com.linkedin.norbert=debug 14 | -------------------------------------------------------------------------------- /examples/src/main/scala/com/linkedin/norbert/cluster/NorbertClusterClientMain.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package cluster 18 | 19 | object NorbertClusterClientMain { 20 | private var cluster: ClusterClient = _ 21 | 22 | def main(args: Array[String]) { 23 | cluster = ClusterClient(args(0), args(1), 30000) 24 | cluster.awaitConnectionUninterruptibly 25 | loop 26 | } 27 | 28 | private def loop { 29 | print("> ") 30 | var line = Console.in.readLine.trim 31 | while (line != null) { 32 | try { 33 | if (line.length > 0) processCommand(line) 34 | } catch { 35 | case ex: Exception => println("Error: %s".format(ex)) 36 | } 37 | 38 | print("> ") 39 | line = Console.in.readLine.trim 40 | } 41 | } 42 | 43 | private def processCommand(line: String) { 44 | val command :: args = line.split(" ").toList.map(_.trim).filter(_.length > 0) 45 | 46 | command match { 47 | case "nodes" => 48 | val ts = System.currentTimeMillis 49 | val nodes = cluster.nodes.toArray 50 | val sortedNodes = nodes.sortWith((node1, node2) => node1.id < node2.id) 51 | if (nodes.size > 0) println(sortedNodes.mkString("\n")) else println("The cluster has no nodes") 52 | 53 | case "join" => 54 | args match { 55 | case nodeId :: url :: Nil => 56 | cluster.addNode(nodeId.toInt, url) 57 | println("Joined Norbert cluster") 58 | 59 | case nodeId :: url :: partitions => 60 | cluster.addNode(nodeId.toInt, url, Set() ++ partitions.map(_.toInt)) 61 | println("Joined Norbert cluster") 62 | 63 | case _ => println("Error: Invalid syntax: join nodeId url partition1 partition2...") 64 | } 65 | println("Joined Norbert cluster") 66 | 67 | case "leave" => 68 | if (args.length < 1) { 69 | println("Invalid syntax: leave nodeId") 70 | } else { 71 | cluster.removeNode(args.head.toInt) 72 | println("Left Norbert cluster") 73 | } 74 | 75 | case "down" => 76 | if (args.length < 1) { 77 | println("Invalid syntax: join nodeId") 78 | } else { 79 | cluster.markNodeUnavailable(args.head.toInt) 80 | println("Marked node offline") 81 | } 82 | 83 | case "up" => 84 | if (args.length < 1) { 85 | println("Invalid syntax: join nodeId") 86 | } else { 87 | cluster.markNodeAvailable(args.head.toInt) 88 | println("Marked node online") 89 | } 90 | 91 | case "exit" => exit 92 | 93 | case "quit" => exit 94 | 95 | case msg => "Unknown command: " + msg 96 | } 97 | } 98 | 99 | private def exit { 100 | cluster.shutdown 101 | System.exit(0) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /examples/src/main/scala/com/linkedin/norbert/network/NorbertNetworkServerMain.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | 19 | 20 | import server.{RequestContext, NetworkServer} 21 | import netty.{RequestContext => NettyRequestContext, NetworkServerConfig, NettyServerFilter} 22 | import org.jboss.netty.logging.{InternalLoggerFactory, Log4JLoggerFactory} 23 | import com.google.protobuf.Message 24 | import protos.NorbertExampleProtos 25 | import cluster.ClusterClient 26 | import norbertutils._ 27 | import network.NorbertNetworkServerMain.LogFilter 28 | import protos.NorbertProtos.NorbertMessage 29 | 30 | 31 | object NorbertNetworkServerMain { 32 | InternalLoggerFactory.setDefaultFactory(new Log4JLoggerFactory) 33 | 34 | def main(args: Array[String]) { 35 | val cc = ClusterClient(args(0), args(1), 30000) 36 | cc.awaitConnectionUninterruptibly 37 | cc.removeNode(1) 38 | cc.addNode(1, "localhost:31313", Set()) 39 | 40 | val config = new NetworkServerConfig 41 | config.clusterClient = cc 42 | 43 | val ns = NetworkServer(config) 44 | 45 | ns.registerHandler(pingHandler) 46 | ns.addFilters(List(new LogFilter)) 47 | 48 | ns.bind(args(2).toInt) 49 | 50 | Runtime.getRuntime.addShutdownHook(new Thread { 51 | override def run = { 52 | cc.shutdown 53 | } 54 | }) 55 | } 56 | 57 | private def pingHandler(ping: Ping): Pong = { 58 | println("Requested ping from client %d milliseconds ago (assuming synchronized clocks)".format(ping.timestamp - System.currentTimeMillis) ) 59 | Pong(System.currentTimeMillis) 60 | } 61 | 62 | class LogFilter extends NettyServerFilter { 63 | val clock = SystemClock 64 | def onRequest(request: Any, context: RequestContext) 65 | { context.attributes += ("START_TIMER" -> clock.getCurrentTimeOffsetMicroseconds) } 66 | 67 | def onResponse(response: Any, context: RequestContext) 68 | { val start: Long = context.attributes.getOrElse("START_TIMER", -1).asInstanceOf[Long] 69 | println("server side time logging: " + (clock.getCurrentTimeOffsetMicroseconds - start) + " micros.") 70 | } 71 | 72 | def onMessage(message: NorbertMessage, context: RequestContext) = 73 | { context.attributes += ("PRE_SERIALIZATION" -> clock.getCurrentTimeOffsetMicroseconds) } 74 | 75 | def postMessage(message: NorbertMessage, context: RequestContext) = 76 | { 77 | val start: Long = context.attributes.getOrElse("PRE_SERIALIZATION", -1).asInstanceOf[Long] 78 | println("server side time logging including serialization: " + (clock.getCurrentTimeOffsetMicroseconds - start) + " micros.") 79 | } 80 | 81 | def onError(error: Exception, context: RequestContext) 82 | {} 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /examples/src/main/scala/com/linkedin/norbert/network/PingRequest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.network 17 | 18 | import com.linkedin.norbert.protos.NorbertExampleProtos 19 | 20 | object Ping { 21 | implicit case object PingSerializer extends Serializer[Ping, Pong] { 22 | def requestName = "ping" 23 | def responseName = "pong" 24 | 25 | def requestToBytes(message: Ping) = 26 | NorbertExampleProtos.Ping.newBuilder.setTimestamp(message.timestamp).build.toByteArray 27 | 28 | def requestFromBytes(bytes: Array[Byte]) = { 29 | Ping(NorbertExampleProtos.Ping.newBuilder.mergeFrom(bytes).build.getTimestamp) 30 | } 31 | 32 | def responseToBytes(message: Pong) = 33 | NorbertExampleProtos.PingResponse.newBuilder.setTimestamp(message.timestamp).build.toByteArray 34 | 35 | def responseFromBytes(bytes: Array[Byte]) = 36 | Pong(NorbertExampleProtos.PingResponse.newBuilder.mergeFrom(bytes).build.getTimestamp) 37 | } 38 | } 39 | 40 | case class Ping(timestamp: Long) 41 | case class Pong(timestamp: Long) 42 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | version=0.6.34 2 | -------------------------------------------------------------------------------- /java-cluster/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'scala' 3 | 4 | dependencies { 5 | compile project(':cluster') 6 | compile externalDependency.scalaLibrary 7 | 8 | scalaTools externalDependency.scalaCompiler 9 | scalaTools externalDependency.scalaLibrary 10 | } 11 | 12 | -------------------------------------------------------------------------------- /java-cluster/src/main/java/com/linkedin/norbert/javacompat/cluster/ClusterListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.cluster; 17 | 18 | import java.util.Set; 19 | 20 | public interface ClusterListener { 21 | /** 22 | * Handle the case that you are now connected to the cluster. 23 | * 24 | * @param nodes the current list of available Nodes stored in the cluster metadata 25 | */ 26 | void handleClusterConnected(Set nodes); 27 | 28 | /** 29 | * Handle the case that the cluster topology has changed. 30 | * 31 | * @param nodes the current list of availableNodes stored in the cluster metadata 32 | */ 33 | void handleClusterNodesChanged(Set nodes); 34 | 35 | /** 36 | * Handle the case that the cluster is now disconnected. 37 | */ 38 | void handleClusterDisconnected(); 39 | 40 | /** 41 | * Handle the case that the cluster is now shutdown. 42 | */ 43 | void handleClusterShutdown(); 44 | } 45 | -------------------------------------------------------------------------------- /java-cluster/src/main/java/com/linkedin/norbert/javacompat/cluster/ClusterListenerAdapter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.cluster; 17 | 18 | import java.util.Set; 19 | 20 | public class ClusterListenerAdapter implements ClusterListener { 21 | 22 | public void handleClusterConnected(Set nodes) { 23 | // do nothing 24 | } 25 | 26 | public void handleClusterNodesChanged(Set nodes) { 27 | // do nothing 28 | } 29 | 30 | public void handleClusterDisconnected() { 31 | // do nothing 32 | } 33 | 34 | public void handleClusterShutdown() { 35 | // do nothing 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /java-cluster/src/main/java/com/linkedin/norbert/javacompat/cluster/Node.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.cluster; 17 | 18 | import java.util.Set; 19 | 20 | public interface Node { 21 | int getId(); 22 | String getUrl(); 23 | Set getPartitionIds(); 24 | boolean isAvailable(); 25 | boolean isCapableOf(Long c); 26 | boolean isCapableOf(Long c, Long pc); 27 | } 28 | -------------------------------------------------------------------------------- /java-cluster/src/main/scala/com/linkedin/norbert/javacompat/cluster/BaseClusterClient.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat 17 | package cluster 18 | 19 | import java.util.concurrent.TimeUnit 20 | import com.linkedin.norbert.cluster.{ClusterEvents, ClusterEvent, ClusterListenerKey} 21 | 22 | abstract class BaseClusterClient extends ClusterClient { 23 | 24 | val underlying: com.linkedin.norbert.cluster.ClusterClient 25 | 26 | def shutdown = underlying.shutdown 27 | 28 | def awaitConnectionUninterruptibly = underlying.awaitConnectionUninterruptibly 29 | 30 | def awaitConnection(timeout: Long, unit: TimeUnit) = underlying.awaitConnection(timeout, unit) 31 | 32 | def awaitConnection = underlying.awaitConnection 33 | 34 | def isShutdown = underlying.isShutdown 35 | 36 | def isConnected = underlying.isConnected 37 | 38 | def removeListener(key: ClusterListenerKey) = underlying.removeListener(key) 39 | 40 | def addListener(listener: ClusterListener) = underlying.addListener(new com.linkedin.norbert.cluster.ClusterListener { 41 | import ClusterEvents._ 42 | 43 | def handleClusterEvent(event: ClusterEvent) = event match { 44 | case Connected(nodes) => listener.handleClusterConnected(nodes) 45 | case NodesChanged(nodes) => listener.handleClusterNodesChanged(nodes) 46 | case Disconnected => listener.handleClusterDisconnected 47 | case Shutdown => listener.handleClusterShutdown 48 | } 49 | }) 50 | 51 | def markNodeUnavailable(nodeId: Int) = underlying.markNodeUnavailable(nodeId) 52 | 53 | def markNodeAvailable(nodeId: Int) = underlying.markNodeAvailable(nodeId) 54 | 55 | def markNodeAvailable(nodeId: Int, capacity: Long) = underlying.markNodeAvailable(nodeId, capacity) 56 | 57 | def removeNode(nodeId: Int) = underlying.removeNode(nodeId) 58 | 59 | def addNode(nodeId: Int, url: String, partitions: java.util.Set[java.lang.Integer]) = { 60 | underlying.addNode(nodeId, url, partitions.asInstanceOf[java.util.Set[Int]]) 61 | } 62 | 63 | def addNode(nodeId: Int, url: String) = underlying.addNode(nodeId, url) 64 | 65 | def getNodeWithId(nodeId: Int) = underlying.nodeWithId(nodeId).getOrElse(null) 66 | 67 | def getNodes = underlying.nodes 68 | 69 | def getServiceName = underlying.serviceName 70 | 71 | def getClientName = underlying.clientName.orNull 72 | } 73 | -------------------------------------------------------------------------------- /java-cluster/src/main/scala/com/linkedin/norbert/javacompat/cluster/InMemoryClusterClient.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat 17 | package cluster 18 | 19 | class InMemoryClusterClient(clientName: String, serviceName: String) extends BaseClusterClient { 20 | val underlying = new com.linkedin.norbert.cluster.memory.InMemoryClusterClient(serviceName, Option(clientName)) 21 | underlying.start 22 | } 23 | -------------------------------------------------------------------------------- /java-cluster/src/main/scala/com/linkedin/norbert/javacompat/cluster/JavaNode.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat 17 | package cluster 18 | 19 | import reflect.BeanProperty 20 | 21 | object JavaNode { 22 | def apply(node: com.linkedin.norbert.cluster.Node): Node = { 23 | if (node == null) { 24 | null 25 | } else { 26 | var s = new java.util.HashSet[java.lang.Integer] 27 | if (node.partitionIds != null) { 28 | node.partitionIds.foreach {id => s.add(id)} 29 | } 30 | JavaNode(node.id, node.url, node.available, s, node.capability, node.persistentCapability) 31 | } 32 | } 33 | } 34 | 35 | case class JavaNode(@BeanProperty id: Int, @BeanProperty url: String, @BeanProperty available: Boolean, @BeanProperty partitionIds: java.util.Set[java.lang.Integer], capability: Option[Long] = None, persistentCapability: Option[Long] = None) extends Node { 36 | def isAvailable = available 37 | def isCapableOf(c: java.lang.Long) : Boolean = isCapableOf(c, 0L) 38 | def isCapableOf(c: java.lang.Long, pc: java.lang.Long) : Boolean = 39 | (capability, persistentCapability) match { 40 | case (Some(nc), Some(npc)) => ((nc & c.longValue) == c.longValue) && ((npc & pc.longValue) == pc.longValue) 41 | case (Some(nc), None) => (nc & c.longValue()) == c.longValue() 42 | case (nc, Some(pc)) => (pc & pc.longValue()) == pc.longValue() 43 | case (None, None) => c.longValue == 0L && pc.longValue() == 0L 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /java-cluster/src/main/scala/com/linkedin/norbert/javacompat/cluster/ZooKeeperClusterClient.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat 17 | package cluster 18 | 19 | class ZooKeeperClusterClient(clientName: String, serviceName: String, zooKeeperConnectString: String, zooKeeperSessionTimeoutMillis: Int) extends BaseClusterClient { 20 | def this(serviceName: String, zooKeeperConnectString: String, zooKeeperSessionTimeoutMillis: Int) = 21 | this(null, serviceName, zooKeeperConnectString, zooKeeperSessionTimeoutMillis) 22 | 23 | val underlying = com.linkedin.norbert.cluster.ClusterClient(clientName, serviceName, zooKeeperConnectString, zooKeeperSessionTimeoutMillis) 24 | underlying.start 25 | } 26 | -------------------------------------------------------------------------------- /java-cluster/src/main/scala/com/linkedin/norbert/javacompat/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | 18 | import javacompat.cluster.{JavaNode, Node => JNode} 19 | import com.linkedin.norbert.cluster.{Node => SNode} 20 | 21 | package object javacompat { 22 | implicit def scalaSetToJavaSet[T](set: Set[T]): java.util.Set[T] = { 23 | val s = new java.util.HashSet[T] 24 | set.foreach { elem => s.add(elem) } 25 | s 26 | } 27 | 28 | implicit def javaSetToImmutableSet[T](nodes: java.util.Set[T]): Set[T] = { 29 | collection.JavaConversions.asScalaSet(nodes).foldLeft(Set[T]()) { (set, n) => set + n } 30 | } 31 | 32 | implicit def javaIntegerSetToScalaIntSet(set: java.util.Set[java.lang.Integer]): Set[Int] = { 33 | collection.JavaConversions.asScalaSet(set).foldLeft(collection.immutable.Set.empty[Int]) { _ + _.intValue } 34 | } 35 | 36 | implicit def scalaIntSetToJavaIntegerSet(set: Set[Int]): java.util.Set[java.lang.Integer] = { 37 | val result = new java.util.HashSet[java.lang.Integer](set.size) 38 | set.foreach (result add _) 39 | result 40 | } 41 | 42 | implicit def scalaNodeToJavaNode(node: SNode): JNode = { 43 | if (node == null) null else JavaNode(node) 44 | } 45 | 46 | implicit def javaNodeToScalaNode(node: JNode): SNode = { 47 | if (node == null) null 48 | else { 49 | val iter = node.getPartitionIds.iterator 50 | var partitionIds = Set.empty[Int] 51 | while(iter.hasNext) { 52 | partitionIds += iter.next.intValue 53 | } 54 | 55 | SNode(node.getId, node.getUrl, node.isAvailable, partitionIds) 56 | } 57 | } 58 | 59 | implicit def convertSNodeSet(set: Set[SNode]): java.util.Set[JNode] = { 60 | var result = new java.util.HashSet[JNode](set.size) 61 | set.foreach(elem => result.add(scalaNodeToJavaNode(elem))) 62 | result 63 | } 64 | 65 | implicit def convertJNodeSet(set: java.util.Set[JNode]): Set[SNode] = { 66 | val iter = set.iterator 67 | var result = Set.empty[SNode] 68 | while(iter.hasNext) 69 | result += javaNodeToScalaNode(iter.next) 70 | result 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /java-network/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'scala' 3 | 4 | dependencies { 5 | compile project(':network') 6 | compile project(':java-cluster') 7 | compile externalDependency.scalaLibrary 8 | 9 | scalaTools externalDependency.scalaCompiler 10 | scalaTools externalDependency.scalaLibrary 11 | } 12 | 13 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/BaseNetworkClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | import java.util.concurrent.Future; 19 | 20 | import com.linkedin.norbert.cluster.ClusterDisconnectedException; 21 | import com.linkedin.norbert.cluster.InvalidNodeException; 22 | import com.linkedin.norbert.javacompat.cluster.Node; 23 | import com.linkedin.norbert.network.ResponseIterator; 24 | import com.linkedin.norbert.network.Serializer; 25 | 26 | public interface BaseNetworkClient { 27 | /** 28 | * Registers a request/response message pair with the NetworkClient. Requests and their associated 29 | * responses must be registered or an InvalidMessageException will be thrown when an attempt to send 30 | * a Message is made. 31 | * 32 | * @param requestMessage an instance of an outgoing request message 33 | * @param responseMessage an instance of the expected response message or null if this is a one way message 34 | */ 35 | // void registerRequest(RequestMsg requestMessage, Message responseMessage); 36 | 37 | /** 38 | * Sends a message to the specified node in the cluster. 39 | * 40 | * @param request the message to send 41 | * @param node the node to send the message to 42 | * @param serializer the serializer needed to encode and decode the request and response pairs to byte arrays 43 | * 44 | * @return a future which will become available when a response to the message is received 45 | * @throws InvalidNodeException thrown if the node specified is not currently available 46 | * @throws ClusterDisconnectedException thrown if the cluster is not connected when the method is called 47 | */ 48 | Future sendRequestToNode(RequestMsg request, Node node, Serializer serializer) throws InvalidNodeException, ClusterDisconnectedException; 49 | 50 | /** 51 | * Broadcasts a message to all the currently available nodes in the cluster. 52 | * 53 | * @param request the message to send 54 | * @param serializer the serializer needed to encode and decode the request and response pairs to byte arrays 55 | * 56 | * @return a ResponseIterator which will provide the responses from the nodes in the cluster 57 | * as they are received 58 | * @throws ClusterDisconnectedException thrown if the cluster is not connected when the method is called 59 | */ 60 | ResponseIterator broadcastMessage(RequestMsg request, Serializer serializer) throws ClusterDisconnectedException; 61 | 62 | /** 63 | * Shuts down the NetworkClient and releases resources held. 64 | */ 65 | void shutdown(); 66 | } 67 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/ConsistentHashPartitionedLoadBalancerFactory.java: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.javacompat.network; 2 | 3 | import com.linkedin.norbert.cluster.InvalidClusterException; 4 | 5 | import java.util.HashSet; 6 | import java.util.Set; 7 | 8 | public class ConsistentHashPartitionedLoadBalancerFactory implements PartitionedLoadBalancerFactory 9 | { 10 | private final int _bucketCount; 11 | private final HashFunction _hashFunction; 12 | private final PartitionedLoadBalancerFactory _fallThrough; 13 | 14 | public ConsistentHashPartitionedLoadBalancerFactory(int bucketCount) 15 | { 16 | this(bucketCount, new HashFunction.MD5HashFunction(), null); 17 | } 18 | 19 | public ConsistentHashPartitionedLoadBalancerFactory(int bucketCount, 20 | PartitionedLoadBalancerFactory fallThrough) 21 | { 22 | this(bucketCount, new HashFunction.MD5HashFunction(), fallThrough); 23 | } 24 | 25 | public ConsistentHashPartitionedLoadBalancerFactory(int bucketCount, HashFunction hashFunction, 26 | PartitionedLoadBalancerFactory fallThrough) 27 | { 28 | _bucketCount = bucketCount; 29 | _hashFunction = hashFunction; 30 | _fallThrough = fallThrough; 31 | } 32 | 33 | @Override 34 | public PartitionedLoadBalancer newLoadBalancer(Set endpoints) 35 | throws InvalidClusterException 36 | { 37 | // PartitionedLoadBalancer inner = _fallThrough == null ? null : _fallThrough.newLoadBalancer(endpoints); 38 | return ConsistentHashPartitionedLoadBalancer.build(_bucketCount, _hashFunction, endpoints, null); 39 | } 40 | 41 | @Override 42 | public Integer getNumPartitions(Set endpoints) 43 | { 44 | Set set = new HashSet(); 45 | for (Endpoint endpoint : endpoints) 46 | { 47 | set.addAll(endpoint.getNode().getPartitionIds()); 48 | } 49 | 50 | return set.size(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/Endpoint.java: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.javacompat.network; 2 | 3 | import com.linkedin.norbert.javacompat.cluster.Node; 4 | 5 | /** 6 | * Copyright 2009-2010 LinkedIn, Inc 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | */ 20 | public interface Endpoint { 21 | Node getNode(); 22 | boolean canServeRequests(); 23 | } 24 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/HashFunction.java: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.javacompat.network; 2 | 3 | import java.nio.charset.Charset; 4 | import java.security.MessageDigest; 5 | /** 6 | * Created by IntelliJ IDEA. 7 | * User: jwang 8 | */ 9 | public interface HashFunction{ 10 | public long hash(V key); 11 | 12 | public static class MD5HashFunction implements HashFunction{ 13 | 14 | private static Charset Utf8 = Charset.forName("UTF-8"); 15 | 16 | @Override 17 | public long hash(String key){ 18 | MessageDigest md; 19 | try{ 20 | md = MessageDigest.getInstance("MD5"); 21 | } 22 | catch(Exception e){ 23 | throw new RuntimeException(e.getMessage(),e); 24 | } 25 | 26 | byte[] kbytes = md.digest(key.getBytes(Utf8)); 27 | long hc = ((long)(kbytes[3]&0xFF) << 24) 28 | | ((long)(kbytes[2]&0xFF) << 16) 29 | | ((long)(kbytes[1]&0xFF) << 8) 30 | | (long)(kbytes[0]&0xFF); 31 | return Math.abs(hc); 32 | } 33 | } 34 | 35 | public static class NativeObjectHashFunction implements HashFunction{ 36 | @Override 37 | public long hash(V key){ 38 | return key.hashCode(); 39 | } 40 | } 41 | 42 | public static class FNVStringHashingStrategy implements HashFunction{ 43 | public static final long FNV1_64_INIT = 0xcbf29ce484222325L; // 14695981039346656037L 44 | private static final long FNV_64_PRIME = 0x100000001b3L; // 1099511628211L 45 | @Override 46 | public long hash(String key){ 47 | long hval = FNV1_64_INIT; 48 | int len = key.length(); 49 | for (int i=0; iLoadBalancer handles calculating the next Node a request should be routed to. 22 | */ 23 | public interface LoadBalancer { 24 | /** 25 | * Returns the next Node a request should be routed to. 26 | * 27 | * @return the Node to route the next request to or null if there are no Nodes available 28 | */ 29 | Node nextNode(); 30 | 31 | /** 32 | * Returns the next Node that fulfill the capability a request should be routed to. 33 | * 34 | * @param capability A Long that representing the minimal capability of the node that's serving the request 35 | * @return the Node to route the next request to or null if there are no Node's available 36 | */ 37 | Node nextNode(Long capability); 38 | 39 | /** 40 | * Returns the next Node that fulfill the capability a request should be routed to. 41 | * 42 | * @param capability A Long that representing the minimal capability of the node that's serving the request 43 | * @param persistentCapability A Long that represents the minimal persistent capability of the node 44 | * @return the Node to route the next request to or null if there are no Node's available 45 | */ 46 | Node nextNode(Long capability, Long persistentCapability); 47 | } 48 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/LoadBalancerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | import java.util.Set; 19 | 20 | import com.linkedin.norbert.cluster.InvalidClusterException; 21 | import com.linkedin.norbert.javacompat.cluster.Node; 22 | 23 | /** 24 | * A factory which can generate LoadBalancers. 25 | */ 26 | public interface LoadBalancerFactory { 27 | /** 28 | * Create a new load balancer instance based on the currently available Nodes. 29 | * 30 | * @param endpoints the currently available Nodes in the cluster 31 | * 32 | * @return a new LoadBalancer instance 33 | * @throws InvalidClusterException thrown to indicate that the current cluster topology is invalid in some way and 34 | * it is impossible to create a LoadBalancer 35 | */ 36 | LoadBalancer newLoadBalancer(Set endpoints) throws InvalidClusterException; 37 | } 38 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/NetworkClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | import java.util.concurrent.Future; 19 | 20 | import com.linkedin.norbert.cluster.ClusterDisconnectedException; 21 | import com.linkedin.norbert.cluster.InvalidClusterException; 22 | import com.linkedin.norbert.network.NoNodesAvailableException; 23 | import com.linkedin.norbert.network.Serializer; 24 | 25 | public interface NetworkClient extends BaseNetworkClient { 26 | /** 27 | * Sends a request to a node in the cluster. The NetworkClient defers to the current 28 | * LoadBalancer to decide which Node the request should be sent to. 29 | * 30 | * @param request the request to send 31 | * @param serializer the serializer needed to encode and decode the request and response pairs to byte arrays 32 | * 33 | * @return a future which will become available when a response to the request is received 34 | * @throws InvalidClusterException thrown if the cluster is currently in an invalid state 35 | * @throws NoNodesAvailableException thrown if the LoadBalancer was unable to provide a Node 36 | * to send the request to 37 | * @throws ClusterDisconnectedException thrown if the cluster is not connected when the method is called 38 | */ 39 | Future sendRequest(RequestMsg request, Serializer serializer) throws InvalidClusterException, NoNodesAvailableException, ClusterDisconnectedException; 40 | } 41 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/NetworkServerConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | import com.linkedin.norbert.cluster.ClusterDefaults; 19 | import com.linkedin.norbert.javacompat.cluster.ClusterClient; 20 | import com.linkedin.norbert.network.NetworkDefaults; 21 | 22 | public class NetworkServerConfig { 23 | private ClusterClient clusterClient; 24 | private String serviceName; 25 | private String zooKeeperConnectString; 26 | private int zooKeeperSessionTimeoutMillis = ClusterDefaults.ZOOKEEPER_SESSION_TIMEOUT_MILLIS(); 27 | private int requestThreadCorePoolSize = NetworkDefaults.REQUEST_THREAD_CORE_POOL_SIZE(); 28 | private int requestThreadMaxPoolSize = NetworkDefaults.REQUEST_THREAD_MAX_POOL_SIZE(); 29 | private int requestThreadKeepAliveTimeSecs = NetworkDefaults.REQUEST_THREAD_KEEP_ALIVE_TIME_SECS(); 30 | 31 | public ClusterClient getClusterClient() { 32 | return clusterClient; 33 | } 34 | 35 | public void setClusterClient(ClusterClient clusterClient) { 36 | this.clusterClient = clusterClient; 37 | } 38 | 39 | public String getServiceName() { 40 | return serviceName; 41 | } 42 | 43 | public void setServiceName(String serviceName) { 44 | this.serviceName = serviceName; 45 | } 46 | 47 | public String getZooKeeperConnectString() { 48 | return zooKeeperConnectString; 49 | } 50 | 51 | public void setZooKeeperConnectString(String zooKeeperConnectString) { 52 | this.zooKeeperConnectString = zooKeeperConnectString; 53 | } 54 | 55 | public int getZooKeeperSessionTimeoutMillis() { 56 | return zooKeeperSessionTimeoutMillis; 57 | } 58 | 59 | public void setZooKeeperSessionTimeoutMillis(int zooKeeperSessionTimeoutMillis) { 60 | this.zooKeeperSessionTimeoutMillis = zooKeeperSessionTimeoutMillis; 61 | } 62 | 63 | public int getRequestThreadCorePoolSize() { 64 | return requestThreadCorePoolSize; 65 | } 66 | 67 | public void setRequestThreadCorePoolSize(int requestThreadCorePoolSize) { 68 | this.requestThreadCorePoolSize = requestThreadCorePoolSize; 69 | } 70 | 71 | public int getRequestThreadMaxPoolSize() { 72 | return requestThreadMaxPoolSize; 73 | } 74 | 75 | public void setRequestThreadMaxPoolSize(int requestThreadMaxPoolSize) { 76 | this.requestThreadMaxPoolSize = requestThreadMaxPoolSize; 77 | } 78 | 79 | public int getRequestThreadKeepAliveTimeSecs() { 80 | return requestThreadKeepAliveTimeSecs; 81 | } 82 | 83 | public void setRequestThreadKeepAliveTimeSecs(int requestThreadKeepAliveTimeSecs) { 84 | this.requestThreadKeepAliveTimeSecs = requestThreadKeepAliveTimeSecs; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/PartitionedLoadBalancerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | import java.util.Set; 19 | 20 | import com.linkedin.norbert.cluster.InvalidClusterException; 21 | 22 | /** 23 | * A factory which can generate PartitionedLoadBalancers. 24 | */ 25 | public interface PartitionedLoadBalancerFactory { 26 | /** 27 | * Create a new load balancer instance based on the currently available Nodes. 28 | * 29 | * @param endpoints the currently available Nodes in the cluster 30 | * 31 | * @return a new PartitionedLoadBalancer instance 32 | * @throws InvalidClusterException thrown to indicate that the current cluster topology is invalid in some way and 33 | * it is impossible to create a LoadBalancer 34 | */ 35 | PartitionedLoadBalancer newLoadBalancer(Set endpoints) throws InvalidClusterException; 36 | 37 | /** 38 | * Returns the number of partitions for a given set of endpoints. Can either use a statically configured set of 39 | * endpoints or take the maximum from zookeeper 40 | * @param endpoints 41 | * @return 42 | */ 43 | Integer getNumPartitions(Set endpoints); 44 | } 45 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/RequestBuilder.java: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.javacompat.network; 2 | 3 | import com.linkedin.norbert.javacompat.cluster.Node; 4 | import java.util.Set; 5 | 6 | public interface RequestBuilder { 7 | RequestMsg apply(Node n, Set ids); 8 | } 9 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/RequestHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | public interface RequestHandler { 19 | ResponseMsg handleRequest(RequestMsg request) throws Exception; 20 | } 21 | -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/RingHashPartitionedLoadBalancerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | import java.util.HashSet; 19 | import java.util.Set; 20 | import org.apache.log4j.Logger; 21 | import com.linkedin.norbert.cluster.InvalidClusterException; 22 | 23 | /** 24 | * Consistent hash load balancer factory. 25 | * 26 | * Consistent hashing is implemented based on http://www.lexemetech.com/2007/11/consistent-hashing.html 27 | * 28 | * @author "Rui Wang" 29 | * 30 | */ 31 | public class RingHashPartitionedLoadBalancerFactory implements PartitionedLoadBalancerFactory 32 | { 33 | public static final Logger log = Logger.getLogger(RingHashPartitionedLoadBalancerFactory.class); 34 | 35 | // number of Replicas is used to help to produce balanced buckets. 36 | private final int _numberOfReplicas; 37 | private final HashFunction _hashingStrategy; 38 | 39 | public RingHashPartitionedLoadBalancerFactory(int numberOfReplicas, HashFunction hashingStrategy) 40 | { 41 | _numberOfReplicas = numberOfReplicas; 42 | _hashingStrategy = hashingStrategy; 43 | } 44 | 45 | public RingHashPartitionedLoadBalancerFactory(int numberOfReplicas){ 46 | this(numberOfReplicas, new HashFunction.MD5HashFunction()); 47 | } 48 | public PartitionedLoadBalancer newLoadBalancer(Set endpoints) throws InvalidClusterException 49 | { 50 | return new RingHashPartitionedLoadBalancer(_numberOfReplicas, endpoints, _hashingStrategy); 51 | } 52 | 53 | @Override 54 | public Integer getNumPartitions(Set endpoints) { 55 | Set partitionIds = new HashSet(); 56 | for(Endpoint endpoint : endpoints) { 57 | partitionIds.addAll(endpoint.getNode().getPartitionIds()); 58 | } 59 | return partitionIds.size(); 60 | } 61 | 62 | private final static double mean(int[] population) 63 | { 64 | double sum =0; 65 | for ( int x: population) 66 | { 67 | sum += x; 68 | } 69 | return sum / population.length; 70 | } 71 | 72 | private final static double variance(int[] population) 73 | { 74 | long n = 0; 75 | double mean = 0; 76 | double s = 0.0; 77 | 78 | for (int x : population) 79 | { 80 | n++; 81 | double delta = x - mean; 82 | mean += delta / n; 83 | s += delta * (x - mean); 84 | } 85 | return (s/n); 86 | } 87 | 88 | private final static double standard_deviation(int[] population) 89 | { 90 | return Math.sqrt(variance(population)); 91 | } 92 | } -------------------------------------------------------------------------------- /java-network/src/main/java/com/linkedin/norbert/javacompat/network/ScatterGatherHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat.network; 17 | 18 | import java.util.Set; 19 | 20 | import com.linkedin.norbert.javacompat.cluster.Node; 21 | import com.linkedin.norbert.network.ResponseIterator; 22 | 23 | /** 24 | * A ScatterGatherHandler is used to customize an outgoing Message and to 25 | * aggregate the incoming responses. 26 | */ 27 | public interface ScatterGatherHandler { 28 | /** 29 | * This method is called after all messages are sent and allows the user to aggregate the responses. 30 | * 31 | * @param responseIterator the ResponseIterator to retrieve responses 32 | * 33 | * @return A user defined value. This value is passed on to the client. 34 | * @throws Exception any exception thrown will be passed on to the client 35 | */ 36 | T gatherResponses(ResponseIterator responseIterator) throws Exception; 37 | } 38 | -------------------------------------------------------------------------------- /java-network/src/main/scala/com/linkedin/norbert/javacompat/network/DefaultPartitionedLoadBalancerFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | 17 | package com.linkedin.norbert 18 | package javacompat 19 | package network 20 | 21 | import com.linkedin.norbert.network.partitioned.loadbalancer.{DefaultPartitionedLoadBalancerFactory => SDefaultPartitionedLoadBalancerFactory} 22 | import EndpointConversions._ 23 | import cluster.Node 24 | import com.linkedin.norbert.network.common.{Endpoint => SEndpoint} 25 | import java.util.{Set => JSet} 26 | 27 | /** 28 | * An adapter for the default, round-robining load balancer 29 | * @param numPartitions 30 | * @param serveRequestsIfPartitionMissing 31 | * @tparam PartitionedId 32 | */ 33 | abstract class DefaultPartitionedLoadBalancerFactory[PartitionedId] 34 | (numPartitions: Int, serveRequestsIfPartitionMissing: Boolean = true) extends PartitionedLoadBalancerFactory[PartitionedId] { 35 | def this(numPartitions: Int) = this(numPartitions, true) 36 | 37 | val underlying = new SDefaultPartitionedLoadBalancerFactory[PartitionedId](numPartitions, serveRequestsIfPartitionMissing) { 38 | protected def calculateHash(id: PartitionedId) = hashPartitionedId(id) 39 | 40 | def getNumPartitions(endpoints: Set[SEndpoint]) = { 41 | if (numPartitions == -1) { 42 | endpoints.flatMap(_.getNode.getPartitionIds).size 43 | } else { 44 | numPartitions 45 | } 46 | } 47 | } 48 | 49 | val adapter = new ScalaLbfToJavaLbf[PartitionedId](underlying) 50 | 51 | def newLoadBalancer(endpoints: JSet[Endpoint]): PartitionedLoadBalancer[PartitionedId] = 52 | adapter.newLoadBalancer(endpoints) 53 | 54 | protected def hashPartitionedId(id : PartitionedId) : Int 55 | } -------------------------------------------------------------------------------- /java-network/src/main/scala/com/linkedin/norbert/javacompat/network/IntegerConsistentHashPartitionedLoadBalancerFactory.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.javacompat.network 2 | 3 | import java.util.Set 4 | import com.linkedin.norbert.EndpointConversions 5 | import EndpointConversions._ 6 | 7 | class IntegerConsistentHashPartitionedLoadBalancerFactory(numPartitions: Int, serveRequestsIfPartitionMissing: Boolean) 8 | extends DefaultPartitionedLoadBalancerFactory[Int](numPartitions, serveRequestsIfPartitionMissing) { 9 | def this(numPartitions: Int) = this(numPartitions, true) 10 | def this() = this(-1, true) 11 | 12 | protected def hashPartitionedId(id: Int) = id.hashCode 13 | 14 | def getNumPartitions(endpoints: Set[Endpoint]) = { 15 | if (numPartitions == -1) { 16 | val endpointSet = convertJavaEndpointSet(endpoints) 17 | endpointSet.flatMap(_.node.partitionIds).size 18 | } else { 19 | numPartitions 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /java-network/src/main/scala/com/linkedin/norbert/javacompat/network/JavaEndpoint.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.javacompat.network 2 | 3 | import com.linkedin.norbert.network.common.{Endpoint => SEndpoint} 4 | import com.linkedin.norbert.javacompat.javaNodeToScalaNode 5 | import com.linkedin.norbert.javacompat.cluster.JavaNode 6 | 7 | object JavaEndpoint { 8 | def apply(endpoint: com.linkedin.norbert.network.common.Endpoint): JavaEndpoint = { 9 | if (endpoint == null) { 10 | null 11 | } else { 12 | new JavaEndpoint(endpoint) 13 | } 14 | } 15 | } 16 | 17 | class JavaEndpoint(endpoint: SEndpoint) extends Endpoint { 18 | def getNode = JavaNode(endpoint.node) 19 | def canServeRequests = endpoint.canServeRequests 20 | 21 | override def hashCode() = endpoint.hashCode() 22 | 23 | override def equals(that: Any) = endpoint.equals(that) 24 | } 25 | -------------------------------------------------------------------------------- /java-network/src/main/scala/com/linkedin/norbert/javacompat/network/JavaLbfToScalaLbf.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert 2 | package javacompat 3 | package network 4 | 5 | import com.linkedin.norbert.network.partitioned.loadbalancer.{PartitionedLoadBalancerFactory => SPartitionedLoadBalancerFactory, PartitionedLoadBalancer => SPartitionedLoadBalancer} 6 | import com.linkedin.norbert.network.client.loadbalancer.{LoadBalancerFactory => SLoadBalancerFactory, LoadBalancer => SLoadBalancer} 7 | 8 | import com.linkedin.norbert.cluster.{Node => SNode} 9 | import com.linkedin.norbert.network.common.{Endpoint => SEndpoint} 10 | 11 | import com.linkedin.norbert.EndpointConversions._ 12 | 13 | class JavaLbfToScalaLbf[PartitionedId](javaLbf: PartitionedLoadBalancerFactory[PartitionedId]) extends SPartitionedLoadBalancerFactory[PartitionedId] { 14 | def newLoadBalancer(nodes: Set[SEndpoint]) = { 15 | val lb = javaLbf.newLoadBalancer(nodes) 16 | new SPartitionedLoadBalancer[PartitionedId] { 17 | def nextNode(id: PartitionedId, capability: Option[Long] = None, persistentCapability: Option[Long] = None) = { 18 | (capability, persistentCapability) match { 19 | case (Some(c),Some(pc)) => Option(lb.nextNode(id, c.longValue, pc.longValue)) 20 | case (None, Some(pc)) => Option(lb.nextNode(id, 0L, pc.longValue)) 21 | case (Some(c), None) => Option(lb.nextNode(id, c.longValue, 0L)) 22 | case (None, None) => Option(lb.nextNode(id)) 23 | } 24 | } 25 | 26 | def nodesForOneReplica(id: PartitionedId, capability: Option[Long] = None, persistentCapability: Option[Long] = None) = { 27 | val jMap = (capability, persistentCapability) match { 28 | case (Some(c),Some(pc)) => lb.nodesForOneReplica(id, c.longValue, pc.longValue) 29 | case (Some(c), None) => lb.nodesForOneReplica(id, c.longValue, 0L) 30 | case (None, Some(pc)) => lb.nodesForOneReplica(id, 0L, pc.longValue) 31 | case (None, None) => lb.nodesForOneReplica(id) 32 | } 33 | var sMap = Map.empty[com.linkedin.norbert.cluster.Node, Set[Int]] 34 | 35 | val entries = jMap.entrySet.iterator 36 | while(entries.hasNext) { 37 | val entry = entries.next 38 | val node = javaNodeToScalaNode(entry.getKey) 39 | val set = entry.getValue.foldLeft(Set.empty[Int]) { (s, elem) => s + elem.intValue} 40 | 41 | sMap += (node -> set) 42 | } 43 | sMap 44 | } 45 | 46 | def nodesForPartitionedId(id: PartitionedId, capability: Option[Long] = None, persistentCapability: Option[Long] = None) = { 47 | val jSet = (capability, persistentCapability) match { 48 | case (Some(c), Some(pc)) => lb.nodesForPartitionedId(id, c.longValue, pc.longValue) 49 | case (Some(c), None) => lb.nodesForPartitionedId(id, c.longValue, 0L) 50 | case (None, Some(pc)) => lb.nodesForPartitionedId(id, 0L, pc.longValue) 51 | case (None, None) => lb.nodesForPartitionedId(id, 0L, 0L) 52 | } 53 | var sSet = Set.empty[SNode] 54 | val entries = jSet.iterator 55 | while(entries.hasNext) { 56 | val node = javaNodeToScalaNode(entries.next) 57 | sSet += node 58 | } 59 | sSet 60 | } 61 | 62 | def nodesForPartitions(id: PartitionedId, partitions: Set[Int], capability: Option[Long] = None, persistentCapability: Option[Long] = None) = { 63 | val jMap = (capability, persistentCapability) match { 64 | case (Some(c), Some(pc)) => lb.nodesForOneReplica(id, c.longValue, pc.longValue) 65 | case (Some(c), None) => lb.nodesForOneReplica(id, c.longValue, 0L) 66 | case (None, Some(pc)) => lb.nodesForOneReplica(id, 0L, pc.longValue) 67 | case (None, None) => lb.nodesForOneReplica(id) 68 | } 69 | var sMap = Map.empty[com.linkedin.norbert.cluster.Node, Set[Int]] 70 | 71 | val entries = jMap.entrySet.iterator 72 | while(entries.hasNext) { 73 | val entry = entries.next 74 | val node = javaNodeToScalaNode(entry.getKey) 75 | val set = entry.getValue.foldLeft(Set.empty[Int]) { (s, elem) => s + elem.intValue} 76 | 77 | sMap += (node -> set) 78 | } 79 | sMap 80 | } 81 | } 82 | 83 | } 84 | 85 | def getNumPartitions(endpoints: Set[SEndpoint]) = javaLbf.getNumPartitions(endpoints).intValue() 86 | } 87 | -------------------------------------------------------------------------------- /java-network/src/main/scala/com/linkedin/norbert/javacompat/network/MultiRingConsistentHashPartitionedLoadBalancerFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | 17 | package com.linkedin.norbert 18 | package javacompat 19 | package network 20 | 21 | import com.linkedin.norbert.network.partitioned.loadbalancer.{PartitionedConsistentHashedLoadBalancerFactory => SPartitionedConsistentHashedLoadBalancerFactory} 22 | import EndpointConversions._ 23 | import cluster.Node 24 | import java.util.{Iterator, Set} 25 | 26 | /** 27 | * An adapter for a partitioned load balancer providing a consistent hash ring per partition. 28 | */ 29 | class MultiRingConsistentHashPartitionedLoadBalancerFactory[PartitionedId](numPartitions: Int, 30 | slicesPerEndpoint: Int, 31 | hashFunction: HashFunction[PartitionedId], 32 | endpointHashFunction: HashFunction[String], 33 | serveRequestsIfPartitionUnavailable: Boolean) extends PartitionedLoadBalancerFactory[PartitionedId] { 34 | 35 | def this(slicesPerEndpoint: Int, hashFunction: HashFunction[PartitionedId], endpointHashFunction: HashFunction[String], serveRequestsIfPartitionMissing: Boolean) = { 36 | this(-1, slicesPerEndpoint, hashFunction, endpointHashFunction, serveRequestsIfPartitionMissing) 37 | } 38 | 39 | val underlying = new SPartitionedConsistentHashedLoadBalancerFactory[PartitionedId]( 40 | numPartitions, 41 | slicesPerEndpoint, 42 | (id: PartitionedId) => (hashFunction.hash(id) % Integer.MAX_VALUE).toInt, 43 | (distKey: String) => (endpointHashFunction.hash(distKey) % Integer.MAX_VALUE).toInt, 44 | serveRequestsIfPartitionUnavailable) 45 | 46 | val lbf = new ScalaLbfToJavaLbf[PartitionedId](underlying) 47 | 48 | def newLoadBalancer(endpoints: Set[Endpoint]) = lbf.newLoadBalancer(endpoints) 49 | 50 | def getNumPartitions(endpoints: Set[Endpoint]) = lbf.getNumPartitions(endpoints) 51 | } -------------------------------------------------------------------------------- /java-network/src/main/scala/com/linkedin/norbert/javacompat/network/NettyNetworkServer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.javacompat 17 | package network 18 | 19 | import cluster.BaseClusterClient 20 | import com.linkedin.norbert.cluster.ClusterClient 21 | import com.linkedin.norbert.network.Serializer 22 | 23 | class NettyNetworkServer(config: NetworkServerConfig) extends NetworkServer { 24 | val c = new com.linkedin.norbert.network.netty.NetworkServerConfig 25 | 26 | c.clusterClient = if (config.getClusterClient != null) 27 | config.getClusterClient.asInstanceOf[BaseClusterClient].underlying 28 | else ClusterClient(null, config.getServiceName, config.getZooKeeperConnectString, config.getZooKeeperSessionTimeoutMillis) 29 | 30 | c.zooKeeperSessionTimeoutMillis = config.getZooKeeperSessionTimeoutMillis 31 | c.requestThreadCorePoolSize = config.getRequestThreadCorePoolSize 32 | c.requestThreadMaxPoolSize = config.getRequestThreadMaxPoolSize 33 | c.requestThreadKeepAliveTimeSecs = config.getRequestThreadKeepAliveTimeSecs 34 | 35 | val underlying = com.linkedin.norbert.network.server.NetworkServer(c) 36 | 37 | def shutdown = underlying.shutdown 38 | 39 | def markUnavailable = underlying.markUnavailable 40 | 41 | def markAvailable(initialCapability: Long) = underlying.markAvailable(initialCapability) 42 | 43 | def markAvailable = underlying.markAvailable 44 | 45 | def getMyNode = underlying.myNode 46 | 47 | def bind(nodeId: Int, markAvailable: Boolean, initialCapacity: Long) = underlying.bind(nodeId, markAvailable, initialCapacity) 48 | 49 | def bind(nodeId: Int, markAvailable: Boolean) = underlying.bind(nodeId, markAvailable) 50 | 51 | def bind(nodeId: Int) = underlying.bind(nodeId) 52 | 53 | def registerHandler[RequestMsg, ResponseMsg](handler: RequestHandler[RequestMsg, ResponseMsg], serializer: Serializer[RequestMsg, ResponseMsg]) = { 54 | underlying.registerHandler((request: RequestMsg) => handler.handleRequest(request))(serializer, serializer) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /java-network/src/main/scala/com/linkedin/norbert/javacompat/network/PartitionedNetworkClientFactory.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.javacompat.network 2 | 3 | /** 4 | * 5 | * @author Dmytro Ivchenko 6 | */ 7 | class PartitionedNetworkClientFactory[PartitionedId](serviceName: String, zooKeeperConnectString: String, 8 | zooKeeperSessionTimeoutMillis: Int, 9 | closeChannelTimeMillis: Long, norbertOutlierMultiplier: Double, norbertOutlierConstant: Double, 10 | partitionedLoadBalancerFactory: PartitionedLoadBalancerFactory[PartitionedId]) 11 | { 12 | val config = new NetworkClientConfig 13 | 14 | config.setServiceName(serviceName); 15 | config.setZooKeeperConnectString(zooKeeperConnectString); 16 | config.setZooKeeperSessionTimeoutMillis(zooKeeperSessionTimeoutMillis); 17 | config.setCloseChannelTimeMillis(closeChannelTimeMillis); 18 | config.setOutlierMuliplier(norbertOutlierMultiplier); 19 | config.setOutlierConstant(norbertOutlierConstant); 20 | 21 | def createPartitionedNetworkClient(): PartitionedNetworkClient[PartitionedId] = 22 | { 23 | new NettyPartitionedNetworkClient[PartitionedId](config, partitionedLoadBalancerFactory); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /java-network/src/main/scala/com/linkedin/norbert/javacompat/network/RoundRobinLoadBalancerFactory.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.javacompat.network 2 | 3 | 4 | import java.util 5 | import com.linkedin.norbert.EndpointConversions 6 | import com.linkedin.norbert.javacompat.cluster.{JavaNode, Node} 7 | 8 | 9 | /** 10 | * 11 | * @author Dmytro Ivchenko 12 | */ 13 | class RoundRobinLoadBalancerFactory extends LoadBalancerFactory 14 | { 15 | val scalaLbf = new com.linkedin.norbert.network.client.loadbalancer.RoundRobinLoadBalancerFactory 16 | 17 | def newLoadBalancer(endpoints: util.Set[Endpoint]): LoadBalancer = { 18 | val loadBalancer = scalaLbf.newLoadBalancer(EndpointConversions.convertJavaEndpointSet(endpoints)) 19 | 20 | new LoadBalancer { 21 | def nextNode: Node = { 22 | nextNode(0L, 0L) 23 | } 24 | 25 | def nextNode(capability: java.lang.Long): Node = { 26 | nextNode(capability, 0L) 27 | } 28 | 29 | def nextNode(capability: java.lang.Long, persistentCapability: java.lang.Long): Node = { 30 | val node = loadBalancer.nextNode(new Some(capability.longValue()), new Some(persistentCapability.longValue())) 31 | if (node.isDefined) 32 | JavaNode.apply(node.get) 33 | else 34 | null 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /java-network/src/main/scala/com/linkedin/norbert/javacompat/network/ScalaLbfToJavaLbf.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert 2 | package javacompat 3 | package network 4 | 5 | import com.linkedin.norbert.network.partitioned.loadbalancer.{PartitionedLoadBalancerFactory => SPartitionedLoadBalancerFactory} 6 | import com.linkedin.norbert.EndpointConversions._ 7 | import javacompat.cluster.Node 8 | import javacompat._ 9 | 10 | 11 | class ScalaLbfToJavaLbf[PartitionedId](scalaLbf: SPartitionedLoadBalancerFactory[PartitionedId]) extends PartitionedLoadBalancerFactory[PartitionedId] { 12 | 13 | def newLoadBalancer(endpoints: java.util.Set[Endpoint]) = { 14 | val scalaBalancer = scalaLbf.newLoadBalancer(endpoints) 15 | 16 | new PartitionedLoadBalancer[PartitionedId] { 17 | def nodesForOneReplica(id: PartitionedId) = nodesForOneReplica(id, 0L, 0L) 18 | 19 | def nodesForOneReplica(id: PartitionedId, capability: java.lang.Long) = { 20 | nodesForOneReplica(id, capability, 0L) 21 | } 22 | 23 | def nodesForOneReplica(id: PartitionedId, capability: java.lang.Long, persistentCapability: java.lang.Long) = { 24 | val replica = scalaBalancer.nodesForOneReplica(id, capability, persistentCapability) 25 | val result = new java.util.HashMap[Node, java.util.Set[java.lang.Integer]](replica.size) 26 | 27 | replica.foreach { case (node, partitions) => 28 | result.put(node, partitions) 29 | } 30 | 31 | result 32 | } 33 | 34 | def nextNode(id: PartitionedId) = nextNode(id, 0L, 0L) 35 | 36 | def nextNode(id: PartitionedId, capability: java.lang.Long) = nextNode(id, capability, 0L) 37 | 38 | def nextNode(id: PartitionedId, capability: java.lang.Long, persistentCapability: java.lang.Long) = { 39 | scalaBalancer.nextNode(id, capability, persistentCapability) match { 40 | case Some(n) =>n 41 | case None => null 42 | } 43 | } 44 | 45 | def nodesForPartitionedId(id: PartitionedId) = nodesForPartitionedId(id, 0L, 0L) 46 | 47 | def nodesForPartitionedId(id: PartitionedId, capability: java.lang.Long) = nodesForPartitionedId(id, capability, 0L) 48 | 49 | def nodesForPartitionedId(id: PartitionedId, capability: java.lang.Long, persistentCapability: java.lang.Long) = { 50 | val set = scalaBalancer.nodesForPartitionedId(id, capability, persistentCapability) 51 | val jSet = new java.util.HashSet[Node]() 52 | set.foldLeft(jSet) { case (jSet, node) => {jSet.add(node); jSet} } 53 | jSet 54 | } 55 | 56 | def nodesForPartitions(id: PartitionedId, partitions: java.util.Set[java.lang.Integer]) = nodesForPartitions(id, partitions, 0L, 0L) 57 | 58 | def nodesForPartitions(id: PartitionedId, partitions: java.util.Set[java.lang.Integer], capability: java.lang.Long) = nodesForPartitions(id, partitions, capability, 0L) 59 | def nodesForPartitions(id: PartitionedId, partitions:java.util.Set[java.lang.Integer], capability: java.lang.Long, persistentCapability: java.lang.Long) = { 60 | val replica = scalaBalancer.nodesForPartitions(id, partitions, capability, persistentCapability) 61 | val result = new java.util.HashMap[Node, java.util.Set[java.lang.Integer]](replica.size) 62 | 63 | replica.foreach { case (node, partitions) => 64 | result.put(node, partitions) 65 | } 66 | 67 | result 68 | } 69 | 70 | implicit def toOption(capability: java.lang.Long) : Option[Long] = { 71 | if (capability.longValue == 0L) None 72 | else Some(capability.longValue) 73 | } 74 | } 75 | 76 | 77 | } 78 | 79 | def getNumPartitions(endpoints: java.util.Set[Endpoint]) = { 80 | scalaLbf.getNumPartitions(endpoints) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /java-network/src/main/scala/com/linkedin/norbert/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | 18 | import com.linkedin.norbert.network.common.{Endpoint => SEndpoint} 19 | import com.linkedin.norbert.cluster.{Node => SNode} 20 | 21 | import javacompat.javaNodeToScalaNode 22 | import javacompat.network.{JavaEndpoint, Endpoint => JEndpoint} 23 | 24 | object EndpointConversions { 25 | implicit def scalaEndpointToJavaEndpoint(endpoint: SEndpoint): JEndpoint = { 26 | if (endpoint == null) null else JavaEndpoint(endpoint) 27 | } 28 | 29 | implicit def javaEndpointToScalaEndpoint(endpoint: JEndpoint): SEndpoint = { 30 | if(endpoint == null) null 31 | else new SEndpoint { 32 | def node: SNode = { 33 | javaNodeToScalaNode(endpoint.getNode) 34 | } 35 | 36 | def canServeRequests = endpoint.canServeRequests 37 | } 38 | } 39 | 40 | implicit def convertScalaEndpointSet(set: Set[SEndpoint]): java.util.Set[JEndpoint] = { 41 | val result = new java.util.HashSet[JEndpoint](set.size) 42 | set.foreach(elem => result.add(scalaEndpointToJavaEndpoint(elem))) 43 | result 44 | } 45 | 46 | implicit def convertJavaEndpointSet(set: java.util.Set[JEndpoint]): Set[SEndpoint] = { 47 | val iterator = set.iterator 48 | var sset = Set.empty[SEndpoint] 49 | while(iterator.hasNext) { 50 | sset += iterator.next 51 | } 52 | sset 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /java-network/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=FATAL, CONSOLE 2 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 3 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 4 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n 5 | -------------------------------------------------------------------------------- /lib/sbt-launch.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linkedin-sna/norbert/079ebeac8d2302b2fe9fb6e61648d4781317ac70/lib/sbt-launch.jar -------------------------------------------------------------------------------- /network/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'scala' 3 | 4 | dependencies { 5 | compile project(':cluster') 6 | compile externalDependency.scalaLibrary 7 | compile externalDependency.netty 8 | compile externalDependency.slf4jApi 9 | compile externalDependency.slf4jLog4j 10 | 11 | testCompile externalDependency.specs 12 | testCompile externalDependency.mockitoAll 13 | testCompile externalDependency.cglib 14 | testCompile externalDependency.objenesis 15 | 16 | scalaTools externalDependency.scalaCompiler 17 | scalaTools externalDependency.scalaLibrary 18 | } 19 | 20 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/JavaSerializer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.network 17 | 18 | import java.io._ 19 | import com.google.protobuf.Message 20 | 21 | object JavaSerializer { 22 | def apply[RequestMsg, ResponseMsg] 23 | (requestClass: Class[RequestMsg], responseClass: Class[ResponseMsg]): JavaSerializer[RequestMsg, ResponseMsg] = 24 | apply( 25 | requestClass.getName, 26 | responseClass.getName, 27 | requestClass, 28 | responseClass) 29 | 30 | def apply[RequestMsg, ResponseMsg] 31 | (requestName: String, responseName: String, requestClass: Class[RequestMsg], responseClass: Class[ResponseMsg]): JavaSerializer[RequestMsg, ResponseMsg] = 32 | new JavaSerializer( 33 | requestName, 34 | responseName, 35 | requestClass, 36 | responseClass) 37 | 38 | def apply[RequestMsg, ResponseMsg] 39 | (requestName: String, requestClass: Class[RequestMsg], responseClass: Class[ResponseMsg]): JavaSerializer[RequestMsg, ResponseMsg] = 40 | apply( 41 | requestName, 42 | responseClass.getName, 43 | requestClass, 44 | responseClass) 45 | 46 | def apply[RequestMsg, ResponseMsg] 47 | (implicit requestManifest: ClassManifest[RequestMsg], responseManifest: ClassManifest[ResponseMsg]): JavaSerializer[RequestMsg, ResponseMsg] = 48 | apply(requestManifest.erasure.asInstanceOf[Class[RequestMsg]], responseManifest.erasure.asInstanceOf[Class[ResponseMsg]]) 49 | } 50 | 51 | class JavaSerializer[RequestMsg, ResponseMsg](reqName: String, resName: String, 52 | requestClass: Class[RequestMsg], responseClass: Class[ResponseMsg]) 53 | extends Serializer[RequestMsg, ResponseMsg] { 54 | def requestName = reqName 55 | def responseName = resName 56 | 57 | private def toBytes[T](message: T): Array[Byte] = { 58 | val baos = new ByteArrayOutputStream 59 | val oos = new ObjectOutputStream(baos) 60 | oos.writeObject(message) 61 | baos.toByteArray 62 | } 63 | 64 | private def fromBytes[T](bytes: Array[Byte]): T = { 65 | val bais = new ByteArrayInputStream(bytes) 66 | val ois = new ObjectInputStream(bais) 67 | ois.readObject.asInstanceOf[T] 68 | } 69 | 70 | def requestToBytes(message: RequestMsg): Array[Byte] = toBytes(message) 71 | 72 | def requestFromBytes(bytes: Array[Byte]): RequestMsg = fromBytes(bytes) 73 | 74 | def responseToBytes(message: ResponseMsg): Array[Byte] = toBytes(message) 75 | 76 | def responseFromBytes(bytes: Array[Byte]): ResponseMsg = fromBytes(bytes) 77 | } 78 | 79 | object ProtobufSerializer { 80 | def build[RequestMsg <: Message, ResponseMsg <: Message](requestPrototype: RequestMsg, responsePrototype: ResponseMsg): ProtobufSerializer[RequestMsg, ResponseMsg] = 81 | new ProtobufSerializer(requestPrototype, responsePrototype) 82 | } 83 | 84 | class ProtobufSerializer[RequestMsg <: Message, ResponseMsg <: Message](requestPrototype: RequestMsg, responsePrototype: ResponseMsg) extends Serializer[RequestMsg, ResponseMsg] { 85 | def requestName = requestPrototype.getDescriptorForType.getFullName 86 | 87 | def responseName = responsePrototype.getDescriptorForType.getFullName 88 | 89 | def requestToBytes(request: RequestMsg) = request.toByteArray 90 | 91 | def responseToBytes(response: ResponseMsg) = response.toByteArray 92 | 93 | def requestFromBytes(bytes: Array[Byte]) = (requestPrototype.newBuilderForType.mergeFrom(bytes).build).asInstanceOf[RequestMsg] 94 | 95 | def responseFromBytes(bytes: Array[Byte]) = (responsePrototype.newBuilderForType.mergeFrom(bytes).build).asInstanceOf[ResponseMsg] 96 | } -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/NetworkServerComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | 19 | import cluster.Node 20 | 21 | /** 22 | * A component which provides a network server. 23 | */ 24 | trait NetworkServerComponent { 25 | val networkServer: NetworkServer 26 | 27 | /** 28 | * A NetworkServer listens for incoming messages and processes them using the handler 29 | * registered for that message type with the MessageRegistry. 30 | */ 31 | trait NetworkServer { 32 | 33 | /** 34 | * Binds the network server to a port and marks the node associated with this server available. 35 | */ 36 | def bind: Unit 37 | 38 | /** 39 | * Binds the network server to a port and, if markAvailable is true, marks the node associated with this 40 | * server available. 41 | * 42 | * @param markAvailable specified whether or not to mark the node associated with this server available after 43 | * binding 44 | */ 45 | def bind(markAvailable: Boolean): Unit 46 | 47 | /** 48 | * Retrieves the node associated with this server. 49 | */ 50 | def currentNode: Node 51 | 52 | /** 53 | * Marks the node associated with this server available. If you call bind(false) you must, eventually, 54 | * call this method before the cluster will start sending this server requests. 55 | */ 56 | def markAvailable: Unit 57 | 58 | /** 59 | * Shuts down the NetworkServer. The server will disconnect from the cluster, unbind from the port, 60 | * wait for all currently processing requests to finish, close the sockets that have been accepted and any opened 61 | * client sockets. 62 | */ 63 | def shutdown: Unit 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/NetworkingException.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | 19 | /** 20 | * Base class from which Norbert's networking related exceptions inherit. 21 | */ 22 | class NetworkingException(message: String, cause: Throwable) extends NorbertException(message, cause) { 23 | def this() = this(null, null) 24 | def this(message: String) = this(message, null) 25 | def this(cause: Throwable) = this(cause.getMessage, cause) 26 | } 27 | 28 | /** 29 | * Exception that indicates that a method was called after the network system has been shut down. 30 | */ 31 | class NetworkShutdownException extends NetworkingException 32 | 33 | /** 34 | * Exception that indicates that an exception occurred remotely while processing a request. 35 | */ 36 | class RemoteException(className: String, errorMsg: String) extends NetworkingException("The remote end threw an exception [%s]: %s".format(className, errorMsg)) 37 | 38 | /** 39 | * Exception that indicates that a message was received which was not registered with the MessageRegistry. 40 | */ 41 | class InvalidMessageException(errorMsg: String) extends NetworkingException(errorMsg) 42 | 43 | /** 44 | * Exception that indicates that a malformed response was received. 45 | */ 46 | class InvalidResponseException(errorMsg: String) extends NetworkingException(errorMsg) 47 | 48 | /** 49 | * Exception that indicates that a method has been called before the networking system has been started. 50 | */ 51 | class NetworkNotStartedException extends NetworkingException 52 | 53 | /** 54 | * Exception that indicates that no nodes are available to process the message. 55 | */ 56 | class NoNodesAvailableException(errorMsg: String) extends NetworkingException(errorMsg) 57 | 58 | /** 59 | * Exception that indicates that a method has been called before the network server has been bound. 60 | */ 61 | class NetworkServerNotBoundException extends NetworkingException 62 | 63 | /** 64 | * Exception that indicates that the message was rejected because the waiting queue is full. 65 | */ 66 | class HeavyLoadException extends NetworkingException 67 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/ResponseIterator.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | 19 | import com.google.protobuf.Message 20 | import java.util.concurrent.{TimeoutException, ExecutionException, TimeUnit} 21 | 22 | /** 23 | * An iterator over the responses from a network request. 24 | */ 25 | trait ResponseIterator[ResponseMsg] extends Iterator[ResponseMsg] { 26 | /** 27 | * Calculates whether you have iterated over all of the responses. A return value of true indicates 28 | * that there are more responses, it does not indicate that those responses have been received and 29 | * are immediately available for processing. 30 | * 31 | * @return true if there are additional responses, false otherwise 32 | */ 33 | def hasNext: Boolean 34 | 35 | /** 36 | * Specifies whether a response is available without blocking. 37 | * 38 | * @return true if a response is available without blocking, false otherwise 39 | */ 40 | def nextAvailable: Boolean 41 | 42 | /** 43 | * Retrieves the next response, if necessary waiting until a response is available. 44 | * 45 | * @return a response 46 | * @throws ExecutionException thrown if there was an error 47 | */ 48 | @throws(classOf[ExecutionException]) 49 | def next: ResponseMsg 50 | 51 | /** 52 | * Retrieves the next response, waiting for the specified time if there are no responses available. 53 | * 54 | * @param timeout how long to wait before giving up, in terms of unit 55 | * @param unit the TimeUnit that timeout should be interpreted in 56 | * 57 | * @return a response 58 | * @throws ExecutionException thrown if there was an error 59 | * @throws TimeoutException thrown if a response wasn't available before the specified timeout 60 | * @throws InterruptedException thrown if the thread was interrupted while waiting for the next response 61 | */ 62 | @throws(classOf[ExecutionException]) 63 | @throws(classOf[TimeoutException]) 64 | @throws(classOf[InterruptedException]) 65 | def next(timeout: Long, unit: TimeUnit): ResponseMsg 66 | } 67 | 68 | 69 | /** 70 | * Internal-use only 71 | */ 72 | trait DynamicResponseIterator[ResponseMsg] extends ResponseIterator[ResponseMsg] { 73 | 74 | /** 75 | * Adjust # of remaining items. 76 | */ 77 | def addAndGet(delta:Int) : Int 78 | 79 | } -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/Serializer.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.network 2 | 3 | import java.io.{OutputStream, InputStream} 4 | 5 | /* 6 | * Copyright 2009-2010 LinkedIn, Inc 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | */ 20 | 21 | /** 22 | * When you don't care about variance. IE Java users 23 | */ 24 | trait Serializer[RequestMsg, ResponseMsg] extends InputSerializer[RequestMsg, ResponseMsg] with OutputSerializer[RequestMsg, ResponseMsg] 25 | 26 | // When there is no response message 27 | trait OneWaySerializer[RequestMsg] extends InputSerializer[RequestMsg, Unit] with OutputSerializer[RequestMsg, Unit] { 28 | override def responseName: String = null 29 | override def responseToBytes(response: Unit): Array[Byte] = null 30 | override def responseFromBytes(bytes: Array[Byte]): Unit = null 31 | } 32 | 33 | // Split up for correct variance 34 | trait OutputSerializer[-RequestMsg, -ResponseMsg] { 35 | def responseName: String 36 | def requestToBytes(request: RequestMsg): Array[Byte] 37 | def responseToBytes(response: ResponseMsg): Array[Byte] 38 | } 39 | 40 | trait InputSerializer[+RequestMsg, +ResponseMsg] { 41 | def requestName: String 42 | def requestFromBytes(bytes: Array[Byte]): RequestMsg 43 | def responseFromBytes(bytes: Array[Byte]): ResponseMsg 44 | } 45 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/client/Filter.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert 2 | package network 3 | package client 4 | 5 | /** 6 | * Currently only handling one way IC 7 | */ 8 | 9 | trait Filter { 10 | def onRequest[RequestMsg, ResponseMsg](request: Request[RequestMsg, ResponseMsg]): Unit 11 | } 12 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/client/NetworkClientComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package client 19 | 20 | /** 21 | * A component which provides a network client for interacting with nodes in a cluster. 22 | */ 23 | trait NetworkClientComponent { 24 | val networkClient: NetworkClient 25 | } 26 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/client/ResponseHandlerComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package client 19 | 20 | import protos.NorbertProtos 21 | import protos.NorbertProtos.NorbertMessage 22 | import logging.Logging 23 | import norbertutils.NamedPoolThreadFactory 24 | import jmx.JMX.MBean 25 | import jmx.JMX 26 | import java.util.concurrent._ 27 | import util.ProtoUtils 28 | 29 | trait ResponseHandlerComponent { 30 | val responseHandler: ResponseHandler 31 | } 32 | 33 | trait ResponseHandler { 34 | def onSuccess[RequestMsg, ResponseMsg](request: Request[RequestMsg, ResponseMsg], message: NorbertProtos.NorbertMessage) 35 | def onFailure[RequestMsg, ResponseMsg](request: Request[RequestMsg, ResponseMsg], error: Throwable) 36 | def shutdown: Unit 37 | } 38 | 39 | class ThreadPoolResponseHandler(clientName: Option[String], 40 | serviceName: String, 41 | corePoolSize: Int, 42 | maxPoolSize: Int, 43 | keepAliveTime: Int, 44 | maxWaitingQueueSize: Int, 45 | avoidByteStringCopy: Boolean) extends ResponseHandler with Logging { 46 | 47 | private val responseQueue = new ArrayBlockingQueue[Runnable](maxWaitingQueueSize) 48 | 49 | private val threadPool = new ThreadPoolExecutor(corePoolSize, maxPoolSize, keepAliveTime, TimeUnit.SECONDS, 50 | responseQueue, new NamedPoolThreadFactory("norbert-response-handler")) 51 | 52 | val statsJmx = JMX.register(new ResponseProcessorMBeanImpl(clientName, serviceName, responseQueue)) 53 | 54 | def shutdown { 55 | threadPool.shutdown 56 | statsJmx.foreach(JMX.unregister(_)) 57 | 58 | log.debug("Thread pool response handler shut down") 59 | } 60 | 61 | def onSuccess[RequestMsg, ResponseMsg](request: Request[RequestMsg, ResponseMsg], message: NorbertMessage) { 62 | try { 63 | threadPool.execute(new Runnable { 64 | def run = { 65 | try { 66 | val data = ProtoUtils.byteStringToByteArray(message.getMessage, avoidByteStringCopy) 67 | request.onSuccess(data) 68 | } catch { 69 | case ex: Exception => 70 | request.onFailure(ex) 71 | } 72 | } 73 | }) 74 | } catch { 75 | case (ex: RejectedExecutionException) => 76 | log.warn("Response processing queue full") 77 | request.onFailure(ex) 78 | } 79 | } 80 | 81 | def onFailure[RequestMsg, ResponseMsg](request: Request[RequestMsg, ResponseMsg], error: Throwable) { 82 | threadPool.execute(new Runnable { 83 | def run = request.onFailure(error) 84 | }) 85 | } 86 | } 87 | 88 | trait ResponseProcessorMBean { 89 | def getQueueSize: Int 90 | } 91 | 92 | class ResponseProcessorMBeanImpl(clientName: Option[String], serviceName: String, queue: ArrayBlockingQueue[Runnable]) 93 | extends MBean(classOf[ResponseProcessorMBean], JMX.name(clientName, serviceName)) with ResponseProcessorMBean { 94 | def getQueueSize = queue.size 95 | } 96 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/client/loadbalancer/LoadBalancerFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package client 19 | package loadbalancer 20 | 21 | import cluster.{InvalidClusterException, Node} 22 | import common.Endpoint 23 | 24 | /** 25 | * A LoadBalancer handles calculating the next Node a message should be routed to. 26 | */ 27 | trait LoadBalancer { 28 | /** 29 | * Returns the next Node a message should be routed to. 30 | * 31 | * @return the Some(node) to route the next message to or None if there are no Nodes 32 | * available 33 | */ 34 | def nextNode: Option[Node] = nextNode(None) 35 | 36 | /** 37 | * Returns the next Node that serving the capability value if not None a message should be routed to. 38 | * @param capability 39 | * @return the Some(node) to route the next message to or None if there are no Node's 40 | * available; 41 | */ 42 | def nextNode(capability : Option[Long] = None, permanentCapability: Option[Long] = None): Option[Node] 43 | } 44 | 45 | 46 | /** 47 | * A factory which can generate LoadBalancers. 48 | */ 49 | trait LoadBalancerFactory { 50 | /** 51 | * Create a new load balancer instance based on the currently available Nodes. 52 | * 53 | * @param nodes the currently available Nodes in the cluster 54 | * 55 | * @return a new LoadBalancer instance 56 | * @throws InvalidClusterException thrown to indicate that the current cluster topology is invalid in some way and 57 | * it is impossible to create a LoadBalancer 58 | */ 59 | @throws(classOf[InvalidClusterException]) 60 | def newLoadBalancer(nodes: Set[Endpoint]): LoadBalancer 61 | } 62 | 63 | /** 64 | * A component which provides a LoadBalancerFactory. 65 | */ 66 | trait LoadBalancerFactoryComponent { 67 | val loadBalancerFactory: LoadBalancerFactory 68 | 69 | } 70 | 71 | trait LoadBalancerHelpers { 72 | import java.util.concurrent.atomic.AtomicInteger 73 | import math._ 74 | 75 | def chooseNext[T](items: Seq[T], counter: AtomicInteger): T = 76 | items(abs(counter.getAndIncrement) % items.size) 77 | 78 | } 79 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/client/loadbalancer/RoundRobinLoadBalancerFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package client 19 | package loadbalancer 20 | 21 | import common.Endpoint 22 | import cluster.Node 23 | import java.util.concurrent.atomic.AtomicInteger 24 | import annotation.tailrec 25 | 26 | class RoundRobinLoadBalancerFactory extends LoadBalancerFactory with LoadBalancerHelpers { 27 | def newLoadBalancer(endpointSet: Set[Endpoint]): LoadBalancer = new LoadBalancer { 28 | 29 | val counter = new AtomicInteger(0) 30 | val endpoints = endpointSet.toArray 31 | 32 | def nextNode(capability: Option[Long] = None, permanentCapability: Option[Long] = None) = { 33 | val activeEndpoints = endpoints.filter{ (e : Endpoint) => e.canServeRequests && e.node.isCapableOf(capability, permanentCapability) } 34 | 35 | if(activeEndpoints.isEmpty) 36 | Some(chooseNext(endpoints.filter(_.node.isCapableOf(capability, permanentCapability)), counter).node) 37 | else if(endpoints.isEmpty) 38 | None 39 | else 40 | Some(chooseNext(activeEndpoints, counter).node) 41 | } 42 | 43 | 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/common/ClusterIoClientComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package common 19 | 20 | import cluster.Node 21 | 22 | trait ClusterIoClientComponent { 23 | val clusterIoClient: ClusterIoClient 24 | 25 | trait ClusterIoClient { 26 | def sendMessage[RequestMsg, ResponseMsg](node: Node, request: Request[RequestMsg, ResponseMsg]): Unit 27 | def nodesChanged(nodes: Set[Node]): Set[Endpoint] 28 | def shutdown: Unit 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/common/Endpoint.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.network.common 2 | 3 | import com.linkedin.norbert.cluster.Node 4 | 5 | trait Endpoint { 6 | def node: Node 7 | 8 | def canServeRequests: Boolean 9 | 10 | 11 | override def hashCode() = node.hashCode 12 | 13 | override def equals(that: Any) = that match { 14 | case e: Endpoint => this.node.equals(e.node) 15 | case _ => false 16 | } 17 | 18 | override def toString = "ID = %d ServingRequests = %b Node = %s".format(node.id, canServeRequests, node.toString) 19 | } -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/common/LocalMessageExecution.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package common 19 | 20 | import server.MessageExecutorComponent 21 | import cluster.{Node, ClusterClientComponent} 22 | 23 | trait LocalMessageExecution extends BaseNetworkClient { 24 | this: MessageExecutorComponent with ClusterClientComponent with ClusterIoClientComponent => 25 | 26 | val myNode: Node 27 | 28 | override protected def doSendRequest[RequestMsg, ResponseMsg](requestCtx: Request[RequestMsg, ResponseMsg]) 29 | (implicit is: InputSerializer[RequestMsg, ResponseMsg], os: OutputSerializer[RequestMsg, ResponseMsg]): Unit = { 30 | if(requestCtx.node == myNode) messageExecutor.executeMessage(requestCtx.message, requestCtx.callback) 31 | else super.doSendRequest(requestCtx) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/common/MessageRegistryComponent.scala: -------------------------------------------------------------------------------- 1 | ///* 2 | // * Copyright 2009-2010 LinkedIn, Inc 3 | // * 4 | // * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | // * use this file except in compliance with the License. You may obtain a copy of 6 | // * the License at 7 | // * 8 | // * http://www.apache.org/licenses/LICENSE-2.0 9 | // * 10 | // * Unless required by applicable law or agreed to in writing, software 11 | // * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | // * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | // * License for the specific language governing permissions and limitations under 14 | // * the License. 15 | // */ 16 | //package com.linkedin.norbert 17 | //package network 18 | //package common 19 | // 20 | //trait MessageRegistryComponent { 21 | // val messageRegistry: MessageRegistry 22 | //} 23 | // 24 | //class MessageRegistry { 25 | // @volatile private var messageMap = Map[String, String]() 26 | //// private var messageMap = Map[String, (Message, Message)]() 27 | // 28 | // 29 | // def contains[RequestMsg, ResponseMsg](request: Request[RequestMsg, ResponseMsg]): Boolean = 30 | // messageMap.contains(request.name) 31 | // 32 | // def hasResponse(requestMessage: Message): Boolean = getMessagePair(requestMessage)._2 != null 33 | // 34 | // def registerMessage(requestMessage: Message, responseMessage: Message) { 35 | // if (requestMessage == null) throw new NullPointerException 36 | // val response = if (responseMessage == null) null else responseMessage.getDefaultInstanceForType 37 | // 38 | // messageMap += (requestMessage.getDescriptorForType.getFullName -> (requestMessage.getDefaultInstanceForType, response)) 39 | // } 40 | // 41 | // def responseMessageDefaultInstanceFor(requestMessage: Message): Message = getMessagePair(requestMessage)._2 42 | // 43 | // def validResponseFor(requestMessage: Message, responseName: String): Boolean = { 44 | // if (requestMessage == null || responseName == null) throw new NullPointerException 45 | // val response = getMessagePair(requestMessage)._2 46 | // 47 | // if (response == null) false 48 | // else response.getDescriptorForType.getFullName == responseName 49 | // } 50 | // 51 | // private def getMessagePair(requestMessage: Message) = { 52 | // val name = requestMessage.getDescriptorForType.getFullName 53 | // messageMap.get(name).getOrElse(throw new InvalidMessageException("No such message of type %s registered".format(name))) 54 | // } 55 | //} 56 | // 57 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/netty/NettyClusterIoClientComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package netty 19 | 20 | import java.net.InetSocketAddress 21 | import java.util.concurrent.{ConcurrentHashMap} 22 | import org.jboss.netty.channel.{Channels, ChannelPipelineFactory} 23 | import cluster.Node 24 | import logging.Logging 25 | import common._ 26 | import norbertutils.SystemClock 27 | 28 | /** 29 | * A ClusterIoClientComponent implementation that uses Netty for network communication. 30 | */ 31 | trait NettyClusterIoClientComponent extends ClusterIoClientComponent { 32 | 33 | class NettyClusterIoClient(channelPoolFactory: ChannelPoolFactory, strategy: CanServeRequestStrategy) extends ClusterIoClient with UrlParser with Logging { 34 | private val channelPools = new ConcurrentHashMap[Node, ChannelPool] 35 | 36 | def sendMessage[RequestMsg, ResponseMsg](node: Node, request: Request[RequestMsg, ResponseMsg]) { 37 | if (node == null || request == null) throw new NullPointerException 38 | 39 | val pool = getChannelPool(node) 40 | try { 41 | pool.sendRequest(request) 42 | } catch { 43 | case ex: ChannelPoolClosedException => 44 | // ChannelPool was closed, try again 45 | sendMessage(node, request) 46 | } 47 | } 48 | 49 | def getChannelPool(node: Node): ChannelPool = { 50 | // TODO: Theoretically, we might be able to get a null reference instead of a channel pool here 51 | import norbertutils._ 52 | atomicCreateIfAbsent(channelPools, node) { n: Node => 53 | val (address, port) = parseUrl(n.url) 54 | channelPoolFactory.newChannelPool(new InetSocketAddress(address, port)) 55 | } 56 | } 57 | 58 | def nodesChanged(nodes: Set[Node]): Set[Endpoint] = { 59 | import scala.collection.JavaConversions._ 60 | channelPools.keySet.foreach { node => 61 | if (!nodes.contains(node)) { 62 | val pool = channelPools.remove(node) 63 | pool.close 64 | log.info("Closing pool for unavailable node: %s".format(node)) 65 | } 66 | } 67 | 68 | nodes.map { n => 69 | val requestStrategy = strategy 70 | 71 | new Endpoint { 72 | def node = n 73 | def canServeRequests = requestStrategy.canServeRequest(node) 74 | } 75 | } 76 | } 77 | 78 | def shutdown = { 79 | import scala.collection.JavaConversions._ 80 | 81 | channelPools.keySet.foreach { key => 82 | channelPools.get(key) match { 83 | case null => // do nothing 84 | case pool => 85 | pool.close 86 | channelPools.remove(key) 87 | } 88 | } 89 | 90 | channelPoolFactory.shutdown 91 | 92 | log.debug("NettyClusterIoClient shut down") 93 | } 94 | } 95 | 96 | private class NorbertChannelPipelineFactory extends ChannelPipelineFactory { 97 | val p = Channels.pipeline 98 | 99 | def getPipeline = p 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/netty/NettyClusterIoServerComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package netty 19 | 20 | import org.jboss.netty.bootstrap.ServerBootstrap 21 | import java.net.InetSocketAddress 22 | import org.jboss.netty.channel.{ChannelException, Channel} 23 | import org.jboss.netty.channel.group.ChannelGroup 24 | import server.ClusterIoServerComponent 25 | import logging.Logging 26 | import cluster.Node 27 | 28 | trait NettyClusterIoServerComponent extends ClusterIoServerComponent { 29 | class NettyClusterIoServer(bootstrap: ServerBootstrap, channelGroup: ChannelGroup) extends ClusterIoServer with UrlParser with Logging { 30 | private var serverChannel: Channel = _ 31 | 32 | def bind(node: Node, wildcard: Boolean) = { 33 | val (_, port) = parseUrl(node.url) 34 | try { 35 | val address = new InetSocketAddress(port) 36 | log.debug("Binding server socket to %s".format(address)) 37 | serverChannel = bootstrap.bind(address) 38 | } catch { 39 | case ex: ChannelException => throw new NetworkingException("Unable to bind to %s".format(node), ex) 40 | } 41 | } 42 | 43 | def shutdown = if (serverChannel != null) { 44 | serverChannel.close.awaitUninterruptibly 45 | channelGroup.close.awaitUninterruptibly 46 | bootstrap.releaseExternalResources 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/netty/NettyServerFilter.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert 2 | package network 3 | package netty 4 | 5 | import server.{Filter, RequestContext => SRequestContext} 6 | import protos.NorbertProtos.NorbertMessage 7 | 8 | trait NettyServerFilter extends Filter { 9 | def onMessage(message: NorbertMessage, context: SRequestContext) : Unit 10 | def postMessage(message: NorbertMessage, context: SRequestContext) : Unit 11 | } 12 | 13 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/netty/UrlParser.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package netty 19 | 20 | import cluster.InvalidNodeException 21 | 22 | trait UrlParser { 23 | def parseUrl(url: String): (String, Int) = try { 24 | val Array(a, p) = url.split(":") 25 | (a, p.toInt) 26 | } catch { 27 | case ex: MatchError => throw new InvalidNodeException("Invalid Node url format, must be in the form address:port") 28 | case ex: NumberFormatException => throw new InvalidNodeException("Invalid Node url format, must be in the form address:port", ex) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/partitioned/PartitionedNetworkClientFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package partitioned 19 | 20 | import loadbalancer.PartitionedLoadBalancerFactory 21 | import client.NetworkClientConfig 22 | import cluster.{ClusterClient} 23 | 24 | /** 25 | * Author: jhartman 26 | */ 27 | class PartitionedNetworkClientFactory[PartitionedId](clientName: String, 28 | serviceName: String, 29 | zooKeeperConnectString: String, 30 | zooKeeperSessionTimeoutMillis: Int, 31 | closeChannelTimeMillis: Long, 32 | norbertOutlierMultiplier: Double, 33 | norbertOutlierConstant: Double, 34 | partitionedLoadBalancerFactory: PartitionedLoadBalancerFactory[PartitionedId]) 35 | { 36 | 37 | def createPartitionedNetworkClient : PartitionedNetworkClient[PartitionedId] = { 38 | val config = new NetworkClientConfig 39 | 40 | config.closeChannelTimeMillis = closeChannelTimeMillis 41 | config.outlierMuliplier = norbertOutlierMultiplier 42 | config.outlierConstant = norbertOutlierConstant 43 | config.clusterClient = ClusterClient(clientName, serviceName, zooKeeperConnectString, zooKeeperSessionTimeoutMillis) 44 | val partitionedNetworkClient = PartitionedNetworkClient(config, partitionedLoadBalancerFactory) 45 | 46 | partitionedNetworkClient 47 | } 48 | } 49 | 50 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/partitioned/loadbalancer/HashFunctions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package partitioned 19 | package loadbalancer 20 | 21 | /** 22 | * Object which provides hash function implementations. 23 | */ 24 | object HashFunctions { 25 | /** 26 | * An implementation of the FNV hash function. 27 | * 28 | * @param bytes the bytes to hash 29 | * 30 | * @return the hashed value of the bytes 31 | * 32 | * @see http://en.wikipedia.org/wiki/Fowler-Noll-Vo_hash_function 33 | */ 34 | def fnv[T <% Array[Byte]](bytes: T): Int = { 35 | val FNV_BASIS = 0x811c9dc5 36 | val FNV_PRIME = (1 << 24) + 0x193 37 | 38 | var hash: Long = FNV_BASIS 39 | var i: Int = 0 40 | var maxIdx: Int = bytes.length 41 | 42 | while (i < maxIdx) { 43 | hash = (hash ^ (0xFF & bytes(i))) * FNV_PRIME 44 | i += 1 45 | } 46 | hash.toInt 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/partitioned/loadbalancer/PartitionUtil.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.network.partitioned.loadbalancer 2 | 3 | import com.linkedin.norbert.network.common.Endpoint 4 | 5 | /* 6 | * Copyright 2009-2010 LinkedIn, Inc 7 | * 8 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 9 | * use this file except in compliance with the License. You may obtain a copy of 10 | * the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 17 | * License for the specific language governing permissions and limitations under 18 | * the License. 19 | */ 20 | 21 | object PartitionUtil { 22 | def wheelEntry[K, V](map: java.util.TreeMap[K, V], key: K): java.util.Map.Entry[K, V] = { 23 | val entry = map.ceilingEntry(key) 24 | if(entry == null) 25 | map.firstEntry 26 | else 27 | entry 28 | } 29 | 30 | def rotateWheel[K, V](map: java.util.TreeMap[K, V], key: K): java.util.Map.Entry[K, V] = { 31 | val entry = map.higherEntry(key) 32 | if(entry == null) 33 | map.firstEntry 34 | else 35 | entry 36 | } 37 | 38 | def searchWheel[T, V](wheel: java.util.TreeMap[T, V], key: T, usable: V => Boolean): Option[V] = { 39 | if(wheel.isEmpty) 40 | return None 41 | 42 | val entry = PartitionUtil.wheelEntry(wheel, key) 43 | var e = entry 44 | do { 45 | if(usable(e.getValue)) 46 | return Some(e.getValue) 47 | 48 | // rotate the wheel 49 | e = PartitionUtil.rotateWheel(wheel, e.getKey) 50 | } 51 | while (e != entry) 52 | 53 | if(e == entry) 54 | return None 55 | else 56 | return Some(e.getValue) 57 | } 58 | 59 | } -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/partitioned/loadbalancer/SimpleConsistentHashedLoadBalancerFactory.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | 17 | package com.linkedin.norbert 18 | package network 19 | package partitioned 20 | package loadbalancer 21 | 22 | import common.Endpoint 23 | import java.util.TreeMap 24 | import cluster.{Node, InvalidClusterException} 25 | 26 | /** 27 | * This load balancer is appropriate when any server could handle the request. In this case, the partitions don't really mean anything. They simply control a percentage of the requests 28 | * that the node would receive. For instance, if node A had partitions 0,1,2 and node B had partitions 2,3, Node B would serve 40% of the traffic. 29 | */ 30 | class SimpleConsistentHashedLoadBalancerFactory[PartitionedId](numReplicas: Int, hashFn: PartitionedId => Int, endpointHashFn: String => Int) extends PartitionedLoadBalancerFactory[PartitionedId] { 31 | @throws(classOf[InvalidClusterException]) 32 | def newLoadBalancer(endpoints: Set[Endpoint]): SimpleConsistentHashedLoadBalancer[PartitionedId] = { 33 | val wheel = new TreeMap[Int, Endpoint] 34 | 35 | endpoints.foreach { endpoint => 36 | endpoint.node.partitionIds.foreach { partitionId => 37 | (0 until numReplicas).foreach { r => 38 | val node = endpoint.node 39 | var distKey = node.id + ":" + partitionId + ":" + node.url 40 | wheel.put(endpointHashFn(distKey), endpoint) 41 | } 42 | } 43 | } 44 | 45 | return new SimpleConsistentHashedLoadBalancer(wheel, hashFn) 46 | } 47 | 48 | def getNumPartitions(endpoints: Set[Endpoint]) = { 49 | endpoints.flatMap(_.node.partitionIds).size 50 | } 51 | } 52 | 53 | class SimpleConsistentHashedLoadBalancer[PartitionedId](wheel: TreeMap[Int, Endpoint], hashFn: PartitionedId => Int) extends PartitionedLoadBalancer[PartitionedId] { 54 | 55 | def nodesForOneReplica(id: PartitionedId, capability: Option[Long] = None, persistentCapability: Option[Long] = None) = throw new UnsupportedOperationException 56 | 57 | def nodesForPartitionedId(id: PartitionedId, capability: Option[Long] = None, persistentCapability: Option[Long] = None) = throw new UnsupportedOperationException 58 | 59 | def nodesForPartitions(id: PartitionedId, partitions: Set[Int], capability: Option[Long] = None, persistentCapability: Option[Long] = None) = throw new UnsupportedOperationException 60 | 61 | def nextNode(id: PartitionedId, capability: Option[Long], persistentCapability: Option[Long]): Option[Node] = { 62 | PartitionUtil.searchWheel(wheel, hashFn(id), (e: Endpoint) => e.canServeRequests && e.node.isCapableOf(capability, persistentCapability)).map(_.node) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/server/ClusterIoServerComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package server 19 | 20 | import cluster.Node 21 | 22 | trait ClusterIoServerComponent { 23 | val clusterIoServer: ClusterIoServer 24 | 25 | trait ClusterIoServer { 26 | def bind(node: Node, wildcardAddress: Boolean): Unit 27 | def shutdown: Unit 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/server/Filter.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.network.server 2 | 3 | trait Filter { 4 | def onRequest(request: Any, context: RequestContext): Unit 5 | def onResponse(response: Any, context: RequestContext): Unit 6 | def onError(error: Exception, context: RequestContext): Unit 7 | } -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/server/MessageHandlerRegistryComponent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package server 19 | 20 | trait MessageHandlerRegistryComponent { 21 | val messageHandlerRegistry: MessageHandlerRegistry 22 | } 23 | 24 | private case class MessageHandlerEntry[RequestMsg, ResponseMsg] 25 | (is: InputSerializer[RequestMsg, ResponseMsg], os: OutputSerializer[RequestMsg, ResponseMsg], handler: RequestMsg => ResponseMsg) 26 | 27 | class MessageHandlerRegistry { 28 | @volatile private var handlerMap = 29 | Map.empty[String, MessageHandlerEntry[_ <: Any, _ <: Any]] 30 | 31 | def registerHandler[RequestMsg, ResponseMsg](handler: RequestMsg => ResponseMsg) 32 | (implicit is: InputSerializer[RequestMsg, ResponseMsg], os: OutputSerializer[RequestMsg, ResponseMsg]) { 33 | if(handler == null) throw new NullPointerException 34 | 35 | handlerMap += (is.requestName -> MessageHandlerEntry(is, os, handler)) 36 | } 37 | 38 | @throws(classOf[InvalidMessageException]) 39 | def inputSerializerFor[RequestMsg, ResponseMsg](messageName: String): InputSerializer[RequestMsg, ResponseMsg] = { 40 | handlerMap.get(messageName).map(_.is) 41 | .getOrElse(throw buildException(messageName)) 42 | .asInstanceOf[InputSerializer[RequestMsg, ResponseMsg]] 43 | } 44 | 45 | @throws(classOf[InvalidMessageException]) 46 | def outputSerializerFor[RequestMsg, ResponseMsg](messageName: String): OutputSerializer[RequestMsg, ResponseMsg] = { 47 | handlerMap.get(messageName).map(_.os) 48 | .getOrElse(throw buildException(messageName)) 49 | .asInstanceOf[OutputSerializer[RequestMsg, ResponseMsg]] 50 | } 51 | 52 | @throws(classOf[InvalidMessageException]) 53 | def handlerFor[RequestMsg, ResponseMsg](request: RequestMsg) 54 | (implicit is: InputSerializer[RequestMsg, ResponseMsg]): RequestMsg => ResponseMsg = { 55 | handlerFor[RequestMsg, ResponseMsg](is.requestName) 56 | } 57 | 58 | @throws(classOf[InvalidMessageException]) 59 | def handlerFor[RequestMsg, ResponseMsg](messageName: String): RequestMsg => ResponseMsg = { 60 | handlerMap.get(messageName).map(_.handler) 61 | .getOrElse(throw buildException(messageName)) 62 | .asInstanceOf[RequestMsg => ResponseMsg] 63 | } 64 | 65 | def buildException(messageName: String) = 66 | new InvalidMessageException("%s is not a registered method. Methods registered are %s".format(messageName, "(" + handlerMap.keys.mkString(",") + ")")) 67 | } -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/server/RequestContext.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert.network.server 2 | 3 | import scala.collection.mutable.Map 4 | 5 | trait RequestContext { 6 | val attributes: Map[String, Any] = Map.empty[String, Any] 7 | } 8 | 9 | -------------------------------------------------------------------------------- /network/src/main/scala/com/linkedin/norbert/network/util/ProtoUtils.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert 2 | package network 3 | package util 4 | 5 | import com.google.protobuf.ByteString 6 | import logging.Logging 7 | import java.lang.reflect.{Field, Constructor} 8 | 9 | /** 10 | * A wrapper for converting from byte[] <-> ByteString. Protocol buffers makes unnecessary 11 | * defensive copies at each conversion, and this class encapsulates logic using reflection 12 | * to bypass those. 13 | */ 14 | object ProtoUtils extends Logging { 15 | private val byteStringConstructor: Constructor[ByteString] = try { 16 | val c = classOf[ByteString].getDeclaredConstructor(classOf[Array[Byte]]) 17 | c.setAccessible(true) 18 | c 19 | } catch { 20 | case ex: Exception => 21 | log.warn(ex, "Cannot eliminate a copy when converting a byte[] to a ByteString") 22 | null 23 | } 24 | 25 | private val byteStringField: Field = try { 26 | val f = classOf[ByteString].getDeclaredField("bytes") 27 | f.setAccessible(true) 28 | f 29 | } catch { 30 | case ex: Exception => 31 | log.warn(ex, "Cannot eliminate a copy when converting a ByteString to a byte[]") 32 | null 33 | } 34 | 35 | def byteArrayToByteString(byteArray: Array[Byte], avoidByteStringCopy: Boolean): ByteString = { 36 | if(avoidByteStringCopy) 37 | fastByteArrayToByteString(byteArray) 38 | else 39 | slowByteArrayToByteString(byteArray) 40 | } 41 | 42 | def byteStringToByteArray(byteString: ByteString, avoidByteStringCopy: Boolean): Array[Byte] = { 43 | if(avoidByteStringCopy) 44 | fastByteStringToByteArray(byteString) 45 | else 46 | slowByteStringToByteArray(byteString) 47 | } 48 | 49 | private final def fastByteArrayToByteString(byteArray: Array[Byte]): ByteString = { 50 | if(byteStringConstructor != null) 51 | try { 52 | byteStringConstructor.newInstance(byteArray) 53 | } catch { 54 | case ex: Exception => 55 | log.warn(ex, "Encountered exception invoking the private ByteString constructor, falling back to safe method") 56 | slowByteArrayToByteString(byteArray) 57 | } 58 | else 59 | slowByteArrayToByteString(byteArray) 60 | } 61 | 62 | private final def slowByteArrayToByteString(byteArray: Array[Byte]): ByteString = { 63 | ByteString.copyFrom(byteArray) 64 | } 65 | 66 | private final def fastByteStringToByteArray(byteString: ByteString): Array[Byte] = { 67 | if(byteStringField != null) 68 | try { 69 | byteStringField.get(byteString).asInstanceOf[Array[Byte]] 70 | } catch { 71 | case ex: Exception => 72 | log.warn(ex, "Encountered exception accessing the private ByteString bytes field, falling back to safe method") 73 | slowByteStringToByteArray(byteString) 74 | } 75 | else 76 | slowByteStringToByteArray(byteString) 77 | } 78 | 79 | private final def slowByteStringToByteArray(byteString: ByteString): Array[Byte] = { 80 | byteString.toByteArray 81 | } 82 | } -------------------------------------------------------------------------------- /network/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=FATAL, CONSOLE 2 | log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender 3 | log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout 4 | log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n 5 | -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/client/loadbalancer/RoundRobinLoadBalancerFactorySpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package client 19 | package loadbalancer 20 | 21 | import org.specs.Specification 22 | import cluster.Node 23 | import common.Endpoint 24 | 25 | class RoundRobinLoadBalancerFactorySpec extends Specification { 26 | "RoundRobinLoadBalancerFactory" should { 27 | "create a round robin load balancer" in { 28 | val nodes = Set(Node(1, "localhost:31310", true), Node(2, "localhost:31311", true), Node(3, "localhost:31312", true), 29 | Node(4, "localhost:31313", true), Node(5, "localhost:31314", true), Node(6, "localhost:31315", true), 30 | Node(7, "localhost:31316", true), Node(8, "localhost:31317", true), Node(9, "localhost:31318", true), 31 | Node(10, "localhost:31319", true)) 32 | val loadBalancerFactory = new RoundRobinLoadBalancerFactory 33 | 34 | val endpoints = nodes.map(n => new Endpoint { 35 | def node = n 36 | 37 | def canServeRequests = true 38 | }) 39 | 40 | val lb = loadBalancerFactory.newLoadBalancer(endpoints) 41 | 42 | for (i <- 0 until 100) { 43 | val node = lb.nextNode 44 | node must beSome[Node].which { nodes must contain(_) } 45 | } 46 | } 47 | 48 | "Not route to offline endpoints" in { 49 | val nodes = for(i <- 0 until 3) yield Node(i, "localhost:3131" + i, true) 50 | 51 | var availabilityMap = nodes.map(n => (n, true)).toMap 52 | 53 | val lbf = new RoundRobinLoadBalancerFactory 54 | 55 | val endpoints = nodes.map(n => new Endpoint { 56 | def node = n 57 | 58 | def canServeRequests = availabilityMap(n) 59 | }).toSet 60 | 61 | var lb = lbf.newLoadBalancer(endpoints) 62 | 63 | for(i <- 0 until 9) { 64 | val node = lb.nextNode.get 65 | val nodeId = i % 3 66 | node must be_==(Node(nodeId, "localhost:3131" + nodeId, true)) 67 | } 68 | 69 | availabilityMap += Node(0, "localhost:31310", true) -> false 70 | 71 | lb = lbf.newLoadBalancer(endpoints) 72 | for(i <- 0 until 9) { 73 | val node = lb.nextNode.get 74 | val nodeId = i % 2 + 1 75 | node must be_==(Node(nodeId, "localhost:3131" + nodeId, true)) 76 | } 77 | } 78 | 79 | "Not route to node cannot fulfill capability" in { 80 | val nodes = for (i <- 0 until 4) yield Node(i, "localhost:3131" + i, true, Set.empty[Int], Some(i), Some(i)) 81 | 82 | val endpoints = nodes.map(n => new Endpoint { 83 | def node = n 84 | def canServeRequests = true 85 | }).toSet 86 | 87 | val loadBalancerFactory = new RoundRobinLoadBalancerFactory 88 | val lb = loadBalancerFactory.newLoadBalancer(endpoints) 89 | 90 | for(i <- 0 until 8) { 91 | val node = lb.nextNode.get 92 | val nodeId = i % 4 93 | node must be_==(Node(nodeId, "localhost:3131" + nodeId, true)) 94 | } 95 | 96 | for(i <- 0 until 8) { 97 | val node = lb.nextNode(Some(1), Some(1)).get 98 | val nodeId = ((i % 2) << 1) | 1 99 | node must be_==(Node(nodeId, "localhost:3131" + nodeId, true)) 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/common/LocalMessageExecutionSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package common 19 | 20 | import org.specs.Specification 21 | import org.specs.mock.Mockito 22 | import client.NetworkClient 23 | import client.loadbalancer.{LoadBalancerFactory, LoadBalancer, LoadBalancerFactoryComponent} 24 | import server._ 25 | import cluster.{Node, ClusterClientComponent, ClusterClient} 26 | import com.google.protobuf.Message 27 | import scala.collection.mutable.MutableList 28 | 29 | class LocalMessageExecutionSpec extends Specification with Mockito with SampleMessage { 30 | val clusterClient = mock[ClusterClient] 31 | 32 | val messageExecutor = new MessageExecutor { 33 | var called = false 34 | var request: Any = _ 35 | val filters = new MutableList[Filter] 36 | def shutdown = {} 37 | 38 | def executeMessage[RequestMsg, ResponseMsg](request: RequestMsg, responseHandler: Option[(Either[Exception, ResponseMsg]) => Unit], context: Option[RequestContext])(implicit is: InputSerializer[RequestMsg, ResponseMsg]) = { 39 | called = true 40 | this.request = request 41 | 42 | val response = null.asInstanceOf[ResponseMsg] 43 | 44 | responseHandler.get(Right(response)) 45 | } 46 | } 47 | 48 | val networkClient = new NetworkClient with ClusterClientComponent with ClusterIoClientComponent with LoadBalancerFactoryComponent 49 | with MessageExecutorComponent with LocalMessageExecution { 50 | val lb = mock[LoadBalancer] 51 | val loadBalancerFactory = mock[LoadBalancerFactory] 52 | val clusterIoClient = mock[ClusterIoClient] 53 | // val messageRegistry = mock[MessageRegistry] 54 | val clusterClient = LocalMessageExecutionSpec.this.clusterClient 55 | val messageExecutor = LocalMessageExecutionSpec.this.messageExecutor 56 | val myNode = Node(1, "localhost:31313", true) 57 | } 58 | 59 | 60 | val nodes = Set(Node(1, "", true), Node(2, "", true), Node(3, "", true)) 61 | val endpoints = nodes.map { n => new Endpoint { 62 | def node = n 63 | def canServeRequests = true 64 | }} 65 | val message = mock[Message] 66 | 67 | // networkClient.messageRegistry.contains(any[Message]) returns true 68 | clusterClient.nodes returns nodes 69 | clusterClient.isConnected returns true 70 | networkClient.clusterIoClient.nodesChanged(nodes) returns endpoints 71 | networkClient.loadBalancerFactory.newLoadBalancer(endpoints) returns networkClient.lb 72 | 73 | "LocalMessageExecution" should { 74 | "call the MessageExecutor if myNode is equal to the node the request is to be sent to" in { 75 | networkClient.lb.nextNode(None, None) returns Some(networkClient.myNode) 76 | 77 | networkClient.start 78 | 79 | networkClient.sendRequest(request) must notBeNull 80 | 81 | messageExecutor.called must beTrue 82 | messageExecutor.request must be_==(request) 83 | } 84 | 85 | "not call the MessageExecutor if myNode is not equal to the node the request is to be sent to" in { 86 | networkClient.lb.nextNode(None, None) returns Some(Node(2, "", true)) 87 | 88 | networkClient.start 89 | networkClient.sendRequest(request) must notBeNull 90 | 91 | messageExecutor.called must beFalse 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/common/MessageRegistrySpec.scala: -------------------------------------------------------------------------------- 1 | ///* 2 | // * Copyright 2009-2010 LinkedIn, Inc 3 | // * 4 | // * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | // * use this file except in compliance with the License. You may obtain a copy of 6 | // * the License at 7 | // * 8 | // * http://www.apache.org/licenses/LICENSE-2.0 9 | // * 10 | // * Unless required by applicable law or agreed to in writing, software 11 | // * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | // * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | // * License for the specific language governing permissions and limitations under 14 | // * the License. 15 | // */ 16 | //package com.linkedin.norbert 17 | //package network 18 | //package common 19 | // 20 | //import org.specs.Specification 21 | //import org.specs.mock.Mockito 22 | //import com.google.protobuf.Message 23 | //import protos.NorbertExampleProtos 24 | // 25 | //class MessageRegistrySpec extends Specification with Mockito { 26 | //// val messageRegistry = new MessageRegistry 27 | // 28 | // val proto = NorbertExampleProtos.Ping.newBuilder.setTimestamp(System.currentTimeMillis).build 29 | // 30 | // "MessageRegistry" should { 31 | // "throw a NullPointerException if requestMessage is null" in { 32 | //// messageRegistry.registerMessage(null, null) must throwA[NullPointerException] 33 | // } 34 | // 35 | // "throw an InvalidMessageExceptio if the requestMessage isn't registered" in { 36 | // messageRegistry.hasResponse(proto) must throwA[InvalidMessageException] 37 | // messageRegistry.responseMessageDefaultInstanceFor(proto) must throwA[InvalidMessageException] 38 | // } 39 | // 40 | // "contains returns true if the specified request message has been registered" in { 41 | // val response = mock[Message] 42 | // 43 | // messageRegistry.contains(proto) must beFalse 44 | // messageRegistry.registerMessage(proto, proto) 45 | // messageRegistry.contains(proto) must beTrue 46 | // } 47 | // 48 | // "return true for hasResponse if the responseMessage is not null" in { 49 | // messageRegistry.registerMessage(proto, null) 50 | // messageRegistry.hasResponse(proto) must beFalse 51 | // 52 | // messageRegistry.registerMessage(proto, proto) 53 | // messageRegistry.hasResponse(proto) must beTrue 54 | // } 55 | // 56 | // "return true if the response message is of the correct type" in { 57 | // val name = "norbert.PingResponse" 58 | // messageRegistry.registerMessage(proto, null) 59 | // messageRegistry.validResponseFor(proto, name) must beFalse 60 | // 61 | // messageRegistry.registerMessage(proto, proto) 62 | // messageRegistry.validResponseFor(proto, name) must beFalse 63 | // 64 | // messageRegistry.validResponseFor(proto, proto.getDescriptorForType.getFullName) must beTrue 65 | // } 66 | // } 67 | //} 68 | -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/common/NorbertFutureSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package common 19 | 20 | import org.specs.Specification 21 | import org.specs.mock.Mockito 22 | import java.util.concurrent.{TimeoutException, ExecutionException, TimeUnit} 23 | import scala.Right 24 | 25 | class NorbertFutureSpec extends Specification with Mockito with SampleMessage { 26 | val future = new FutureAdapter[Ping] 27 | 28 | "NorbertFuture" should { 29 | "not be done when created" in { 30 | future.isDone must beFalse 31 | } 32 | 33 | "be done when value is set" in { 34 | future.apply(Right(new Ping)) 35 | future.isDone must beTrue 36 | } 37 | 38 | "return the value that is set" in { 39 | val message = new Ping 40 | future.apply(Right(request)) 41 | future.get must be(request) 42 | future.get(1, TimeUnit.MILLISECONDS) must be(request) 43 | } 44 | 45 | "throw a TimeoutException if no response is available" in { 46 | future.get(1, TimeUnit.MILLISECONDS) must throwA[TimeoutException] 47 | } 48 | 49 | "throw an ExecutionExcetion for an error" in { 50 | val ex = new Exception 51 | future.apply(Left(ex)) 52 | future.get must throwA[ExecutionException] 53 | future.get(1, TimeUnit.MILLISECONDS) must throwA[ExecutionException] 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/common/NorbertResponseIteratorSpec.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert 2 | package network 3 | package common 4 | 5 | import org.specs.Specification 6 | import org.specs.mock.Mockito 7 | import java.util.concurrent.{ExecutionException, TimeoutException, TimeUnit} 8 | 9 | class NorbertResponseIteratorSpec extends Specification with Mockito with SampleMessage { 10 | val responseQueue = new ResponseQueue[Ping] 11 | val it = new NorbertResponseIterator[Ping](2, responseQueue) 12 | 13 | "NorbertResponseIterator" should { 14 | // "return true for next until all responses have been consumed" in { 15 | // it.hasNext must beTrue 16 | // 17 | // responseQueue += (Right(new Ping)) 18 | // responseQueue += (Right(new Ping)) 19 | // it.next must notBeNull 20 | // it.hasNext must beTrue 21 | // 22 | // it.next must notBeNull 23 | // it.hasNext must beFalse 24 | // } 25 | // 26 | // "return true for nextAvailable if any responses are available" in { 27 | // it.nextAvailable must beFalse 28 | // responseQueue += (Right(new Ping)) 29 | // it.nextAvailable must beTrue 30 | // } 31 | // 32 | // "throw a TimeoutException if no response is available" in { 33 | // it.next(1, TimeUnit.MILLISECONDS) must throwA[TimeoutException] 34 | // } 35 | // 36 | // "throw an ExecutionException for an error" in { 37 | // val ex = new Exception 38 | // responseQueue += (Left(ex)) 39 | // it.next must throwA[ExecutionException] 40 | // } 41 | 42 | "Handle exceptions at the end of the stream" in { 43 | val responseQueue = new ResponseQueue[Int] 44 | responseQueue += Right(0) 45 | responseQueue += Right(1) 46 | 47 | val norbertResponseIterator = new NorbertResponseIterator[Int](3, responseQueue) 48 | val timeoutIterator = new TimeoutIterator(norbertResponseIterator, 40L) 49 | val exceptionIterator = new ExceptionIterator(timeoutIterator) 50 | 51 | val partialIterator = new PartialIterator(exceptionIterator) 52 | 53 | partialIterator.hasNext mustBe true 54 | partialIterator.next mustBe 0 55 | 56 | partialIterator.hasNext mustBe true 57 | partialIterator.next mustBe 1 58 | 59 | partialIterator.hasNext mustBe false 60 | } 61 | 62 | "Handle exceptions in the middle of the stream" in { 63 | val responseQueue = new ResponseQueue[Int] 64 | responseQueue += Right(0) 65 | responseQueue += Left(new TimeoutException) 66 | responseQueue += Right(1) 67 | 68 | val norbertResponseIterator = new NorbertResponseIterator[Int](3, responseQueue) 69 | val timeoutIterator = new TimeoutIterator(norbertResponseIterator, 100L) 70 | val exceptionIterator = new ExceptionIterator(timeoutIterator) 71 | 72 | val partialIterator = new PartialIterator(exceptionIterator) 73 | 74 | partialIterator.hasNext mustBe true 75 | partialIterator.next mustBe 0 76 | 77 | partialIterator.hasNext mustBe true 78 | partialIterator.next mustBe 1 79 | 80 | partialIterator.hasNext mustBe false 81 | } 82 | } 83 | } -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/common/SampleMessage.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert.network.common 17 | 18 | import com.linkedin.norbert.protos.NorbertExampleProtos 19 | import com.linkedin.norbert.network.Serializer 20 | 21 | trait SampleMessage { 22 | object Ping { 23 | implicit case object PingSerializer extends Serializer[Ping, Ping] { 24 | def requestName = "ping" 25 | def responseName = "pong" 26 | 27 | def requestToBytes(message: Ping) = 28 | NorbertExampleProtos.Ping.newBuilder.setTimestamp(message.timestamp).build.toByteArray 29 | 30 | def requestFromBytes(bytes: Array[Byte]) = { 31 | Ping(NorbertExampleProtos.Ping.newBuilder.mergeFrom(bytes).build.getTimestamp) 32 | } 33 | 34 | def responseToBytes(message: Ping) = 35 | requestToBytes(message) 36 | 37 | def responseFromBytes(bytes: Array[Byte]) = 38 | requestFromBytes(bytes) 39 | } 40 | } 41 | 42 | case class Ping(timestamp: Long = System.currentTimeMillis) 43 | val request = new Ping 44 | } -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/netty/ClientChannelHandlerSpec.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert 2 | package network 3 | package netty 4 | 5 | import org.specs.Specification 6 | import org.specs.mock.Mockito 7 | import client.ResponseHandler 8 | import org.jboss.netty.channel._ 9 | import protos.NorbertProtos.NorbertMessage.Status 10 | import protos.NorbertProtos 11 | import cluster.Node 12 | import java.net.{SocketAddress} 13 | import common.SampleMessage 14 | 15 | /** 16 | * Test to cover association of RequestAccess with remote exception 17 | */ 18 | class ClientChannelHandlerSpec extends Specification with Mockito with SampleMessage { 19 | 20 | val responseHandler = mock[ResponseHandler] 21 | val clientChannelHandler = new ClientChannelHandler(clientName = Some("booClient"), 22 | serviceName = "booService", 23 | staleRequestTimeoutMins = 3000, 24 | staleRequestCleanupFrequencyMins = 3000, 25 | requestStatisticsWindow = 3000L, 26 | outlierMultiplier = 2, 27 | outlierConstant = 2, 28 | responseHandler = responseHandler, 29 | avoidByteStringCopy = true) 30 | 31 | def sendMockRequest(ctx: ChannelHandlerContext, request: Request[Ping, Ping]) { 32 | val writeEvent = mock[MessageEvent] 33 | writeEvent.getChannel returns ctx.getChannel 34 | val channelFuture = mock[ChannelFuture] 35 | writeEvent.getFuture returns channelFuture 36 | writeEvent.getMessage returns request 37 | clientChannelHandler.writeRequested(ctx, writeEvent) 38 | } 39 | 40 | "ClientChannelHandler" should { 41 | "throw exception with RequestAccess when server response is HEAVYLOAD" in { 42 | val channel = mock[Channel] 43 | channel.getRemoteAddress returns mock[SocketAddress] 44 | val ctx = mock[ChannelHandlerContext] 45 | ctx.getChannel returns channel 46 | val request = Request[Ping, Ping](Ping(System.currentTimeMillis), Node(1, "localhost:1234", true), Ping.PingSerializer, Ping.PingSerializer, Some({ e => e })) 47 | sendMockRequest(ctx, request) 48 | 49 | val readEvent = mock[MessageEvent] 50 | val norbertMessage = NorbertProtos.NorbertMessage.newBuilder().setStatus(Status.HEAVYLOAD) 51 | .setRequestIdLsb(request.id.getLeastSignificantBits) 52 | .setRequestIdMsb(request.id.getMostSignificantBits) 53 | .setMessageName("Boo") 54 | .build 55 | readEvent.getMessage returns norbertMessage 56 | clientChannelHandler.messageReceived(ctx, readEvent) 57 | there was one(responseHandler).onFailure(any[Request[_,_]], any[Throwable with RequestAccess[Request[_, _]]]) 58 | } 59 | 60 | "throw exception with RequestAccess when server response is ERROR" in { 61 | val channel = mock[Channel] 62 | channel.getRemoteAddress returns mock[SocketAddress] 63 | val ctx = mock[ChannelHandlerContext] 64 | ctx.getChannel returns channel 65 | val request = Request[Ping, Ping](Ping(System.currentTimeMillis), Node(1, "localhost:1234", true), Ping.PingSerializer, Ping.PingSerializer, Some({ e => e })) 66 | sendMockRequest(ctx, request) 67 | 68 | val norbertMessage = NorbertProtos.NorbertMessage.newBuilder().setStatus(Status.ERROR) 69 | .setRequestIdLsb(request.id.getLeastSignificantBits) 70 | .setRequestIdMsb(request.id.getMostSignificantBits) 71 | .setMessageName("Boo") 72 | .setErrorMessage("BooBoo") 73 | .build 74 | val readEvent = mock[MessageEvent] 75 | readEvent.getMessage returns norbertMessage 76 | clientChannelHandler.messageReceived(ctx, readEvent) 77 | there was one(responseHandler).onFailure(any[Request[_,_]], any[Throwable with RequestAccess[Request[_, _]]]) 78 | } 79 | } 80 | 81 | } -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/netty/ClientStatisticsRequestStrategySpec.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | 17 | package com.linkedin.norbert.network.netty 18 | 19 | import org.specs.Specification 20 | import org.specs.mock.Mockito 21 | import java.util.UUID 22 | import com.linkedin.norbert.norbertutils.MockClock 23 | import com.linkedin.norbert.cluster.Node 24 | import com.linkedin.norbert.network.common.CachedNetworkStatistics 25 | 26 | class ClientStatisticsRequestStrategySpec extends Specification with Mockito { 27 | "ClientStatisticsRequestStrategy" should { 28 | "route away from misbehaving nodes" in { 29 | val statsActor = CachedNetworkStatistics[Node, UUID](MockClock, 1000L, 200L) 30 | 31 | val strategy = new ClientStatisticsRequestStrategy(statsActor, 2.0, 10.0, MockClock) 32 | 33 | val nodes = (0 until 5).map { nodeId => Node(nodeId, "foo", true) } 34 | 35 | // Give everybody 10 requests 36 | val requests = nodes.map { node => 37 | val uuids = (0 until 10).map(i => UUID.randomUUID) 38 | uuids.foreach{ uuid => statsActor.beginRequest(node, uuid) } 39 | (node, uuids) 40 | }.toMap 41 | 42 | nodes.dropRight(1).foreach{ node => 43 | MockClock.currentTime = (node.id + 1) 44 | 45 | val uuids = requests(node) 46 | uuids.zipWithIndex.foreach { case (uuid, index) => 47 | statsActor.endRequest(node, uuid) 48 | } 49 | } 50 | 51 | MockClock.currentTime = 5 52 | 53 | nodes.foreach { node => 54 | strategy.canServeRequest(node) must beTrue 55 | 56 | } 57 | 58 | MockClock.currentTime = 15 59 | strategy.canServeRequest(nodes(4)) must beTrue 60 | 61 | // requests(4).zipWithIndex.foreach { case(uuid, index) => 62 | // val nodeId = 4 63 | // MockClock.currentTime = 2 * (nodeId + 1) * index 64 | // statsActor ! statsActor.Stats.EndRequest(nodeId, uuid) 65 | // } 66 | 67 | // strategy.canServeRequest(nodes(4)) must beFalse 68 | 69 | } 70 | } 71 | } -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/netty/NettyClusterIoClientComponentSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package netty 19 | 20 | import org.specs.Specification 21 | import java.net.InetSocketAddress 22 | import org.specs.util.WaitFor 23 | import org.specs.mock.Mockito 24 | import cluster.{InvalidNodeException, Node} 25 | import common.AlwaysAvailableRequestStrategy 26 | 27 | class NettyClusterIoClientComponentSpec extends Specification with Mockito with WaitFor with NettyClusterIoClientComponent { 28 | val messageRegistry = null 29 | 30 | val channelPoolFactory = mock[ChannelPoolFactory] 31 | val channelPool = mock[ChannelPool] 32 | 33 | val clusterIoClient = new NettyClusterIoClient(channelPoolFactory, AlwaysAvailableRequestStrategy) 34 | 35 | val node = Node(1, "localhost:31313", true) 36 | val address = new InetSocketAddress("localhost", 31313) 37 | 38 | "NettyClusterIoClient" should { 39 | "create a new ChannelPool if no pool is available" in { 40 | doNothing.when(channelPool).sendRequest(any[Request[_, _]]) 41 | channelPoolFactory.newChannelPool(address) returns channelPool 42 | 43 | 44 | 45 | clusterIoClient.sendMessage(node, mock[Request[_, _]]) 46 | 47 | got { 48 | one(channelPool).sendRequest(any[Request[_, _]]) 49 | one(channelPoolFactory).newChannelPool(address) 50 | } 51 | } 52 | 53 | "not create a ChannelPool if a pool is available" in { 54 | channelPoolFactory.newChannelPool(address) returns channelPool 55 | doNothing.when(channelPool).sendRequest(any[Request[_, _]]) 56 | 57 | clusterIoClient.sendMessage(node, mock[Request[_, _]]) 58 | clusterIoClient.sendMessage(node, mock[Request[_, _]]) 59 | 60 | got { 61 | two(channelPool).sendRequest(any[Request[_, _]]) 62 | one(channelPoolFactory).newChannelPool(address) 63 | } 64 | } 65 | 66 | "close an open ChannelPool if the Node is no longer available" in { 67 | doNothing.when(channelPool).sendRequest(any[Request[_, _]]) 68 | doNothing.when(channelPool).close 69 | channelPoolFactory.newChannelPool(address) returns channelPool 70 | 71 | clusterIoClient.sendMessage(node, mock[Request[_, _]]) 72 | clusterIoClient.nodesChanged(Set()) 73 | 74 | got { 75 | one(channelPoolFactory).newChannelPool(address) 76 | one(channelPool).close 77 | } 78 | } 79 | 80 | "throw an InvalidNodeException if a Node with an invalid url is provided" in { 81 | channelPoolFactory.newChannelPool(address) returns channelPool 82 | clusterIoClient.sendMessage(Node(1, "foo", true), mock[Request[_, _]]) must throwA[InvalidNodeException] 83 | clusterIoClient.sendMessage(Node(1, "foo:foo", true), mock[Request[_, _]]) must throwA[InvalidNodeException] 84 | } 85 | 86 | "close all ChannelPools when shutdown is called" in { 87 | channelPoolFactory.newChannelPool(address) returns channelPool 88 | doNothing.when(channelPool).close 89 | doNothing.when(channelPoolFactory).shutdown 90 | 91 | clusterIoClient.sendMessage(node, mock[Request[_, _]]) 92 | clusterIoClient.shutdown 93 | 94 | got { 95 | one(channelPool).close 96 | one(channelPoolFactory).shutdown 97 | } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/netty/NettyClusterIoServerComponentSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package netty 19 | 20 | import org.specs.Specification 21 | import org.specs.mock.Mockito 22 | import org.jboss.netty.bootstrap.ServerBootstrap 23 | import java.net.InetSocketAddress 24 | import org.jboss.netty.channel.{ChannelFuture, Channel} 25 | import org.jboss.netty.channel.group.{ChannelGroupFuture, ChannelGroup} 26 | import cluster.{InvalidNodeException, Node} 27 | 28 | class NettyClusterIoServerComponentSpec extends Specification with Mockito with NettyClusterIoServerComponent { 29 | val bootstrap = mock[ServerBootstrap] 30 | val channelGroup = mock[ChannelGroup] 31 | val clusterIoServer = new NettyClusterIoServer(bootstrap, channelGroup) 32 | 33 | "NettyClusterIoServer" should { 34 | "bind should fail if the node's url is in the incorrect format" in { 35 | clusterIoServer.bind(Node(1, "", false), true) must throwA[InvalidNodeException] 36 | clusterIoServer.bind(Node(1, "localhost", false), true) must throwA[InvalidNodeException] 37 | clusterIoServer.bind(Node(1, "localhost:foo", false), true) must throwA[InvalidNodeException] 38 | } 39 | 40 | "binds to the specified port" in { 41 | val address = new InetSocketAddress(31313) 42 | val channel = mock[Channel] 43 | bootstrap.bind(address) returns channel 44 | 45 | clusterIoServer.bind(Node(1, "localhost:31313", false), true) 46 | 47 | there was one(bootstrap).bind(address) 48 | } 49 | 50 | "shutdown should shutdown opened sockets" in { 51 | val channel = mock[Channel] 52 | val socketFuture = mock[ChannelFuture] 53 | val groupFuture = mock[ChannelGroupFuture] 54 | channel.close returns socketFuture 55 | channelGroup.close returns groupFuture 56 | socketFuture.awaitUninterruptibly returns socketFuture 57 | groupFuture.awaitUninterruptibly returns groupFuture 58 | bootstrap.bind(any[InetSocketAddress]) returns channel 59 | 60 | 61 | clusterIoServer.bind(Node(1, "localhost:31313", false), true) 62 | clusterIoServer.shutdown 63 | 64 | got { 65 | one(channel).close 66 | one(socketFuture).awaitUninterruptibly 67 | 68 | one(channelGroup).close 69 | one(groupFuture).awaitUninterruptibly 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/partitioned/loadbalancer/DefaultPartitionedLoadBalancerSpec.scala: -------------------------------------------------------------------------------- 1 | package com.linkedin.norbert 2 | package network 3 | package partitioned 4 | package loadbalancer 5 | 6 | import org.specs.Specification 7 | import cluster.{InvalidClusterException, Node} 8 | import common.Endpoint 9 | 10 | import com.linkedin.norbert.cluster.Node 11 | import com.linkedin.norbert.network.common.Endpoint 12 | 13 | 14 | /** 15 | * @auther: rwang 16 | * @date: 10/11/12 17 | * @version: $Revision$ 18 | */ 19 | 20 | class TestSetCoverPartitionedLoadBalancer extends Specification { 21 | class TestLBF(numPartitions: Int, csr: Boolean = true) 22 | extends DefaultPartitionedLoadBalancerFactory[Int](numPartitions,csr) 23 | { 24 | protected def calculateHash(id: Int) = HashFunctions.fnv(BigInt(id).toByteArray) 25 | 26 | def getNumPartitions(endpoints: Set[Endpoint]) = numPartitions 27 | } 28 | 29 | val loadBalancerFactory = new TestLBF(6) 30 | 31 | class TestEndpoint(val node: Node, var csr: Boolean) extends Endpoint { 32 | def canServeRequests = csr 33 | 34 | def setCsr(ncsr: Boolean) { 35 | csr = ncsr 36 | } 37 | } 38 | 39 | def toEndpoints(nodes: Set[Node]): Set[Endpoint] = nodes.map(n => new TestEndpoint(n, true)) 40 | 41 | def markUnavailable(endpoints: Set[Endpoint], id: Int) { 42 | endpoints.foreach { endpoint => 43 | if (endpoint.node.id == id) { 44 | endpoint.asInstanceOf[TestEndpoint].setCsr(false) 45 | } 46 | } 47 | } 48 | 49 | val nodes = Set(Node(1, "node 1", true, Set(0,1,2)), 50 | Node(2, "node 2", true, Set(3,4,0)), 51 | Node(3, "node 3", true, Set(1,2,3)), 52 | Node(4, "node 4", true, Set(4,5,0)), 53 | Node(5, "node 5", true, Set(1,2,3)), 54 | Node(6, "node 6", true, Set(4,5,0))) 55 | 56 | 57 | "Set cover load balancer" should { 58 | "nodesForPartitions returns nodes cover the input partitioned Ids" in { 59 | val loadbalancer = loadBalancerFactory.newLoadBalancer(toEndpoints(nodes)) 60 | val res = loadbalancer.nodesForPartitionedIds(Set(0,1,3,4), Some(0L), Some(0L)) 61 | res.values.flatten.toSet must be_==(Set(1, 0, 3, 4)) 62 | } 63 | 64 | "nodesForPartitions returns partial results when not all partitions available" in { 65 | val endpoints = toEndpoints(nodes) 66 | markUnavailable(endpoints, 1) 67 | markUnavailable(endpoints, 3) 68 | markUnavailable(endpoints, 5) 69 | 70 | val loadbalancer = loadBalancerFactory.newLoadBalancer(endpoints) 71 | val res = loadbalancer.nodesForPartitionedIds(Set(3,5), Some(0L), Some(0L)) 72 | res must be_== (Map()) 73 | } 74 | 75 | "throw NoNodeAvailable if partition is missing and serveRequestsIfPartitionMissing set to false" in { 76 | val endpoints = toEndpoints(nodes) 77 | val loadbalancer = new TestLBF(6, false).newLoadBalancer(endpoints) 78 | val res = loadbalancer.nodesForPartitionedIds(Set(1,3,4,5), Some(0L), Some(0L)) 79 | res.values.flatten.toSet must be_==(Set(1,3,4,5)) 80 | 81 | markUnavailable(endpoints, 1) 82 | markUnavailable(endpoints, 3) 83 | markUnavailable(endpoints, 5) 84 | loadbalancer.nodesForPartitionedIds(Set(1,3,4,5), Some(0L), Some(0L)) must throwA[NoNodesAvailableException] 85 | } 86 | 87 | } 88 | } -------------------------------------------------------------------------------- /network/src/test/scala/com/linkedin/norbert/network/server/MessageHandlerRegistrySpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2009-2010 LinkedIn, Inc 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | * use this file except in compliance with the License. You may obtain a copy of 6 | * the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | * License for the specific language governing permissions and limitations under 14 | * the License. 15 | */ 16 | package com.linkedin.norbert 17 | package network 18 | package server 19 | 20 | import org.specs.Specification 21 | import org.specs.mock.Mockito 22 | import protos.NorbertExampleProtos 23 | import common.SampleMessage 24 | 25 | class MessageHandlerRegistrySpec extends Specification with Mockito with SampleMessage { 26 | val messageHandlerRegistry = new MessageHandlerRegistry 27 | 28 | var handled: Ping = _ 29 | val handler = (ping: Ping) => { 30 | handled = ping 31 | ping 32 | } 33 | 34 | 35 | "MessageHandlerRegistry" should { 36 | "return the handler for the specified request message" in { 37 | messageHandlerRegistry.registerHandler(handler) 38 | 39 | val h = messageHandlerRegistry.handlerFor[Ping, Ping](Ping.PingSerializer.requestName) 40 | 41 | h(request) must be_==(request) 42 | handled must be_==(request) 43 | } 44 | 45 | "throw an InvalidMessageException if no handler is registered" in { 46 | messageHandlerRegistry.handlerFor(Ping.PingSerializer.requestName) must throwA[InvalidMessageException] 47 | } 48 | 49 | "return true if the provided response is a valid response for the given request" in { 50 | messageHandlerRegistry.registerHandler(handler) 51 | // messageHandlerRegistry.validResponseFor(proto, NorbertExampleProtos.Ping.newBuilder.setTimestamp(System.currentTimeMillis).build) must beTrue 52 | } 53 | 54 | "return false if the provided response is not a valid response for the given request" in { 55 | messageHandlerRegistry.registerHandler(handler) 56 | // messageHandlerRegistry.validResponseFor(proto, mock[Message]) must beFalse 57 | } 58 | 59 | // "correctly handles null in validResponseFor" in { 60 | // messageHandlerRegistry.registerHandler(proto, null, handler) 61 | // messageHandlerRegistry.validResponseFor(proto, null) must beTrue 62 | // 63 | // messageHandlerRegistry.registerHandler(proto, proto, handler) 64 | // messageHandlerRegistry.validResponseFor(proto, null) must beFalse 65 | // } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /norbert/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | 3 | configurations { 4 | compile { 5 | transitive = false 6 | } 7 | } 8 | 9 | dependencies { 10 | compile project(':cluster') 11 | compile project(':network') 12 | compile project(':java-cluster') 13 | compile project(':java-network') 14 | } 15 | 16 | jar { 17 | from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } } 18 | } 19 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.10.1 -------------------------------------------------------------------------------- /sbt: -------------------------------------------------------------------------------- 1 | java $SBT_OPS -Xms1g -Xmx1g -server -XX:PermSize=128m -XX:MaxPermSize=256m -XX:+UseParallelOldGC -jar `dirname $0`/bin/sbt-launch.jar "$@" 2 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include "cluster" 2 | include "network" 3 | include "java-cluster" 4 | include "java-network" 5 | include "norbert" 6 | 7 | --------------------------------------------------------------------------------