├── .dockerignore ├── .gitignore ├── Dockerfile ├── build.sbt ├── mesh-agent └── src │ └── main │ ├── resources │ └── application.conf │ └── scala │ └── mesh │ ├── Consumer.scala │ ├── EtcdManager.scala │ ├── Provider.scala │ ├── RequestHandler.scala │ ├── Server.scala │ ├── proxy │ ├── DubboBackend.scala │ ├── DubboFrontend.scala │ └── DubboInitializer.scala │ └── utils │ ├── Bytes.scala │ └── DubboFlow.scala ├── project ├── build.properties └── plugins.sbt ├── readme.md ├── repositories └── start-agent.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### JetBrains template 3 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 4 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 5 | 6 | # User-specific stuff 7 | .idea/**/workspace.xml 8 | .idea/**/tasks.xml 9 | .idea/**/dictionaries 10 | .idea/**/shelf 11 | 12 | # Sensitive or high-churn files 13 | .idea/**/dataSources/ 14 | .idea/**/dataSources.ids 15 | .idea/**/dataSources.local.xml 16 | .idea/**/sqlDataSources.xml 17 | .idea/**/dynamic.xml 18 | .idea/**/uiDesigner.xml 19 | 20 | # Gradle 21 | .idea/**/gradle.xml 22 | .idea/**/libraries 23 | 24 | # CMake 25 | cmake-build-debug/ 26 | cmake-build-release/ 27 | 28 | # Mongo Explorer plugin 29 | .idea/**/mongoSettings.xml 30 | 31 | # File-based project format 32 | *.iws 33 | 34 | # IntelliJ 35 | out/ 36 | 37 | # mpeltonen/sbt-idea plugin 38 | .idea_modules/ 39 | 40 | # JIRA plugin 41 | atlassian-ide-plugin.xml 42 | 43 | # Cursive Clojure plugin 44 | .idea/replstate.xml 45 | 46 | # Crashlytics plugin (for Android Studio and IntelliJ) 47 | com_crashlytics_export_strings.xml 48 | crashlytics.properties 49 | crashlytics-build.properties 50 | fabric.properties 51 | 52 | # Editor-based Rest Client 53 | .idea/httpRequests 54 | ### SBT template 55 | # Simple Build Tool 56 | # http://www.scala-sbt.org/release/docs/Getting-Started/Directories.html#configuring-version-control 57 | 58 | dist/* 59 | target/ 60 | lib_managed/ 61 | src_managed/ 62 | project/boot/ 63 | project/plugins/project/ 64 | .history 65 | .cache 66 | .lib/ 67 | ### Scala template 68 | *.class 69 | *.log 70 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Scala template 3 | *.class 4 | *.log 5 | ### SBT template 6 | # Simple Build Tool 7 | # http://www.scala-sbt.org/release/docs/Getting-Started/Directories.html#configuring-version-control 8 | 9 | dist/* 10 | target/ 11 | lib_managed/ 12 | src_managed/ 13 | project/boot/ 14 | project/plugins/project/ 15 | .history 16 | .cache 17 | .lib/ 18 | ### JetBrains template 19 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 20 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 21 | 22 | # User-specific stuff 23 | .idea 24 | .idea/**/tasks.xml 25 | .idea/**/shelf 26 | 27 | # Sensitive or high-churn files 28 | .idea/**/dataSources/ 29 | .idea/**/dataSources.ids 30 | .idea/**/dataSources.local.xml 31 | .idea/**/sqlDataSources.xml 32 | .idea/**/dynamic.xml 33 | 34 | # Gradle 35 | .idea/**/gradle.xml 36 | 37 | # CMake 38 | cmake-build-debug/ 39 | cmake-build-release/ 40 | 41 | # Mongo Explorer plugin 42 | .idea/**/mongoSettings.xml 43 | 44 | # File-based project format 45 | *.iws 46 | 47 | # IntelliJ 48 | out/ 49 | 50 | # mpeltonen/sbt-idea plugin 51 | .idea_modules/ 52 | 53 | # JIRA plugin 54 | atlassian-ide-plugin.xml 55 | 56 | # Cursive Clojure plugin 57 | .idea/replstate.xml 58 | 59 | # Crashlytics plugin (for Android Studio and IntelliJ) 60 | com_crashlytics_export_strings.xml 61 | crashlytics.properties 62 | crashlytics-build.properties 63 | fabric.properties 64 | 65 | # Editor-based Rest Client 66 | .idea/httpRequests 67 | 68 | 将删除 .idea/ 69 | 将删除 project/project/ 70 | 将删除 project/target/ 71 | 将删除 src/test/ 72 | 将删除 target/ 73 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Builder container 2 | FROM registry.cn-hangzhou.aliyuncs.com/aliware2018/services AS builder 3 | 4 | # Install sbt and cache dependencies. 5 | RUN echo 'deb http://mirrors.aliyun.com/debian/ stretch main non-free contrib\n \ 6 | deb http://mirrors.aliyun.com/debian/ stretch-proposed-updates main non-free contrib\n \ 7 | deb-src http://mirrors.aliyun.com/debian/ stretch main non-free contrib\n \ 8 | deb-src http://mirrors.aliyun.com/debian/ stretch-proposed-updates main non-free contrib' | tee /etc/apt/sources.list 9 | RUN apt-get update -y && apt-get install -y apt-transport-https 10 | RUN echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list 11 | RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 12 | RUN apt-get update -y && apt-get install -y sbt 13 | RUN mkdir -p /root/.sbt 14 | COPY ./repositories /root/.sbt 15 | RUN git clone https://github.com/WayneWang12/akka-sample.git && cd akka-sample && sbt compile 16 | 17 | COPY . /root/workspace/agent 18 | WORKDIR /root/workspace/agent 19 | RUN set -ex && git show -s --format=%B && sbt clean compile assembly 20 | 21 | 22 | # Runner container 23 | FROM registry.cn-hangzhou.aliyuncs.com/aliware2018/debian-jdk8 24 | 25 | COPY --from=builder /root/workspace/services/mesh-provider/target/mesh-provider-1.0-SNAPSHOT.jar /root/dists/mesh-provider.jar 26 | COPY --from=builder /root/workspace/services/mesh-consumer/target/mesh-consumer-1.0-SNAPSHOT.jar /root/dists/mesh-consumer.jar 27 | COPY --from=builder /root/workspace/agent/mesh-agent/target/scala-2.12/mesh-agent-assembly-1.0-SNAPSHOT.jar /root/dists/mesh-agent.jar 28 | 29 | COPY --from=builder /usr/local/bin/docker-entrypoint.sh /usr/local/bin 30 | COPY ./start-agent.sh /usr/local/bin 31 | 32 | RUN set -ex && mkdir -p /root/logs && chmod 777 /usr/local/bin/start-agent.sh 33 | 34 | ENTRYPOINT ["docker-entrypoint.sh"] 35 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | 2 | val nettyVersion = "4.1.17.Final" 3 | 4 | lazy val `mesh-agent` = (project in file("mesh-agent")) 5 | .settings( 6 | name := "mesh-agent", 7 | version := "1.0-SNAPSHOT", 8 | scalaVersion := "2.12.6", 9 | mainClass in assembly := Some("mesh.Server"), 10 | assemblyMergeStrategy in assembly := { 11 | case PathList("META-INF", xs@_*) if xs.exists(_.endsWith(".so")) => 12 | MergeStrategy.first 13 | case PathList("META-INF", xs@_*) => 14 | MergeStrategy.discard 15 | case x if x.endsWith(".properties") => 16 | MergeStrategy.first 17 | case x if x.endsWith(".conf") => 18 | MergeStrategy.concat 19 | case x => 20 | MergeStrategy.deduplicate 21 | }, 22 | // assemblyJarName in assembly := "agent.jar", 23 | libraryDependencies ++= Seq( 24 | "com.alibaba" % "fastjson" % "1.2.46", 25 | "com.typesafe.akka" %% "akka-stream" % "2.5.12", 26 | "com.coreos" % "jetcd-core" % "0.0.2", 27 | "io.netty" % "netty-transport-native-epoll" % nettyVersion classifier "linux-x86_64" 28 | ) 29 | ) 30 | 31 | 32 | -------------------------------------------------------------------------------- /mesh-agent/src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | actor { 3 | default-dispatcher { 4 | 5 | executor = "default-executor" 6 | 7 | fork-join-executor { 8 | # Settings this to 1 instead of 3 seems to improve performance. 9 | parallelism-factor = 1.0 10 | parallelism-factor = ${?factor} 11 | parallelism-factor = ${?customer.factor} 12 | # default. 13 | parallelism-min = 4 14 | parallelism-max = 8 15 | 16 | task-peeking-mode = FIFO 17 | } 18 | 19 | } 20 | } 21 | } 22 | 23 | mpsc { 24 | executor = "thread-pool-executor" 25 | type = PinnedDispatcher 26 | mailbox-type = "akka.dispatch.SingleConsumerOnlyUnboundedMailbox" 27 | } 28 | -------------------------------------------------------------------------------- /mesh-agent/src/main/scala/mesh/Consumer.scala: -------------------------------------------------------------------------------- 1 | package mesh 2 | 3 | import java.net.InetSocketAddress 4 | 5 | import akka.actor.{Actor, ActorLogging, ActorRef, Props} 6 | import akka.io.Tcp._ 7 | import akka.io.{IO, Tcp} 8 | import akka.stream.Materializer 9 | 10 | class Consumer(host:String, etcdManager: ActorRef)(implicit materializer: Materializer) extends Actor with ActorLogging { 11 | 12 | import context.system 13 | 14 | IO(Tcp) ! Bind(self, new InetSocketAddress("127.0.0.1", 20000)) 15 | 16 | val requestHandler: ActorRef = context.actorOf(Props(new RequestHandler).withDispatcher("mpsc"), "request-handler") 17 | 18 | def receive: Receive = { 19 | case Bound(localAddress) ⇒ 20 | etcdManager ! "consumer" 21 | log.info(s"service started at ${localAddress.getHostString}:${localAddress.getPort}") 22 | case CommandFailed(_: Bind) ⇒ 23 | context stop self 24 | case Connected(_, _) ⇒ 25 | val connection = sender() 26 | connection ! Register(requestHandler) 27 | } 28 | } 29 | 30 | -------------------------------------------------------------------------------- /mesh-agent/src/main/scala/mesh/EtcdManager.scala: -------------------------------------------------------------------------------- 1 | package mesh 2 | 3 | import java.net.InetAddress 4 | import java.text.MessageFormat 5 | 6 | import akka.actor.{Actor, ActorLogging, ActorSystem} 7 | import akka.event.Logging 8 | import com.coreos.jetcd.Client 9 | import com.coreos.jetcd.data.ByteSequence 10 | import com.coreos.jetcd.kv.GetResponse 11 | import com.coreos.jetcd.options.{GetOption, PutOption} 12 | import mesh.ProviderScale.ProviderScale 13 | 14 | class EtcdHandler(etcdUrl: String)(implicit actorSystem: ActorSystem) { 15 | 16 | private val log = Logging(actorSystem, this.getClass) 17 | 18 | log.info(s"get etcd uri $etcdUrl") 19 | 20 | val client: Client = Client.builder().endpoints(etcdUrl).build() 21 | 22 | val kvClient = client.getKVClient 23 | val lease = client.getLeaseClient 24 | var leaseId: Long = _ 25 | 26 | val rootPath = "dubbomesh" 27 | val serviceName = "com.alibaba.dubbo.performance.demo.provider.IHelloService" 28 | 29 | try { 30 | val id = lease.grant(30).get.getID 31 | leaseId = id 32 | } catch { 33 | case e: Throwable => e.printStackTrace() 34 | } 35 | 36 | def keepAlive(): Unit = { 37 | try { 38 | val listener = lease.keepAlive(leaseId) 39 | listener.listen 40 | log.info("KeepAlive lease:" + leaseId + "; Hex format:" + leaseId.toHexString) 41 | } catch { 42 | case e: Exception => 43 | e.printStackTrace() 44 | } 45 | } 46 | 47 | def register(serviceName: String, port: Int, scale: ProviderScale): Unit = { // 服务注册的key为: /dubbomesh/com.some.package.IHelloService/192.168.100.100:2000 48 | val strKey = MessageFormat.format("/{0}/{1}/{2}:{3}", rootPath, serviceName, IpHelper.getHostIp, String.valueOf(port)) 49 | 50 | val key = ByteSequence.fromString(strKey) 51 | val value = ByteSequence.fromString(scale.toString) 52 | kvClient.put(key, value, PutOption.newBuilder.withLeaseId(leaseId).build).get() 53 | log.info("Register a new service at:" + strKey) 54 | } 55 | 56 | def find(serviceName: String): Set[Endpoint] = { 57 | val strKey: String = MessageFormat.format("/{0}/{1}", rootPath, serviceName) 58 | val key: ByteSequence = ByteSequence.fromString(strKey) 59 | val response: GetResponse = kvClient.get(key, GetOption.newBuilder.withPrefix(key).build).get 60 | 61 | import scala.collection.JavaConverters._ 62 | val ed = for (kv <- response.getKvs.asScala) yield { 63 | val scale = kv.getValue.toStringUtf8 64 | val s: String = kv.getKey.toStringUtf8 65 | val index: Int = s.lastIndexOf("/") 66 | val endpointStr: String = s.substring(index + 1, s.length) 67 | val host: String = endpointStr.split(":")(0) 68 | val port: Int = Integer.valueOf(endpointStr.split(":")(1)) 69 | Endpoint(host, port, ProviderScale.withName(scale)) 70 | } 71 | ed.toSet 72 | } 73 | 74 | 75 | } 76 | 77 | class EtcdManager(etcdHandler: EtcdHandler, serverPort: Int) extends Actor with ActorLogging { 78 | 79 | import etcdHandler._ 80 | 81 | var endpoints = Set.empty[Endpoint] 82 | 83 | override def receive: Receive = { 84 | case "consumer" => 85 | val found = find(serviceName) 86 | if (found != endpoints) { 87 | endpoints = found 88 | log.info(s"found new endpoints $found") 89 | context.system.eventStream.publish(EndpointsUpdate(endpoints)) 90 | client.close() 91 | context stop self 92 | } 93 | } 94 | 95 | } 96 | 97 | object ProviderScale extends Enumeration { 98 | type ProviderScale = Value 99 | val Small, Medium, Large = Value 100 | } 101 | 102 | case class Endpoint(host: String, port: Int, scale: ProviderScale) 103 | 104 | case class EndpointsUpdate(endpoints: Set[Endpoint]) 105 | 106 | object IpHelper { 107 | def getHostIp: String = { 108 | val ip = InetAddress.getLocalHost.getHostAddress 109 | ip 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /mesh-agent/src/main/scala/mesh/Provider.scala: -------------------------------------------------------------------------------- 1 | package mesh 2 | 3 | import akka.actor.ActorSystem 4 | import akka.event.Logging 5 | import io.netty.bootstrap.ServerBootstrap 6 | import io.netty.buffer.PooledByteBufAllocator 7 | import io.netty.channel.epoll.{EpollEventLoopGroup, EpollServerSocketChannel} 8 | import io.netty.channel.{ChannelOption, EventLoopGroup} 9 | import mesh.ProviderScale.ProviderScale 10 | import mesh.proxy.DubboInitializer 11 | 12 | import scala.util.control.NonFatal 13 | 14 | class Provider(localhost: String, port: Int, dubboPort: Int, scale: ProviderScale)(implicit actorSystem: ActorSystem) { 15 | 16 | val log = Logging(actorSystem, this.getClass) 17 | 18 | def startService: Unit = { 19 | // Configure the bootstrap. 20 | val bossGroup: EventLoopGroup = new EpollEventLoopGroup(1) 21 | val workerGroup: EventLoopGroup = new EpollEventLoopGroup(4) 22 | try { 23 | val b = new ServerBootstrap() 24 | val c = b.group(bossGroup, workerGroup).channel(classOf[EpollServerSocketChannel]) 25 | .childHandler(new DubboInitializer(localhost, dubboPort)) 26 | .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) 27 | .childOption(ChannelOption.AUTO_READ, java.lang.Boolean.FALSE) 28 | .bind(localhost, port) 29 | .sync() 30 | val bounded = c.channel().localAddress() 31 | if(bounded != null) { 32 | log.info(s"provider with scale ${scale.toString} started on $bounded ") 33 | } else { 34 | log.info("provider failed to start.") 35 | } 36 | c.channel().closeFuture().sync() 37 | } catch { 38 | case NonFatal(t) => 39 | t.printStackTrace() 40 | case e: Exception => 41 | e.printStackTrace() 42 | } finally { 43 | bossGroup.shutdownGracefully() 44 | workerGroup.shutdownGracefully() 45 | } 46 | } 47 | 48 | } 49 | 50 | -------------------------------------------------------------------------------- /mesh-agent/src/main/scala/mesh/RequestHandler.scala: -------------------------------------------------------------------------------- 1 | package mesh 2 | 3 | import java.nio.ByteOrder 4 | 5 | import akka.NotUsed 6 | import akka.actor.{Actor, ActorLogging} 7 | import akka.io.Tcp.Received 8 | import akka.stream._ 9 | import akka.stream.scaladsl.{Flow, Framing, GraphDSL, Merge, Partition, Source, SourceQueueWithComplete, Tcp} 10 | import akka.util.ByteString 11 | import mesh.utils.DubboFlow 12 | 13 | import scala.util.Random 14 | 15 | class RequestHandler(implicit materializer: Materializer) extends Actor with ActorLogging { 16 | 17 | import context.{dispatcher, system} 18 | 19 | context.system.eventStream.subscribe(self, classOf[EndpointsUpdate]) 20 | 21 | def proportionalEndpoints(endpoints: List[(Endpoint, Int)]): Flow[ByteString, ByteString, NotUsed] = { 22 | Flow.fromGraph(GraphDSL.create() { implicit builder => 23 | import GraphDSL.Implicits._ 24 | val sum = endpoints.size 25 | 26 | val list = endpoints.map(_._2) 27 | val listUpperBound = list.sum 28 | val ranges = list.scanLeft(0)(_ + _).sliding(2, 1).toList.map(t => (t.head, t(1))) 29 | val routeStrategy = (bs: ByteString) => { 30 | val x = Random.nextInt(listUpperBound) 31 | ranges.indexWhere(t => x >= t._1 && x < t._2) 32 | } 33 | 34 | val merge = builder.add(Merge[ByteString](sum)) 35 | val balancer = builder.add(Partition[ByteString](sum, routeStrategy)) 36 | 37 | val framing = Framing.lengthField(4, 12, Int.MaxValue, ByteOrder.BIG_ENDIAN) 38 | 39 | endpoints.foreach { 40 | case (ed, _) => 41 | val tcp = Flow[ByteString] 42 | .buffer(200, OverflowStrategy.backpressure) 43 | .via(Tcp().outgoingConnection(ed.host, ed.port)) 44 | balancer ~> tcp.async ~> framing.async ~> merge 45 | } 46 | FlowShape(balancer.in, merge.out) 47 | }) 48 | } 49 | 50 | def endpointsFlow(endpoints: Set[Endpoint]) = { 51 | val tcpFlows = endpoints.toList.map { endpoint => 52 | endpoint.scale match { 53 | case ProviderScale.Small => 54 | (endpoint, 1) 55 | case ProviderScale.Medium => 56 | (endpoint, 2) 57 | case ProviderScale.Large => 58 | (endpoint, 8) 59 | } 60 | } 61 | 62 | proportionalEndpoints(tcpFlows.filter(_._2 > 0)) 63 | } 64 | 65 | def getSourceByEndpoints(endpoints: Set[Endpoint]): SourceQueueWithComplete[(Long, ByteString)] = { 66 | val handleFlow = Flow[(Long, ByteString)] 67 | .via(DubboFlow.connectionIdFlow) 68 | .via(endpointsFlow(endpoints)) 69 | .to(DubboFlow.decoder) 70 | Source.queue[(Long, ByteString)](512, OverflowStrategy.backpressure) 71 | .to(handleFlow).run() 72 | } 73 | 74 | var source: SourceQueueWithComplete[(Long, ByteString)] = _ 75 | 76 | override def receive: Receive = { 77 | case Received(bs) => 78 | source.offer(sender().path.name.toLong, bs) 79 | case EndpointsUpdate(newEndpoints) => 80 | log.info(s"start new source for endpoints $newEndpoints") 81 | source.complete() 82 | source = getSourceByEndpoints(newEndpoints) 83 | case _ => 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /mesh-agent/src/main/scala/mesh/Server.scala: -------------------------------------------------------------------------------- 1 | package mesh 2 | 3 | import akka.actor.{ActorSystem, Props} 4 | import akka.stream.ActorMaterializer 5 | import com.typesafe.config.ConfigFactory 6 | 7 | object Server extends App { 8 | 9 | val config = ConfigFactory.load() 10 | 11 | val serviceType = config.getString("type") 12 | val serverPort = config.getInt("server.port") 13 | 14 | implicit val actorSystem: ActorSystem = ActorSystem("dubbo-mesh") 15 | implicit val materializer: ActorMaterializer = ActorMaterializer() 16 | 17 | val hostIp = IpHelper.getHostIp 18 | 19 | val etcdHost = config.getString("etcd.url") 20 | 21 | val etcdHandler = new EtcdHandler(etcdHost) 22 | 23 | serviceType match { 24 | case "consumer" => 25 | val etcdManager = actorSystem.actorOf(Props( 26 | new EtcdManager(etcdHandler, serverPort) 27 | )) 28 | actorSystem.actorOf(Props(new Consumer(hostIp, etcdManager)), "consumer-agent") 29 | case "provider" => 30 | val dubboPort = config.getInt("dubbo.protocol.port") 31 | val scale = config.getString("scale") 32 | val providerScale = ProviderScale.withName(scale) 33 | etcdHandler.register(etcdHandler.serviceName, serverPort, providerScale) 34 | etcdHandler.keepAlive() 35 | val provider = new Provider(hostIp, serverPort, dubboPort, providerScale) 36 | provider.startService 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /mesh-agent/src/main/scala/mesh/proxy/DubboBackend.scala: -------------------------------------------------------------------------------- 1 | package mesh.proxy 2 | 3 | import io.netty.channel.{Channel, ChannelFuture, ChannelHandlerContext, ChannelInboundHandlerAdapter} 4 | 5 | class DubboBackend(val inboundChannel: Channel) extends ChannelInboundHandlerAdapter { 6 | override def channelActive(ctx: ChannelHandlerContext): Unit = { 7 | ctx.read 8 | } 9 | 10 | override def channelRead(ctx: ChannelHandlerContext, msg: Any): Unit = { 11 | inboundChannel.writeAndFlush(msg).addListener((future: ChannelFuture) => { 12 | if (future.isSuccess) ctx.channel.read 13 | else future.channel.close 14 | }) 15 | } 16 | 17 | override def channelInactive(ctx: ChannelHandlerContext): Unit = { 18 | } 19 | 20 | override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable): Unit = { 21 | cause.printStackTrace() 22 | // HexDumpProxyFrontendHandler.closeOnFlush(ctx.channel) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /mesh-agent/src/main/scala/mesh/proxy/DubboFrontend.scala: -------------------------------------------------------------------------------- 1 | package mesh.proxy 2 | 3 | import io.netty.bootstrap.Bootstrap 4 | import io.netty.buffer.Unpooled 5 | import io.netty.channel._ 6 | 7 | class DubboFrontend(val remoteHost: String, val remotePort: Int) extends ChannelInboundHandlerAdapter { 8 | // As we use inboundChannel.eventLoop() when building the Bootstrap this does not need to be volatile as 9 | // the outboundChannel will use the same EventLoop (and therefore Thread) as the inboundChannel. 10 | private var outboundChannel: Channel = _ 11 | 12 | override def channelActive(ctx: ChannelHandlerContext): Unit = { 13 | val inboundChannel = ctx.channel 14 | // Start the connection attempt. 15 | val b = new Bootstrap() 16 | b.group(inboundChannel.eventLoop).channel(ctx.channel.getClass) 17 | .handler(new DubboBackend(inboundChannel)) 18 | .option(ChannelOption.AUTO_READ, java.lang.Boolean.FALSE) 19 | val f = b.connect(remoteHost, remotePort) 20 | outboundChannel = f.channel 21 | f.addListener((future: ChannelFuture) => { 22 | if (future.isSuccess) { // connection complete start to read first data 23 | inboundChannel.read 24 | } 25 | else { // Close the connection if the connection attempt has failed. 26 | inboundChannel.close 27 | } 28 | }) 29 | } 30 | 31 | override def channelRead(ctx: ChannelHandlerContext, msg: Any): Unit = { 32 | if (outboundChannel.isActive) 33 | outboundChannel.writeAndFlush(msg).addListener((future: ChannelFuture) => { 34 | if (future.isSuccess) { // was able to flush out data, start to read the next chunk 35 | ctx.channel.read 36 | } 37 | else future.channel.close 38 | }) 39 | } 40 | 41 | override def channelInactive(ctx: ChannelHandlerContext): Unit = { 42 | if (outboundChannel != null) closeOnFlush(outboundChannel) 43 | } 44 | 45 | override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable): Unit = { 46 | cause.printStackTrace() 47 | closeOnFlush(ctx.channel) 48 | } 49 | 50 | 51 | def closeOnFlush(ch: Channel): Unit = { 52 | if (ch.isActive) ch.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /mesh-agent/src/main/scala/mesh/proxy/DubboInitializer.scala: -------------------------------------------------------------------------------- 1 | package mesh.proxy 2 | 3 | import io.netty.channel.ChannelInitializer 4 | import io.netty.channel.socket.SocketChannel 5 | 6 | 7 | class DubboInitializer(val remoteHost: String, val remotePort: Int) extends ChannelInitializer[SocketChannel] { 8 | override def initChannel(ch: SocketChannel): Unit = { 9 | ch.pipeline 10 | .addLast("proxy", new DubboFrontend(remoteHost, remotePort)) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /mesh-agent/src/main/scala/mesh/utils/Bytes.scala: -------------------------------------------------------------------------------- 1 | package mesh.utils 2 | 3 | import akka.util.ByteString 4 | 5 | /** 6 | * CodecUtils. 7 | */ 8 | object Bytes { 9 | /** 10 | * to byte array. 11 | * 12 | * @param v value. 13 | * @param b byte array. 14 | */ 15 | def short2bytes(v: Short, b: Array[Byte]): Unit = { 16 | short2bytes(v, b, 0) 17 | } 18 | 19 | def short2bytes(v: Short, b: Array[Byte], off: Int): Unit = { 20 | b(off + 1) = v.toByte 21 | b(off + 0) = (v >>> 8).toByte 22 | } 23 | 24 | /** 25 | * to byte array. 26 | * 27 | * @param v value. 28 | * @param b byte array. 29 | * @param off array offset. 30 | */ 31 | def int2bytes(v: Int, b: Array[Byte], off: Int): Unit = { 32 | b(off + 3) = v.toByte 33 | b(off + 2) = (v >>> 8).toByte 34 | b(off + 1) = (v >>> 16).toByte 35 | b(off + 0) = (v >>> 24).toByte 36 | } 37 | 38 | def long2bytes(v: Long, b: Array[Byte], off: Int): Unit = { 39 | b(off + 7) = v.toByte 40 | b(off + 6) = (v >>> 8).toByte 41 | b(off + 5) = (v >>> 16).toByte 42 | b(off + 4) = (v >>> 24).toByte 43 | b(off + 3) = (v >>> 32).toByte 44 | b(off + 2) = (v >>> 40).toByte 45 | b(off + 1) = (v >>> 48).toByte 46 | b(off + 0) = (v >>> 56).toByte 47 | } 48 | 49 | /** 50 | * to long. 51 | * 52 | * @param b byte array. 53 | * @param off offset. 54 | * @return long. 55 | */ 56 | def bytes2long(b: ByteString, off: Int): Long = ((b(off + 7) & 0xFFL) << 0) + ((b(off + 6) & 0xFFL) << 8) + ((b(off + 5) & 0xFFL) << 16) + ((b(off + 4) & 0xFFL) << 24) + ((b(off + 3) & 0xFFL) << 32) + ((b(off + 2) & 0xFFL) << 40) + ((b(off + 1) & 0xFFL) << 48) + (b(off + 0).toLong << 56) 57 | } 58 | 59 | -------------------------------------------------------------------------------- /mesh-agent/src/main/scala/mesh/utils/DubboFlow.scala: -------------------------------------------------------------------------------- 1 | package mesh.utils 2 | 3 | import akka.NotUsed 4 | import akka.actor.ActorSystem 5 | import akka.io.Tcp.Write 6 | import akka.stream.OverflowStrategy 7 | import akka.stream.scaladsl.{Flow, Sink} 8 | import akka.util.ByteString 9 | 10 | import scala.collection.mutable 11 | import scala.concurrent.ExecutionContext 12 | 13 | object DubboFlow { 14 | 15 | private val HEADER_LENGTH = 16 16 | 17 | private val MAGIC = 0xdabb.toShort 18 | private val FLAG_REQUEST = 0x80.toByte 19 | private val FLAG_TWOWAY = 0x40.toByte 20 | private val FLAG_EVENT = 0x20.toByte 21 | private val dubboVersion = ByteString("\"2.6.0\"\n") 22 | private val interface = ByteString("\"com.alibaba.dubbo.performance.demo.provider.IHelloService\"\n") 23 | private val method = ByteString("\"hash\"\n") 24 | private val version = ByteString("\"0.0.0\"\n") 25 | private val pType = ByteString("\"Ljava/lang/String;\"\n") 26 | private val dubboEnd = ByteString("{\"empty\":false,\"traversableAgain\":true}\n") 27 | val slicer = "parameter=".map(_.toByte) 28 | val quote = ByteString('\"') 29 | val QuoteAndCarriageReturn = ByteString("\"\r\n") 30 | val HttpOkStatus = ByteString("HTTP/1.1 200 OK\r\n") 31 | val KeepAlive = ByteString("Connection: Keep-Alive\r\n") 32 | val ContentType = ByteString("Content-Type: application/octet-stream\r\n") 33 | val HeaderDelimter = ByteString("\r\n") 34 | val SlashN = ByteString("\n") 35 | 36 | val dubboHeader = { 37 | val array = new Array[Byte](4) 38 | Bytes.short2bytes(MAGIC, array) 39 | array(2) = (FLAG_REQUEST | 6).toByte 40 | array(2) = (array(2) | FLAG_TWOWAY).toByte 41 | ByteString.fromArrayUnsafe(array) 42 | } 43 | 44 | def map2DubboByteString(requestId: Long, parameter: ByteString): ByteString = { 45 | val idAndLength = new Array[Byte](12) 46 | Bytes.long2bytes(requestId, idAndLength, 0) 47 | val data = dubboVersion ++ interface ++ version ++ method ++ pType ++ parameter ++ dubboEnd 48 | Bytes.int2bytes(data.size, idAndLength, 8) 49 | dubboHeader ++ ByteString.fromArrayUnsafe(idAndLength) ++ data 50 | } 51 | 52 | 53 | def cLength(length: Int) = { 54 | ByteString(s"Content-Length: $length\r\n") 55 | } 56 | 57 | def connectionIdFlow(implicit executionContext: ExecutionContext): Flow[(Long, ByteString), ByteString, NotUsed] = 58 | Flow[(Long, ByteString)].map { 59 | case (cid, bs) => 60 | http2DubboByteString(cid, bs) 61 | } 62 | 63 | def http2DubboByteString(cid: Long, bs: ByteString): ByteString = { 64 | val n = bs.indexOfSlice(slicer) 65 | val s = bs.drop(n + slicer.size) 66 | val d = quote ++ s ++ QuoteAndCarriageReturn 67 | val data = map2DubboByteString(cid, d) 68 | data 69 | } 70 | 71 | val headers = HttpOkStatus ++ KeepAlive ++ ContentType 72 | 73 | def decoder(implicit actorSystem: ActorSystem) = { 74 | Sink.foreach[ByteString] { 75 | val map = mutable.Map.empty[Int, ByteString] 76 | bs => 77 | val cid = Bytes.bytes2long(bs, 4) 78 | val data = bs.slice(18, bs.size - 1) 79 | val resp = headers ++ map.getOrElseUpdate(data.size, cLength(data.size)) ++ HeaderDelimter ++ data 80 | val actor = actorSystem.actorSelection(s"/system/IO-TCP/selectors/$$a/$cid") 81 | actor ! Write(resp) 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.1.5 -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6") 2 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # 第四届阿里中间件性能大赛初赛攻略——RDP飞起来 2 | 3 | ## 赛题背景分析及理解 4 | 5 | 初赛题目是吸引我参加比赛的最大原因。其中一段描述了Service Mesh的作用: 6 | 7 | > 作为 Service Mesh 的核心组件之一，高性能的服务间代理（Agent）是不可或缺的，其可以做到请求转发、协议转换、服务注册与发现、动态路由、负载均衡、流量控制、降级和熔断等诸多功能，也是区别于传统微服务架构的重要特征。 8 | 9 | 而这种思想与《反应式设计模式》不约而同。在反应式系统设计的过程中，很重要的一块就是如何与现存的非反应式系统进行交互。非反应式系统典型地都具有同步阻塞调用者、无界输入队列、不遵循有界响应延迟的原则等缺点，这使得流量控制、资源高效利用以及降级、熔断等功能都比较难以实现。《反应式设计模式》一书中专门推荐了要使用单独的资源来与这些系统整合，并赋予他们“反应式”的假象。而Service Mesh中的Agent，则可以看作成专门用来与非反应式系统进行整合的组件。在第14章的资源管理模式中，描述了如何使用这样的资源与之进行交互的方法，尤其是托管阻塞模式；而第16章的流量控制模式，则指导了我们如何在调用过程中行之有效进行流量控制。当然，对于比赛来说，这些设计相对来说过于概括。不过我们可以先基于这种概括性地原则构建出体系架构来，之后我们再具体优化相关的细节，提高成绩。而基于之前描述的原因，我的第一版使用了Akka来进行开发。接下来我们先分析一下具体的题目。 10 | 11 | ### 题目分析 12 | 13 | 题目的要求是： 14 | 15 | >实现一个高性能的 Service Mesh Agent 组件，并包含如下一些功能： 16 | > 1. 服务注册与发现 17 | > 2. 协议转换 18 | > 3. 负载均衡 19 | 20 | 服务注册与发现是为了获得资源的访问方式。这个过程最好不要与正式的调用过程耦合。所以我们用一个单独的Actor来做服务发现。如果是在Consumer中，这个Actor会去监听ETCD的变更，如果发现Endpoints发生了变化，则将信息发布到ActorSystem的事件流中。之后关注`EndpointsUpdated`事件的Actor就会收到此消息，并根据它来更新自己的端点列表，进行负载的动态变更。 21 | 22 | 协议转换相对来说是一个打铁的活，根据Dubbo协议一点点写好就行了。 23 | 24 | 重要的则在于负载均衡。进一步回到题目描述中： 25 | 26 | > 1. 每轮评测在一台服务器中启动五个服务（以 Docker 实例的形式启动），一个 etcd 服务作为注册表、一个 Consumer 服务和三个 Provider 服务； 27 | > 2. 使用一台独立的施压服务器，分不同压力场景对 Consumer 服务进行压力测试，得到 QPS； 28 | 29 | 总共有3轮压力测试，分别是128、256、512个连接。由于每次请求的往返时间最少也是50ms，那么每秒钟，按照512连接的最大速度，则是1000 / 50 * 512 = 10240的最大QPS。 30 | 31 | 其中，三台Provider的负载能力有所不同，按照CPU的quota分配以及内存的大小分配，正常情况下应该是1比2比3。只是由于Provider的dubbo端最多同时只能处理200个请求，多出来的直接被reject掉。那么最好的分配比例在512条件下则是 `112 : 200 : 200`。 32 | 33 | 当然，反应式系统的设计原则并不是固定分配比例的。它希望的理想情形是`你先告诉我你能处理多少任务，一旦任务来了，我就尽量按照这个数量发给你`。不要Consumer去强行推，不要Provider一直来拉。而这种模式最好的实现方式，就是利用Akka Stream啦。 34 | 35 | ## 核心思路 36 | 37 | 按照前面的分析，核心思路就是将每个Provider的处理过程看作是一条流。来自调用端的所有请求先汇聚到一个队列里面，之后根据后端Provider的处理能力，分别分配到三个不同的流中。而如果汇聚队列的长度达到了界限值，则降级服务，对外部请求进行按比例丢弃，直到与系统的处理能力重新匹配（详情参见《反应式设计模式》第十六章丢弃模式）。这样整个系统就又健壮又迅速。 38 | 39 | ## 关键代码 40 | 41 | 下面一段是用来抽象Consumer的Actor里面的代码，所有连接的请求都被注册到RequestHandler这个Actor了。 42 | 43 | ```scala 44 | val requestHandler: ActorRef = context.actorOf(Props(new RequestHandler).withDispatcher("mpsc"), "request-handler") 45 | 46 | def receive: Receive = { 47 | case Bound(localAddress) ⇒ 48 | etcdManager ! "consumer" 49 | log.info(s"service started at ${localAddress.getHostString}:${localAddress.getPort}") 50 | case CommandFailed(_: Bind) ⇒ 51 | context stop self 52 | case Connected(_, _) ⇒ 53 | val connection = sender() 54 | connection ! Register(requestHandler) 55 | //将Connection全部注册到RequestHandler，就是说所有连接发过来的数据都回转发到这个Actor 56 | //注意这个是Hack写法。正统的还是应该一个Actor一个连接，这样逻辑才会清晰。 57 | } 58 | ``` 59 | 60 | 然后在`Requesthandler`里面，接收到的`ByteString`直接作为元素提供给后面的处理流代码里面。 61 | 62 | ```scala 63 | var source: SourceQueueWithComplete[(Long, ByteString)] = _ 64 | 65 | override def receive: Receive = { 66 | case Received(bs) => 67 | source.offer(sender().path.name.toLong, bs) //这里的sender是处理连接的actor，它们的名字刚刚好是ID，所以直接复用。 68 | case EndpointsUpdate(newEndpoints) => 69 | log.info(s"start new source for endpoints $newEndpoints") 70 | source.complete() 71 | source = getSourceByEndpoints(newEndpoints) 72 | ``` 73 | 74 | 这里的`source`是一个可完成`Source`。`Source`, `Flow`, `Sink`是Akka Stream里面的基本构建块。其大体意义如下： 75 | 76 | 1. Source: 只有一个输出流的构件块; 77 | 2. Sink: 只接收一个输入流的构件块; 78 | 3. Flow: 接收一个输入流，并拥有一个输出流的构件块。 79 | 4. Graph: 一个打包好的流处理拓扑，它可以拥有一组输入端口或者输出端口。 80 | 81 | 我们这里是一个可完成`Source`，它由`Source.queue`声明并物化后产生： 82 | 83 | ```scala 84 | def getSourceByEndpoints(endpoints: Set[Endpoint]): SourceQueueWithComplete[(Long, ByteString)] = { 85 | val handleFlow = Flow[(Long, ByteString)] 86 | .via(DubboFlow.connectionIdFlow) 87 | .via(endpointsFlow(endpoints)) 88 | .to(DubboFlow.decoder) 89 | 90 | Source.queue[(Long, ByteString)](512, OverflowStrategy.backpressure) 91 | .to(handleFlow).run() 92 | } 93 | ``` 94 | 95 | 这里是由这个函数基于`Endpoint`的个数构建。第一段`handleFlow`是构建了一个`Flow`，这个`Flow`可以接收一个二元组`(Long, ByteString)`，并将其交给`DubboFlow.connectionIdFlow`来encode成自定义协议，之后将其发送到`endpointsFlow`进行对Provider的调用，并得到结果。得到结果之后，经由`DubboFlow.decoder`来decode，并发送回给各个连接Actor，由其返回给客户端。 96 | 97 | 上面的内容里面，`DubboFlow.connectionIdFlow`和`DubboFlow.decoder`不多说，都是打铁代码。核心逻辑`endpointsFlow(endPoints)`贴出如下： 98 | 99 | ```scala 100 | def endpointsFlow(endpoints: List[Endpoint]) = { 101 | 102 | val tcpFlows = endpoints.map { endpoint => 103 | Tcp().outgoingConnection(endpoint.host, endpoint.port).async 104 | } 105 | 106 | val framing = Framing.lengthField(4, 12, Int.MaxValue, ByteOrder.BIG_ENDIAN) 107 | 108 | Flow.fromGraph(GraphDSL.create(tcpFlows) { implicit builder => 109 | tcpFlows => 110 | import GraphDSL.Implicits._ 111 | 112 | val balance = builder.add(Balance[ByteString](tcpFlows.size)) 113 | val bigMerge = builder.add(Merge[ByteString](tcpFlows.size)) 114 | tcpFlows.foreach { tcp => 115 | balance ~> tcp ~> framing ~> bigMerge 116 | } 117 | 118 | FlowShape(balance.in, bigMerge.out) 119 | }) 120 | } 121 | ``` 122 | 123 | 每一个`endpoint`都被映射成为一个`Tcp`的`Flow`，通往Provider端。之后使用Akka Stream的DSL方法，构建了一个`Graph`。这个`Graph`用图形表示，其拓扑结果则是如下： 124 | 125 | ``` 126 | +------> Small Provider +--------> Framing +-------+ 127 | | | 128 | Input | | Output 129 | +--------> Balancer ---------> Medium Provider+--------> Framing +----------------> Merge +--------> 130 | | | 131 | | | 132 | +------> Large Provider +--------> Framing +-------+ 133 | ``` 134 | 135 | 数据由左边输入，经过Balancer，这个Balancer是由Akka Stream提供的现成组件，它可以将上游的元素路由到下游，其特性如下： 136 | 137 | 1. 一个`Balance`由一个`in`端口和2到多个`out`端口， 138 | 2. 当任意下游端口停止回压之后，它输出元素到下游输出端口； 139 | 3. 当下游所有端口都在回压的时候，它就回压上游； 140 | 4. 当上游完成时，它也完成； 141 | 5. 当其`eagerCancel`参数设置为`true`时，任意下游取消，则其也取消；设置为`false`的时候，当所有下游取消，它才取消。 142 | 143 | 由上面的拓扑结构可以看到，当任意Provider向上游表示可以处理请求的时候，Balancer就会在有请求到来的时候，向其输出；Provider处理完的请求，经过TCP拆包过程之后，就合并到一起，交由下游的流继续处理。如此，只要连接有请求过来，那么整个流就能一直运转。这个过程中，即使某个通往provider的连接断掉了，Balancer也能继续将请求路由到其他两个连接上。而这个时候，负责服务发现的Actor就会发出`EndpointsUpdated`的消息，此时`RequestHandler`会进入第二个匹配，用新的Endpoint来更新我们的处理流： 144 | 145 | ```scala 146 | case EndpointsUpdate(newEndpoints) => 147 | log.info(s"start new source for endpoints $newEndpoints") 148 | source.complete() 149 | source = getSourceByEndpoints(newEndpoints) 150 | ``` 151 | 152 | 注意这里的`complete`是表示流不再接收新的请求，这之前已经入队的请求仍然会继续完成，直到全部处理完毕。 153 | 154 | Provider的代码相对Consumer就简单很多： 155 | 156 | ```scala 157 | val handleFlow = Tcp().outgoingConnection(host, dubboPort).async 158 | 159 | def startService: Future[Done] = { 160 | Tcp().bind(host, port).runForeach { conn => 161 | conn.handleWith(handleFlow) 162 | } 163 | } 164 | ``` 165 | 166 | 它只需要将Consumer过来的连接转发给后端的Dubbo，或者为了性能原因，它需要将自定义协议包装成Dubbo协议，然后发过去，再将结果转回，即可。 167 | 168 | 到这里，我们用了大约不到300行代码，就完成了初赛题目的所有要求。并且代码的普适性和健壮性都很不错，后续还能依据需求，快速地实现任意一端的限流要求(`Flow[Request].throttle(...)`)，或者加入断路器，进行快速失败。 169 | 170 | 这套代码在CPU资源充足的时候，例如在我本地(注意，已经按照docker参数限定了CPU quota和内存)，256连接的时候可以跑4960, 512的时候可以跑9500。 171 | 172 | 然而线上则表现不好，分别最多4500和6400。这是为什么呢？ 173 | 174 | 经过查询源码以后发现，问题出现在这一段： 175 | 176 | ```scala 177 | @tailrec def innerRead(buffer: ByteBuffer, remainingLimit: Int): ReadResult = 178 | if (remainingLimit > 0) { 179 | // never read more than the configured limit 180 | buffer.clear() 181 | val maxBufferSpace = math.min(DirectBufferSize, remainingLimit) 182 | buffer.limit(maxBufferSpace) 183 | val readBytes = channel.read(buffer) 184 | buffer.flip() 185 | 186 | if (TraceLogging) log.debug("Read [{}] bytes.", readBytes) 187 | if (readBytes > 0) info.handler ! Received(ByteString(buffer)) //这一段 188 | 189 | readBytes match { 190 | case `maxBufferSpace` ⇒ if (pullMode) MoreDataWaiting else innerRead(buffer, remainingLimit - maxBufferSpace) 191 | case x if x >= 0 ⇒ AllRead 192 | case -1 ⇒ EndOfStream 193 | case _ ⇒ 194 | throw new IllegalStateException("Unexpected value returned from read: " + readBytes) 195 | } 196 | } else MoreDataWaiting 197 | ``` 198 | 199 | 其中`info.handler ! Received(ByteString(buffer))`是将`SocketChannel`接收到的数据复制成`ByteString`类型之后，再发送出去的，所以相当于是从堆外把数据复制了出去，于是导致整个流程都是非zero copy的。本来在正常的逻辑下，不ZC是必然的，因为肯定要把数据读出来进行处理。但是在本次比赛的场景里，这种复制就是非常昂贵的操作了，直接导致Akka版本的代码无法和各位竞争，即使代码再精简，思想再先进，也无法取得好的成绩。所以在第二个版本中，我换用了netty来跑分。 200 | 201 | Netty版本下的核心代码，分ConsumerAgent和调用PrivderAgent的NettyClient列出如下： 202 | 203 | ConsumerAgent: 204 | ```scala 205 | override def channelRead(ctx: ChannelHandlerContext, msg: scala.Any): Unit = { 206 | msg match { 207 | case in: ByteBuf => 208 | val meshRequest = MeshRequest(cid) //cid是connectionId,在连接建立的时候获取 209 | val maybeClient = ClientChannelHolder.clientChannelCache.get() //client的channel存在了ThreadLocal里面，直接通过ThreadLocal获取到channelHandler 210 | maybeClient.writeAndFlush(meshRequest.toCustomProtocol(in), maybeClient.voidPromise()) //将流入的bytebuffer转变成自定义协议的格式，并直接向client的channel刷入数据 211 | meshRequest.recycle //回收MeshRequest对象 212 | } 213 | } 214 | 215 | ``` 216 | 217 | NettyClient: 218 | ```scala 219 | override def channelRead(ctx: ChannelHandlerContext, msg: scala.Any): Unit = { 220 | msg match { 221 | case bs: ByteBuf => 222 | val cid = bs.getLong(4) //从ByteBuffer中获取connectionId 223 | val resp = MeshResponse(cid) //包装成MeshResponse 224 | val ch = ServerChannelHolder.serverChannelMap.get().get(cid) //根据connectionId获取这个连接的SocketChannel 225 | if(ch != null) { //如果存在的话，则刷入响应 226 | ch.writeAndFlush(resp.toHttpResponse(bs), ch.voidPromise()) 227 | } 228 | resp.recycle //回收MeshResponse对象 229 | } 230 | } 231 | ``` 232 | 233 | 这个是我所发现的最短的路线。其中省略了路由的过程。整体的线程设置如下： 234 | 235 | ```scala 236 | val acceptorGroup = new EpollEventLoopGroup(1) 237 | val threadFactory = new DefaultThreadFactory("atf_wrk") 238 | val workerNumber = 3 239 | val workerGroup = new EpollEventLoopGroup(workerNumber, threadFactory) 240 | ``` 241 | 一个负责IO的线程，三个负责处理请求的线程。三个NettyClient分别使用三个worker中的一个就好了： 242 | 243 | ```scala 244 | val eventLoop = workerGroup.next() 245 | new NettyClient(ed.host, ed.port, eventLoop, 1, ed.scale) 246 | ``` 247 | 248 | 主要的trick就是我只起了4个线程，1个负责IO，3个负责请求处理。通过连接绑定的线程来进行路由，所以少了很多人加权轮询的步骤，而且每个连接只通过同一个线程进行流转，所以也少了context switch的过程。情况好的话，4个线程应该pin到它们的cpu上，没有任何的上下文切换。 249 | 250 | 至于其他就是一些打铁的小细节，比如使用Recycler生成对象池来回收对象，使用池化的ByteBuf来避免堆外内存分配的开销，预先定义好一些要用来包装请求和回复的对象，使用`Unpooled.unreleasableBuffer(buffer)`来反复利用。如此，整个过程下来，不会有FGC，而YGC最多也就两三次而已。Recycler的代码列出如下： 251 | 252 | ```scala 253 | class MeshRequest private(handle: Handle[MeshRequest]) { 254 | private var cid: Long = _ 255 | private var buffer: ByteBuf = _ 256 | private var composite: CompositeByteBuf = _ 257 | 258 | def recycle = { 259 | cid = 0l 260 | buffer = null 261 | composite = null 262 | handle.recycle(this) 263 | } 264 | 265 | def toCustomProtocol(bb: ByteBuf) = { 266 | val n = bb.indexOf(280, bb.readableBytes(), '='.toByte) 267 | val parameter = bb.skipBytes(n + 1) 268 | buffer.writeLong(cid) 269 | buffer.writeInt(parameter.readableBytes()) 270 | composite.addComponents(true, buffer, parameter) 271 | } 272 | 273 | } 274 | 275 | object MeshRequest { 276 | private val RECYCLER = new Recycler[MeshRequest]() { 277 | override def newObject(handle: Recycler.Handle[MeshRequest]): MeshRequest = { 278 | new MeshRequest(handle) 279 | } 280 | } 281 | 282 | def apply(id: Long): MeshRequest = { 283 | val request = RECYCLER.get() 284 | request.cid = id 285 | request.buffer = ConsumerAgent.allocator.directBuffer(12) 286 | request.composite = ConsumerAgent.allocator.compositeBuffer() 287 | request 288 | } 289 | } 290 | ``` 291 | 292 | 最终，Netty版本的代码停留在6894，而Akka版本我没记错的话，应该是6400左右。 293 | 294 | ## 比赛经验总结和感想 295 | 296 | 其实是第一次参加这种编程的比赛，开始的时候看得蛮轻，因为按照实际生产的场景来说，我的第一种方案肯定是非常好的，编码简单、健壮、可扩展性强，应该是能够出彩的。但是因为比赛是唯成绩论的，或者说至少在初赛和复赛的时候是唯成绩论的，所以后续不得已，只能放弃我对Akka的信仰，使用Netty写了一个版本的打铁代码，以往前冲击一个比较好的名次，然后来向大家吹嘘Akka。事实证明，限定场景来做极致优化的话，Netty确实好很多，不过，在通用场景下，用Akka stream的思想，则可以迅速构建出一个集各种流控功能于一体，也非常好扩展，并且性能也不会相差太多的组件。所以，不管怎样，到最终的总结还是，如果是我来开发这个Service Mesh组件，Akka和Akka Stream绝对会是主力，而Netty则可以被应用在不需要将数据读出内存的场景(如只负责转发或者解析自定义协议的Provider端)。两者相结合，应该可以达到比较好的平衡。 297 | 298 | 299 | -------------------------------------------------------------------------------- /repositories: -------------------------------------------------------------------------------- 1 | [repositories] 2 | local 3 | aliyun-ivy: http://maven.aliyun.com/nexus/content/groups/public, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] 4 | aliyun-maven: http://maven.aliyun.com/nexus/content/groups/public 5 | repo2-ivy:http://repo1.maven.org/maven2, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] 6 | repo2:http://repo1.maven.org/maven2/ 7 | typesafe-mave:https://dl.bintray.com/typesafe/maven-releases/ 8 | ivy-typesafe:https://dl.bintray.com/typesafe/ivy-releases, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] 9 | ivy-sbt-plugin:https://dl.bintray.com/sbt/sbt-plugin-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] 10 | sonatype:https://oss.sonatype.org/content/repositories/snapshots 11 | 12 | -------------------------------------------------------------------------------- /start-agent.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ETCD_HOST=etcd 4 | ETCD_PORT=2379 5 | ETCD_URL=http://$ETCD_HOST:$ETCD_PORT 6 | 7 | echo ETCD_URL = $ETCD_URL 8 | 9 | if [[ "$1" == "consumer" ]]; then 10 | echo "Starting consumer agent..." 11 | java -jar \ 12 | -Xms1536M \ 13 | -Xmx1536M \ 14 | -Dtype=consumer \ 15 | -Dserver.port=20000 \ 16 | -Detcd.url=$ETCD_URL \ 17 | -Dlogs.dir=/root/logs \ 18 | /root/dists/mesh-agent.jar 19 | elif [[ "$1" == "provider-small" ]]; then 20 | echo "Starting small provider agent..." 21 | java -jar \ 22 | -Xms512M \ 23 | -Xmx512M \ 24 | -Dtype=provider \ 25 | -Dfactor=0.5 \ 26 | -Dscale=Small \ 27 | -Ddubbo.protocol.port=20880 \ 28 | -Dserver.port=30000 \ 29 | -Detcd.url=$ETCD_URL \ 30 | -Dlogs.dir=/root/logs \ 31 | /root/dists/mesh-agent.jar 32 | elif [[ "$1" == "provider-medium" ]]; then 33 | echo "Starting medium provider agent..." 34 | java -jar \ 35 | -Xms1536M \ 36 | -Xmx1536M \ 37 | -Dtype=provider \ 38 | -Dfactor=0.5 \ 39 | -Dscale=Medium \ 40 | -Ddubbo.protocol.port=20880 \ 41 | -Dserver.port=30000 \ 42 | -Detcd.url=$ETCD_URL \ 43 | -Dlogs.dir=/root/logs \ 44 | /root/dists/mesh-agent.jar 45 | elif [[ "$1" == "provider-large" ]]; then 46 | echo "Starting large provider agent..." 47 | java -jar \ 48 | -Xms2560M \ 49 | -Xmx2560M \ 50 | -Dtype=provider \ 51 | -Dfactor=0.5 \ 52 | -Dscale=Large \ 53 | -Ddubbo.protocol.port=20880 \ 54 | -Dserver.port=30000 \ 55 | -Detcd.url=$ETCD_URL \ 56 | -Dlogs.dir=/root/logs \ 57 | /root/dists/mesh-agent.jar 58 | else 59 | echo "Unrecognized arguments, exit." 60 | exit 1 61 | fi 62 | --------------------------------------------------------------------------------