├── src └── main │ ├── java │ └── me │ │ └── josephzhu │ │ └── proxytest │ │ ├── AllocatorType.java │ │ ├── BackendThreadModel.java │ │ ├── ServerConfig.java │ │ ├── TestController.java │ │ ├── Application.java │ │ ├── layer4 │ │ ├── Layer4ProxyServer.java │ │ ├── BackendHandler.java │ │ └── FrontendHandler.java │ │ ├── layer7 │ │ ├── Layer7ProxyServer.java │ │ ├── BackendHandler.java │ │ └── FrontendHandler.java │ │ └── ProxyServer.java │ └── resources │ ├── application.yml │ └── application-prod.yml ├── .gitignore ├── pom.xml └── README.md /src/main/java/me/josephzhu/proxytest/AllocatorType.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest; 2 | 3 | public enum AllocatorType { 4 | Pooled, Unpooled 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/BackendThreadModel.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest; 2 | 3 | public enum BackendThreadModel { 4 | ReuseServerGroup, IndividualGroup, ReuseServerThread 5 | } 6 | -------------------------------------------------------------------------------- /src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | logging: 2 | level: 3 | ROOT: INFO 4 | server: 5 | type: Layer7ProxyServer 6 | serverIp: 127.0.0.1 7 | serverPort: 8888 8 | backendIp: 127.0.0.1 9 | backendPort: 8080 10 | backendThreadModel: ReuseServerThread #ReuseServerGroup, IndividualGroup, ReuseServerThread 11 | allocatorType: Unpooled 12 | receiveBuffer: 10240 13 | sendBuffer: 10240 14 | maxContentLength: 2000000 -------------------------------------------------------------------------------- /src/main/resources/application-prod.yml: -------------------------------------------------------------------------------- 1 | logging: 2 | level: 3 | ROOT: INFO 4 | server: 5 | type: Layer7ProxyServer 6 | serverIp: 172.26.5.213 7 | serverPort: 8888 8 | backendIp: 172.26.5.214 9 | backendPort: 80 10 | backendThreadModel: ReuseServerThread #ReuseServerGroup, IndividualGroup, ReuseServerThread 11 | allocatorType: Unpooled 12 | receiveBuffer: 10240 13 | sendBuffer: 10240 14 | maxContentLength: 2000 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/** 5 | !**/src/test/** 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | 30 | ### VS Code ### 31 | .vscode/ 32 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/ServerConfig.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest; 2 | 3 | import lombok.Data; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | 6 | @ConfigurationProperties(prefix = "server") 7 | @Data 8 | public class ServerConfig { 9 | private String type; 10 | private String serverIp; 11 | private int serverPort; 12 | private String backendIp; 13 | private int backendPort; 14 | private BackendThreadModel backendThreadModel; 15 | private int receiveBuffer; 16 | private int sendBuffer; 17 | private AllocatorType allocatorType; 18 | private int maxContentLength; 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/TestController.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.springframework.web.bind.annotation.GetMapping; 5 | import org.springframework.web.bind.annotation.RequestHeader; 6 | import org.springframework.web.bind.annotation.RestController; 7 | 8 | import java.util.stream.Collectors; 9 | import java.util.stream.IntStream; 10 | 11 | @RestController 12 | @Slf4j 13 | public class TestController { 14 | 15 | private static String payload = IntStream.rangeClosed(1, 1000).mapToObj(i -> "a").collect(Collectors.joining("")); 16 | 17 | @GetMapping("/test") 18 | public String test(@RequestHeader(value = "aa", required = false) String aa) { 19 | //log.info("aa:{}",aa); 20 | return payload; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/Application.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.boot.CommandLineRunner; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 9 | import org.springframework.context.ApplicationContext; 10 | 11 | @SpringBootApplication 12 | @EnableConfigurationProperties(ServerConfig.class) 13 | @Slf4j 14 | public class Application implements CommandLineRunner { 15 | 16 | @Autowired 17 | ApplicationContext applicationContext; 18 | @Autowired 19 | ServerConfig serverConfig; 20 | 21 | public static void main(String[] args) { 22 | SpringApplication.run(Application.class, args); 23 | } 24 | 25 | @Override 26 | public void run(String... args) throws Exception { 27 | applicationContext.getBean(serverConfig.getType(), ProxyServer.class).start(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/layer4/Layer4ProxyServer.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest.layer4; 2 | 3 | import io.netty.bootstrap.ServerBootstrap; 4 | import io.netty.channel.Channel; 5 | import io.netty.channel.ChannelInitializer; 6 | import io.netty.channel.ChannelOption; 7 | import lombok.extern.slf4j.Slf4j; 8 | import me.josephzhu.proxytest.ProxyServer; 9 | import org.springframework.stereotype.Component; 10 | 11 | @Component("Layer4ProxyServer") 12 | @Slf4j 13 | public class Layer4ProxyServer extends ProxyServer { 14 | 15 | @Override 16 | protected void config(ServerBootstrap b) { 17 | b.childOption(ChannelOption.AUTO_READ, false); 18 | } 19 | 20 | @Override 21 | protected ChannelInitializer getChannelInitializer() { 22 | return new ChannelInitializer() { 23 | @Override 24 | protected void initChannel(Channel ch) { 25 | ch.pipeline().addLast(new FrontendHandler(serverConfig.getBackendIp(), serverConfig.getBackendPort(), serverConfig.getBackendThreadModel())); 26 | } 27 | }; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/layer7/Layer7ProxyServer.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest.layer7; 2 | 3 | import io.netty.channel.Channel; 4 | import io.netty.channel.ChannelInitializer; 5 | import io.netty.handler.codec.http.HttpObjectAggregator; 6 | import io.netty.handler.codec.http.HttpServerCodec; 7 | import lombok.extern.slf4j.Slf4j; 8 | import me.josephzhu.proxytest.ProxyServer; 9 | import org.springframework.stereotype.Component; 10 | 11 | @Component("Layer7ProxyServer") 12 | @Slf4j 13 | public class Layer7ProxyServer extends ProxyServer { 14 | 15 | @Override 16 | protected ChannelInitializer getChannelInitializer() { 17 | return new ChannelInitializer() { 18 | @Override 19 | protected void initChannel(Channel ch) { 20 | ch.pipeline().addLast(new HttpServerCodec(), new HttpObjectAggregator(serverConfig.getMaxContentLength())); 21 | ch.pipeline().addLast(new FrontendHandler(serverConfig.getBackendIp(), serverConfig.getBackendPort(), serverConfig.getBackendThreadModel())); 22 | } 23 | }; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/layer7/BackendHandler.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest.layer7; 2 | 3 | import io.netty.channel.Channel; 4 | import io.netty.channel.ChannelHandlerContext; 5 | import io.netty.channel.ChannelInboundHandlerAdapter; 6 | import io.netty.handler.codec.http.FullHttpResponse; 7 | 8 | public class BackendHandler extends ChannelInboundHandlerAdapter { 9 | 10 | private final Channel inboundChannel; 11 | 12 | public BackendHandler(Channel inboundChannel) { 13 | this.inboundChannel = inboundChannel; 14 | } 15 | 16 | @Override 17 | public void channelRead(final ChannelHandlerContext ctx, Object msg) { 18 | if (msg instanceof FullHttpResponse) { 19 | FullHttpResponse httpResponse = (FullHttpResponse) msg; 20 | httpResponse.headers().add("cc", "dd"); 21 | inboundChannel.writeAndFlush(httpResponse); 22 | } else { 23 | ctx.channel().close(); 24 | } 25 | } 26 | 27 | @Override 28 | public void channelInactive(ChannelHandlerContext ctx) { 29 | FrontendHandler.closeOnFlush(inboundChannel); 30 | } 31 | 32 | @Override 33 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 34 | cause.printStackTrace(); 35 | FrontendHandler.closeOnFlush(ctx.channel()); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/layer4/BackendHandler.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest.layer4; 2 | 3 | import io.netty.channel.Channel; 4 | import io.netty.channel.ChannelFutureListener; 5 | import io.netty.channel.ChannelHandlerContext; 6 | import io.netty.channel.ChannelInboundHandlerAdapter; 7 | 8 | public class BackendHandler extends ChannelInboundHandlerAdapter { 9 | 10 | private final Channel inboundChannel; 11 | 12 | public BackendHandler(Channel inboundChannel) { 13 | this.inboundChannel = inboundChannel; 14 | } 15 | 16 | @Override 17 | public void channelActive(ChannelHandlerContext ctx) { 18 | ctx.read(); 19 | } 20 | 21 | @Override 22 | public void channelRead(final ChannelHandlerContext ctx, Object msg) { 23 | inboundChannel.writeAndFlush(msg).addListener((ChannelFutureListener) future -> { 24 | if (future.isSuccess()) { 25 | ctx.channel().read(); 26 | } else { 27 | future.channel().close(); 28 | } 29 | }); 30 | } 31 | 32 | @Override 33 | public void channelInactive(ChannelHandlerContext ctx) { 34 | FrontendHandler.closeOnFlush(inboundChannel); 35 | } 36 | 37 | @Override 38 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 39 | cause.printStackTrace(); 40 | FrontendHandler.closeOnFlush(ctx.channel()); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.springframework.boot 8 | spring-boot-starter-parent 9 | 2.1.6.RELEASE 10 | 11 | 12 | me.josephzhu 13 | proxytest 14 | 1.0-SNAPSHOT 15 | 16 | 17 | 1.8 18 | 19 | 20 | 21 | 22 | io.netty 23 | netty-all 24 | 4.1.36.Final 25 | 26 | 27 | org.projectlombok 28 | lombok 29 | true 30 | 31 | 32 | org.springframework.boot 33 | spring-boot-starter 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-starter-web 38 | 39 | 40 | 41 | 42 | 43 | 44 | org.springframework.boot 45 | spring-boot-maven-plugin 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/ProxyServer.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest; 2 | 3 | import io.netty.bootstrap.ServerBootstrap; 4 | import io.netty.buffer.PooledByteBufAllocator; 5 | import io.netty.buffer.UnpooledByteBufAllocator; 6 | import io.netty.channel.Channel; 7 | import io.netty.channel.ChannelInitializer; 8 | import io.netty.channel.ChannelOption; 9 | import io.netty.channel.EventLoopGroup; 10 | import io.netty.channel.nio.NioEventLoopGroup; 11 | import io.netty.channel.socket.nio.NioServerSocketChannel; 12 | import lombok.extern.slf4j.Slf4j; 13 | import org.springframework.beans.factory.annotation.Autowired; 14 | 15 | @Slf4j 16 | public abstract class ProxyServer { 17 | 18 | public static final EventLoopGroup serverBossGroup = new NioEventLoopGroup(); 19 | public static final EventLoopGroup serverWorkerGroup = new NioEventLoopGroup(); 20 | public static final EventLoopGroup backendWorkerGroup = new NioEventLoopGroup(); 21 | @Autowired 22 | protected ServerConfig serverConfig; 23 | 24 | protected abstract ChannelInitializer getChannelInitializer(); 25 | 26 | protected void config(ServerBootstrap b) { 27 | } 28 | 29 | public void start() throws Exception { 30 | 31 | try { 32 | ServerBootstrap b = new ServerBootstrap(); 33 | b.group(serverBossGroup, serverWorkerGroup) 34 | .channel(NioServerSocketChannel.class) 35 | .childHandler(getChannelInitializer()); 36 | b.childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBuffer()) 37 | .childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBuffer()); 38 | if (serverConfig.getAllocatorType() == AllocatorType.Pooled) 39 | b.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); 40 | if (serverConfig.getAllocatorType() == AllocatorType.Unpooled) 41 | b.childOption(ChannelOption.ALLOCATOR, UnpooledByteBufAllocator.DEFAULT); 42 | config(b); 43 | b.bind(serverConfig.getServerIp(), serverConfig.getServerPort()) 44 | .addListener(future -> log.info("{} Started with config: {}", getClass().getSimpleName(), serverConfig)) 45 | .sync().channel().closeFuture().sync(); 46 | } finally { 47 | serverBossGroup.shutdownGracefully(); 48 | serverWorkerGroup.shutdownGracefully(); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/layer4/FrontendHandler.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest.layer4; 2 | 3 | import io.netty.bootstrap.Bootstrap; 4 | import io.netty.buffer.Unpooled; 5 | import io.netty.channel.*; 6 | import io.netty.channel.socket.nio.NioSocketChannel; 7 | import me.josephzhu.proxytest.BackendThreadModel; 8 | 9 | public class FrontendHandler extends ChannelInboundHandlerAdapter { 10 | 11 | private final String remoteHost; 12 | private final int remotePort; 13 | private final BackendThreadModel backendThreadModel; 14 | private Channel outboundChannel; 15 | 16 | public FrontendHandler(String remoteHost, int remotePort, BackendThreadModel backendThreadModel) { 17 | this.remoteHost = remoteHost; 18 | this.remotePort = remotePort; 19 | this.backendThreadModel = backendThreadModel; 20 | } 21 | 22 | static void closeOnFlush(Channel ch) { 23 | if (ch.isActive()) { 24 | ch.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE); 25 | } 26 | } 27 | 28 | @Override 29 | public void channelActive(ChannelHandlerContext ctx) { 30 | final Channel inboundChannel = ctx.channel(); 31 | 32 | Bootstrap b = new Bootstrap(); 33 | switch (backendThreadModel) { 34 | case ReuseServerGroup: { 35 | b.group(Layer4ProxyServer.serverWorkerGroup); 36 | break; 37 | } 38 | case IndividualGroup: { 39 | b.group(Layer4ProxyServer.backendWorkerGroup); 40 | break; 41 | } 42 | case ReuseServerThread: { 43 | b.group(inboundChannel.eventLoop()); 44 | break; 45 | } 46 | default: 47 | break; 48 | } 49 | b.option(ChannelOption.AUTO_READ, true) 50 | .channel(NioSocketChannel.class) 51 | .handler(new BackendHandler(inboundChannel)); 52 | ChannelFuture f = b.connect(remoteHost, remotePort).addListener((ChannelFutureListener) future -> { 53 | if (future.isSuccess()) { 54 | inboundChannel.read(); 55 | } else { 56 | inboundChannel.close(); 57 | } 58 | }); 59 | outboundChannel = f.channel(); 60 | } 61 | 62 | @Override 63 | public void channelRead(final ChannelHandlerContext ctx, Object msg) { 64 | if (outboundChannel.isActive()) { 65 | outboundChannel.writeAndFlush(msg).addListener((ChannelFutureListener) future -> { 66 | if (future.isSuccess()) { 67 | ctx.channel().read(); 68 | } else { 69 | future.channel().close(); 70 | } 71 | }); 72 | } 73 | } 74 | 75 | @Override 76 | public void channelInactive(ChannelHandlerContext ctx) { 77 | if (outboundChannel != null) { 78 | closeOnFlush(outboundChannel); 79 | } 80 | } 81 | 82 | @Override 83 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 84 | cause.printStackTrace(); 85 | closeOnFlush(ctx.channel()); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/me/josephzhu/proxytest/layer7/FrontendHandler.java: -------------------------------------------------------------------------------- 1 | package me.josephzhu.proxytest.layer7; 2 | 3 | import io.netty.bootstrap.Bootstrap; 4 | import io.netty.buffer.Unpooled; 5 | import io.netty.channel.*; 6 | import io.netty.channel.socket.nio.NioSocketChannel; 7 | import io.netty.handler.codec.http.FullHttpRequest; 8 | import io.netty.handler.codec.http.HttpClientCodec; 9 | import io.netty.handler.codec.http.HttpObjectAggregator; 10 | import me.josephzhu.proxytest.BackendThreadModel; 11 | import org.springframework.http.HttpHeaders; 12 | 13 | public class FrontendHandler extends ChannelInboundHandlerAdapter { 14 | 15 | private final String remoteHost; 16 | private final int remotePort; 17 | private final BackendThreadModel backendThreadModel; 18 | private Channel outboundChannel; 19 | 20 | public FrontendHandler(String remoteHost, int remotePort, BackendThreadModel backendThreadModel) { 21 | this.remoteHost = remoteHost; 22 | this.remotePort = remotePort; 23 | this.backendThreadModel = backendThreadModel; 24 | } 25 | 26 | static void closeOnFlush(Channel ch) { 27 | if (ch.isActive()) { 28 | ch.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE); 29 | } 30 | } 31 | 32 | private void read(final ChannelHandlerContext ctx, Object msg) { 33 | if (msg instanceof FullHttpRequest) { 34 | FullHttpRequest httpRequest = (FullHttpRequest) msg; 35 | String hostPort = remoteHost + ":" + remotePort; 36 | httpRequest.headers().set(HttpHeaders.HOST, hostPort); 37 | httpRequest.headers().add("aa", "bb"); 38 | outboundChannel.writeAndFlush(msg); 39 | } else { 40 | closeOnFlush(ctx.channel()); 41 | } 42 | } 43 | 44 | @Override 45 | public void channelRead(final ChannelHandlerContext ctx, Object msg) { 46 | if (outboundChannel == null) { 47 | final Channel inboundChannel = ctx.channel(); 48 | 49 | Bootstrap b = new Bootstrap(); 50 | switch (backendThreadModel) { 51 | case ReuseServerGroup: { 52 | b.group(Layer7ProxyServer.serverWorkerGroup); 53 | break; 54 | } 55 | case IndividualGroup: { 56 | b.group(Layer7ProxyServer.backendWorkerGroup); 57 | break; 58 | } 59 | case ReuseServerThread: { 60 | b.group(inboundChannel.eventLoop()); 61 | break; 62 | } 63 | default: 64 | break; 65 | } 66 | 67 | b.channel(NioSocketChannel.class).handler(new ChannelInitializer() { 68 | @Override 69 | protected void initChannel(Channel ch) { 70 | ch.pipeline().addLast(new HttpClientCodec(), new HttpObjectAggregator(2000)); 71 | ch.pipeline().addLast(new BackendHandler(inboundChannel)); 72 | } 73 | }); 74 | ChannelFuture f = b.connect(remoteHost, remotePort).addListener((ChannelFutureListener) future -> { 75 | if (future.isSuccess()) { 76 | future.channel().writeAndFlush(msg); 77 | } else { 78 | future.channel().close(); 79 | } 80 | }); 81 | outboundChannel = f.channel(); 82 | } else { 83 | read(ctx, msg); 84 | } 85 | } 86 | 87 | @Override 88 | public void channelInactive(ChannelHandlerContext ctx) { 89 | if (outboundChannel != null) { 90 | closeOnFlush(outboundChannel); 91 | } 92 | } 93 | 94 | @Override 95 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 96 | cause.printStackTrace(); 97 | closeOnFlush(ctx.channel()); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # 基于Netty的四层和七层代理性能方面的一些压力测试 3 | 4 | 本文我们主要是想测试和研究几点： 5 | 6 | - 基于Netty写的最简单的转发HTTP请求的程序，四层和七层性能的差异 7 | - 三种代理线程模型性能的差异，下文会详细解释三种线程模型 8 | - 池和非池化ByteBuffer性能的差异 9 | 10 | 本文测试使用的代码在： 11 | https://github.com/JosephZhu1983/proxytest 12 | 13 | 在代码里我们实现了两套代理程序： 14 | ![image_1demdig32ia6184m64sppm8vp90.png-55.9kB][1] 15 | 16 | 测试使用的机器配置是（阿里云ECS）： 17 | ![image_1dembkev02d2sll1ijc18fl4r48j.png-91.9kB][2] 18 | 一共三台机器： 19 | 20 | - server 服务器安装了nginx，作为后端 21 | - client 服务器安装了wrk，作为压测客户端 22 | - proxy 服务器安装了我们的测试代码（代理） 23 | 24 | 25 | ## Nginx后端 26 | 27 | nginx 配置的就是默认的测试页（删了点内容，减少内网带宽）： 28 | ![image_1dembfnk81i9m19tkvli148c13h86.png-122.8kB][3] 29 | 直接对着nginx压测下来的qps是26.6万： 30 | ![image_1delvmebjcpe39n1hdni41hss13.png-55.2kB][4] 31 | 32 | ## 有关四层和七层 33 | 34 | 四层的代理，我们仅仅是使用Netty来转发ByteBuf。 35 | 七层的代理，会有更多额外的开销，主要是Http请求的编码解码以及Http请求的聚合，服务端： 36 | 37 | ![image_1demdm2m82vg1i6b4ng1uitjcp9d.png-136.8kB][5] 38 | 39 | 客户端： 40 | ![image_1demdoius2ekjds1kbr5a1vld9q.png-63.2kB][6] 41 | 42 | 这里我们可以想到，四层代理因为少了Http数据的编解码过程，性能肯定比七层好很多，好多少我们可以看看测试结果。 43 | 44 | ## 有关线程模型 45 | 46 | 我们知道作为一个代理，我们需要开启服务端从上游来获取请求，然后再作为客户端把请求转发到下游，从下游获取到响应后，返回给上游。我们的服务端和客户端都需要Worker线程来处理IO请求，有三种做法； 47 | 48 | - A：客户端Bootstrap和服务端ServerBootstrap独立的线程池NioEventLoopGroup，简称IndividualGroup 49 | - B：客户端和服务端共享一套线程池，简称ReuseServerGroup 50 | - C：客户端直接复用服务端线程EventLoop，简称ReuseServerThread 51 | 52 | 以七层代理的代码为例： 53 | ![image_1demdqavbn5i19ff1g1hrp2gbsan.png-98.4kB][7] 54 | 55 | 接下去的测试我们会来测试这三种线程模型，这里想当然的猜测是方案A的性能是最好的，因为独立了线程池不相互影响，我们接下去看看结果 56 | 57 | ## 四层代理 + ReuseServerThread线程模型 58 | Layer4ProxyServer Started with config: ServerConfig(type=Layer4ProxyServer, serverIp=172.26.5.213, serverPort=8888, backendIp=172.26.5.214, backendPort=80, backendThreadModel=ReuseServerThread, receiveBuffer=10240, sendBuffer=10240, allocatorType=Unpooled, maxContentLength=2000) 59 | ![image_1delvsom6v03e5pngacv714901g.png-54kB][8] 60 | 61 | ## 四层代理 + IndividualGroup线程模型 62 | Layer4ProxyServer Started with config: ServerConfig(type=Layer4ProxyServer, serverIp=172.26.5.213, serverPort=8888, backendIp=172.26.5.214, backendPort=80, backendThreadModel=IndividualGroup, receiveBuffer=10240, sendBuffer=10240, allocatorType=Unpooled, maxContentLength=2000) 63 | ![image_1dem04l2alqs1l4u1ripg9a1fcu1t.png-54.8kB][9] 64 | 65 | ## 四层代理 + ReuseServerGroup线程模型 66 | Layer4ProxyServer Started with config: ServerConfig(type=Layer4ProxyServer, serverIp=172.26.5.213, serverPort=8888, backendIp=172.26.5.214, backendPort=80, backendThreadModel=ReuseServerGroup, receiveBuffer=10240, sendBuffer=10240, allocatorType=Unpooled, maxContentLength=2000) 67 | ![image_1dem0br3r1rr3qmj1mk519nn111v2a.png-55.2kB][10] 68 | 69 | 看到这里其实已经有结果了，ReuseServerThread性能是最好的，其次是ReuseServerGroup，最差是IndividualGroup，和我们猜的不一致。 70 | 71 | ### 四层系统监控图 72 | 从网络带宽上可以看到，先测试的ReuseServerThread跑到了最大的带宽（后面三个高峰分别代表了三次测试）： 73 | ![image_1dem0chjrimkn5va5810dk1vk62n.png-52.8kB][11] 74 | 从CPU监控上可以看到，性能最好的ReuseServerThread使用了最少的CPU资源（后面三个高峰分别代表了三次测试）： 75 | ![image_1dem0ekoq1l59ju1vvn1lp575u34.png-32.5kB][12] 76 | 77 | ## 七层代理 + ReuseServerThread线程模型 78 | Layer7ProxyServer Started with config: ServerConfig(type=Layer7ProxyServer, serverIp=172.26.5.213, serverPort=8888, backendIp=172.26.5.214, backendPort=80, backendThreadModel=ReuseServerThread, receiveBuffer=10240, sendBuffer=10240, allocatorType=Unpooled, maxContentLength=2000) 79 | ![image_1dem0mduhkdc11hc2ue12rd433h.png-55kB][13] 80 | 81 | ## 七层代理 + IndividualGroup线程模型 82 | Layer7ProxyServer Started with config: ServerConfig(type=Layer7ProxyServer, serverIp=172.26.5.213, serverPort=8888, backendIp=172.26.5.214, backendPort=80, backendThreadModel=IndividualGroup, receiveBuffer=10240, sendBuffer=10240, allocatorType=Unpooled, maxContentLength=2000) 83 | ![image_1dem0tgtv13ev3h9sl51appi083u.png-55.2kB][14] 84 | 85 | ## 七层代理 + ReuseServerGroup线程模型 86 | Layer7ProxyServer Started with config: ServerConfig(type=Layer7ProxyServer, serverIp=172.26.5.213, serverPort=8888, backendIp=172.26.5.214, backendPort=80, backendThreadModel=ReuseServerGroup, receiveBuffer=10240, sendBuffer=10240, allocatorType=Unpooled, maxContentLength=2000) 87 | ![image_1dem14prr1e7kr0gi1ggiqu7l4b.png-55kB][15] 88 | 89 | 结论一样，ReuseServerThread性能是最好的，其次是ReuseServerGroup，最差是IndividualGroup。我觉得是这么一个道理： 90 | 91 | - 复用IO线程的话，上下文切换会比较少，性能是最好的，后来我也通过pidstat观察验证了这个结论，但是当时忘记截图 92 | - 复用线程池，客户端有机会能复用到服务端线程，避免部分上下文切换，性能中等 93 | - 独立线程池，大量上下文切换（观察下来是复用IO线程的4x），性能最差 94 | 95 | ## 七层系统监控图 96 | 97 | 下面分别是网络带宽和CPU监控图： 98 | ![image_1dem1fh7m1f0cl8s1d1ic7563765.png-39.3kB][16] 99 | ![image_1dem1e3g01asrq8r9u16ce5e94r.png-60.1kB][17] 100 | 可以看到明显七层代理消耗更多的资源，但是带宽相比四层少了一些（QPS少了很多）。 101 | 出流量比入流量多一点，应该是代码里多加的请求头导致： 102 | ![image_1demf0bhrikp1rh0r5i1q3c1iltc1.png-150.8kB][18] 103 | 104 | ## 试试HttpObjectAggregator设置较大maxContentLength 105 | Layer7ProxyServer Started with config: ServerConfig(type=Layer7ProxyServer, serverIp=172.26.5.213, serverPort=8888, backendIp=172.26.5.214, backendPort=80, backendThreadModel=ReuseServerThread, receiveBuffer=10240, sendBuffer=10240, allocatorType=Pooled, maxContentLength=100000000) 106 | ![image_1dem1qe4v1ddd1c2311pjej81bf16v.png-54.9kB][19] 107 | 108 | ## 试试PooledByteBufAllocator 109 | Layer7ProxyServer Started with config: ServerConfig(type=Layer7ProxyServer, serverIp=172.26.5.213, serverPort=8888, backendIp=172.26.5.214, backendPort=80, backendThreadModel=ReuseServerThread, receiveBuffer=10240, sendBuffer=10240, allocatorType=Pooled, maxContentLength=2000) 110 | ![image_1dem1ifds1hoi1lkka691vekmlt6i.png-54.8kB][20] 111 | 112 | 可以看到Netty 4.1中已经把默认的分配器设置为了PooledByteBufAllocator 113 | ![image_1demg35il1ambhdb1o3m42c1j9ce.png-43.9kB][21] 114 | 115 | 116 | ## 总结 117 | 118 | 这里总结了一个表格，性能损失比例都以第一行直接压Nginx为参照： 119 | ![image_1demepcbume4eoacntrb11mh2b4.png-39.1kB][22] 120 | 121 | 结论是： 122 | 123 | - Nginx很牛，其实机器配置不算太好，在配置比较好的物理机服务器上跑的化，Nginx单机百万没问题 124 | - Netty很牛，毕竟是Java的服务端，四层转发仅损失3% QPS 125 | - 不管是七层还是四层，复用线程的方式明显性能最好，占用CPU最少 126 | - 因为上下文切换的原因，使用Netty开发网络代理应该复用IO线程 127 | - 七层的消耗比四层大很多，即使是Netty也避免不了，这是HTTP协议的问题 128 | - PooledByteBufAllocator性能比UnpooledByteBufAllocator有一定提升（接近3%） 129 | - HttpObjectAggregator如果设置较大的最大内容长度，会略微影响点性能 130 | 131 | 之所以写这个文章做这个分析的原因是因为最近在做我们自研网关的性能优化和压力测试https://github.com/spring-avengers/tesla。 132 | 我发现有一些其它开源的基于Netty的代理项目并不是复用连接的，可能作者没有意识到这个问题，我看了下Zuul的代码，它也是复用的。 133 | 134 | 135 | [1]: http://static.zybuluo.com/powerzhuye/hd8coufexmmu123bpn2ysnhb/image_1demdig32ia6184m64sppm8vp90.png 136 | [2]: http://static.zybuluo.com/powerzhuye/hj8cd7qzdphfmxnk4mkbv3af/image_1dembkev02d2sll1ijc18fl4r48j.png 137 | [3]: http://static.zybuluo.com/powerzhuye/39dl9cxlom1f9srcf9ytuinc/image_1dembfnk81i9m19tkvli148c13h86.png 138 | [4]: http://static.zybuluo.com/powerzhuye/ktvuqbcntfaz6eka57aagvwy/image_1delvmebjcpe39n1hdni41hss13.png 139 | [5]: http://static.zybuluo.com/powerzhuye/ds97unzbuvcygsei1c79am70/image_1demdm2m82vg1i6b4ng1uitjcp9d.png 140 | [6]: http://static.zybuluo.com/powerzhuye/b8iad03tws1hljf5ebpcw8is/image_1demdoius2ekjds1kbr5a1vld9q.png 141 | [7]: http://static.zybuluo.com/powerzhuye/44zorsf7ddxw93o9nnfpauve/image_1demdqavbn5i19ff1g1hrp2gbsan.png 142 | [8]: http://static.zybuluo.com/powerzhuye/6sv76v00ubtjh958p8s47yjl/image_1delvsom6v03e5pngacv714901g.png 143 | [9]: http://static.zybuluo.com/powerzhuye/zvan1ms4xm82yluna87nnsjd/image_1dem04l2alqs1l4u1ripg9a1fcu1t.png 144 | [10]: http://static.zybuluo.com/powerzhuye/xz43jnb7hxnq3fopdindecma/image_1dem0br3r1rr3qmj1mk519nn111v2a.png 145 | [11]: http://static.zybuluo.com/powerzhuye/ug057h22bmmqfe7hg6wbmnq0/image_1dem0chjrimkn5va5810dk1vk62n.png 146 | [12]: http://static.zybuluo.com/powerzhuye/pl7xqus0vu3terdocjf7oq7r/image_1dem0ekoq1l59ju1vvn1lp575u34.png 147 | [13]: http://static.zybuluo.com/powerzhuye/30g713a9qlivs8v9ve5aqv4r/image_1dem0mduhkdc11hc2ue12rd433h.png 148 | [14]: http://static.zybuluo.com/powerzhuye/n0prcj684x3unok8md7v81xo/image_1dem0tgtv13ev3h9sl51appi083u.png 149 | [15]: http://static.zybuluo.com/powerzhuye/79bqjwi4ci0sk54m6m5xs0uw/image_1dem14prr1e7kr0gi1ggiqu7l4b.png 150 | [16]: http://static.zybuluo.com/powerzhuye/ujmozpj5shcl9mlzez5xgpxe/image_1dem1fh7m1f0cl8s1d1ic7563765.png 151 | [17]: http://static.zybuluo.com/powerzhuye/4v99rrz1dw3bayghfr4pify0/image_1dem1e3g01asrq8r9u16ce5e94r.png 152 | [18]: http://static.zybuluo.com/powerzhuye/3juf6hh3onaenkn1e0ttsnz8/image_1demf0bhrikp1rh0r5i1q3c1iltc1.png 153 | [19]: http://static.zybuluo.com/powerzhuye/shhkjuuyt409w6fj49a0std7/image_1dem1qe4v1ddd1c2311pjej81bf16v.png 154 | [20]: http://static.zybuluo.com/powerzhuye/44nl3scrjg10gxn3ahn2uuyg/image_1dem1ifds1hoi1lkka691vekmlt6i.png 155 | [21]: http://static.zybuluo.com/powerzhuye/q7kso8v2foreco8u8bewwx14/image_1demg35il1ambhdb1o3m42c1j9ce.png 156 | [22]: http://static.zybuluo.com/powerzhuye/3uyf2tca2jjtvor97g7uu9lm/image_1demepcbume4eoacntrb11mh2b4.png 157 | --------------------------------------------------------------------------------