├── demo.gif ├── gateway ├── Dockerfile ├── src │ ├── main │ │ ├── java │ │ │ └── me │ │ │ │ └── lawrenceli │ │ │ │ └── gateway │ │ │ │ ├── GatewayApp.java │ │ │ │ ├── server │ │ │ │ ├── FanoutSender.java │ │ │ │ ├── ServiceNode.java │ │ │ │ └── RedisSubscriber.java │ │ │ │ ├── config │ │ │ │ ├── CorsConfig.java │ │ │ │ ├── RedisConfig.java │ │ │ │ ├── NamingServiceConfig.java │ │ │ │ ├── GatewayHashRingConfig.java │ │ │ │ └── WebSocketProperties.java │ │ │ │ ├── discovery │ │ │ │ └── DiscoveryController.java │ │ │ │ ├── docker │ │ │ │ ├── DockerController.java │ │ │ │ └── DockerService.java │ │ │ │ └── filter │ │ │ │ ├── CustomLoadBalanceFilter.java │ │ │ │ ├── CustomReactiveLoadBalanceFilter.java │ │ │ │ └── WebSocketSessionLoadBalancer.java │ │ └── resources │ │ │ └── application.yaml │ └── test │ │ └── java │ │ └── me │ │ └── lawrenceli │ │ └── gateway │ │ ├── config │ │ └── NamingServiceConfigTest.java │ │ ├── server │ │ ├── ServiceNodeTest.java │ │ └── RedisSubscriberTest.java │ │ ├── filter │ │ └── CustomLoadBalanceFilterTest.java │ │ ├── docker │ │ ├── DockerControllerTest.java │ │ └── DockerServiceTest.java │ │ └── discovery │ │ └── DiscoveryControllerTest.java └── pom.xml ├── websocket ├── Dockerfile ├── src │ ├── main │ │ ├── java │ │ │ └── me │ │ │ │ └── lawrenceli │ │ │ │ └── websocket │ │ │ │ ├── WebSocketApp.java │ │ │ │ ├── config │ │ │ │ ├── WebSocketConfig.java │ │ │ │ └── MQConfig.java │ │ │ │ ├── server │ │ │ │ ├── FanoutReceiver.java │ │ │ │ └── WebSocketEndpoint.java │ │ │ │ ├── spring │ │ │ │ └── BeanUtils.java │ │ │ │ ├── controller │ │ │ │ └── WebSocketController.java │ │ │ │ └── event │ │ │ │ ├── ServerDownEventHandler.java │ │ │ │ └── ServerUpEventHandler.java │ │ └── resources │ │ │ └── application.yaml │ └── test │ │ └── java │ │ └── me │ │ └── lawrenceli │ │ └── websocket │ │ ├── event │ │ ├── ServerDownEventHandlerTest.java │ │ └── ServerUpEventHandlerTest.java │ │ ├── controller │ │ └── WebSocketControllerTest.java │ │ ├── config │ │ └── MQConfigTest.java │ │ └── server │ │ └── WebSocketEndpointTest.java └── pom.xml ├── common ├── src │ ├── main │ │ └── java │ │ │ └── me │ │ │ └── lawrenceli │ │ │ ├── hashring │ │ │ ├── Node.java │ │ │ ├── HashAlgorithm.java │ │ │ ├── VirtualNode.java │ │ │ └── ConsistentHashRouter.java │ │ │ ├── utils │ │ │ ├── StringPattern.java │ │ │ └── JSON.java │ │ │ ├── model │ │ │ ├── MessageType.java │ │ │ └── WebSocketMessage.java │ │ │ └── constant │ │ │ └── GlobalConstant.java │ └── test │ │ └── java │ │ └── me │ │ └── lawrenceli │ │ ├── hashring │ │ ├── ConsistentHashRouterTest.java │ │ └── VirtualNodeTest.java │ │ ├── utils │ │ ├── JSONTest.java │ │ └── StringPatternTest.java │ │ └── model │ │ └── WebSocketMessageTest.java └── pom.xml ├── docker-compose.yml ├── Makefile ├── .github ├── dependabot.yml └── workflows │ └── sonar.yml ├── .gitignore ├── pom.xml ├── README.md └── README-en.md /demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/la3rence/websocket-cluster/HEAD/demo.gif -------------------------------------------------------------------------------- /gateway/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jdk-alpine 2 | VOLUME /tmp 3 | ARG JAR_FILE=target/gateway-1.0.0.jar 4 | COPY ${JAR_FILE} app.jar 5 | ENTRYPOINT java ${JAVA_OPTS} -Duser.timezone=GMT+08 -Djava.security.egd=file:/dev/./urandom -jar /app.jar -------------------------------------------------------------------------------- /websocket/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jdk-alpine 2 | VOLUME /tmp 3 | ARG JAR_FILE=target/websocket-1.0.0.jar 4 | COPY ${JAR_FILE} app.jar 5 | ENTRYPOINT java ${JAVA_OPTS} -Duser.timezone=GMT+08 -Djava.security.egd=file:/dev/./urandom -jar /app.jar -------------------------------------------------------------------------------- /common/src/main/java/me/lawrenceli/hashring/Node.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.hashring; 2 | 3 | /** 4 | * Hash 环的节点 5 | * 6 | * @author lawrence 7 | * @since 2021/3/23 8 | */ 9 | public interface Node { 10 | 11 | String getKey(); 12 | } 13 | -------------------------------------------------------------------------------- /common/src/main/java/me/lawrenceli/hashring/HashAlgorithm.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.hashring; 2 | 3 | /** 4 | * @author lawrence 5 | * @since 2021/3/23 6 | */ 7 | public interface HashAlgorithm { 8 | 9 | /** 10 | * @param key to be hashed 11 | * @return hash value 12 | */ 13 | long hash(String key); 14 | } 15 | -------------------------------------------------------------------------------- /common/src/main/java/me/lawrenceli/utils/StringPattern.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.utils; 2 | 3 | /** 4 | * @author lawrence 5 | * @since 2022/3/5 6 | */ 7 | public class StringPattern { 8 | 9 | private StringPattern() { 10 | } 11 | 12 | public static String replacePatternBreaking(String string) { 13 | return string.replaceAll("[\n\r\t]", "_"); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | gateway-server: 4 | image: gateway:1.0.0 5 | hostname: gateway-server 6 | networks: 7 | - network 8 | ports: 9 | - "7000:7000" 10 | websocket-server: 11 | image: websocket:1.0.0 12 | hostname: websocket-serverr 13 | networks: 14 | - network 15 | networks: 16 | network: 17 | external: 18 | name: compose-network -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: up 2 | up: 3 | mvn clean 4 | mvn install -pl gateway -am -amd 5 | mvn install -pl websocket -am -amd 6 | docker build -t websocket:1.0.0 websocket/. 7 | docker build -t gateway:1.0.0 gateway/. 8 | docker-compose up -d 9 | docker ps 10 | 11 | .PHONY: down 12 | down: 13 | docker-compose down 14 | docker exec redis redis-cli flushall 15 | docker rmi $$(docker images | grep "none" | awk '{print $$3}') 16 | 17 | .PHONY: start 18 | start: 19 | docker-compose up 20 | 21 | .PHONY: new 22 | new: down up 23 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: maven 9 | directory: "/" 10 | schedule: 11 | interval: daily 12 | time: "21:00" 13 | open-pull-requests-limit: 10 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | .DS_Store 3 | target/ 4 | !.mvn/wrapper/maven-wrapper.jar 5 | !**/src/main/**/target/ 6 | !**/src/test/**/target/ 7 | 8 | ### STS ### 9 | .apt_generated 10 | .classpath 11 | .factorypath 12 | .project 13 | .settings 14 | .springBeans 15 | .sts4-cache 16 | 17 | ### IntelliJ IDEA ### 18 | .idea 19 | *.iws 20 | *.iml 21 | *.ipr 22 | 23 | ### NetBeans ### 24 | /nbproject/private/ 25 | /nbbuild/ 26 | /dist/ 27 | /nbdist/ 28 | /.nb-gradle/ 29 | build/ 30 | !**/src/main/**/build/ 31 | !**/src/test/**/build/ 32 | 33 | ### VS Code ### 34 | .vscode/ 35 | -------------------------------------------------------------------------------- /common/src/main/java/me/lawrenceli/model/MessageType.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.model; 2 | 3 | /** 4 | * WebSocket Message's Type enum 5 | * 6 | * @author lawrence 7 | * @since 2021/3/22 8 | */ 9 | public enum MessageType { 10 | 11 | // 用户向消息，纯业务侧消息类型 12 | FOR_USER { 13 | public Integer code() { 14 | return 1; 15 | } 16 | }, 17 | 18 | // 服务向消息，前端的服务端信息展示类型 19 | FOR_SERVER { 20 | public Integer code() { 21 | return 2; 22 | } 23 | }; 24 | 25 | public abstract Integer code(); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/GatewayApp.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.client.discovery.EnableDiscoveryClient; 6 | 7 | /** 8 | * @author lawrence 9 | * @since 2021/3/19 10 | */ 11 | @EnableDiscoveryClient 12 | @SpringBootApplication 13 | public class GatewayApp { 14 | public static void main(String[] args) { 15 | SpringApplication.run(GatewayApp.class, args); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /websocket/src/main/java/me/lawrenceli/websocket/WebSocketApp.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.client.discovery.EnableDiscoveryClient; 6 | 7 | /** 8 | * @author lawrence 9 | * @since 2021/3/19 10 | */ 11 | @SpringBootApplication 12 | @EnableDiscoveryClient 13 | public class WebSocketApp { 14 | 15 | public static void main(String[] args) { 16 | SpringApplication.run(WebSocketApp.class); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /websocket/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 7100 3 | shutdown: immediate 4 | 5 | logging: 6 | level: 7 | me.lawrenceli: debug 8 | 9 | spring: 10 | application: 11 | name: 12 | websocket-server 13 | redis: 14 | host: 'docker.for.mac.host.internal' 15 | port: 6379 16 | rabbitmq: 17 | host: 'docker.for.mac.host.internal' 18 | port: 5672 19 | username: admin 20 | password: admin 21 | cloud: 22 | nacos: 23 | discovery: 24 | server-addr: docker.for.mac.host.internal:8848 25 | namespace: 6e78ac6b-71bb-4611-994a-9b48140bf1ef -------------------------------------------------------------------------------- /websocket/src/main/java/me/lawrenceli/websocket/config/WebSocketConfig.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.config; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.web.socket.server.standard.ServerEndpointExporter; 6 | 7 | /** 8 | * @author lawrence 9 | * @since 2021/3/19 10 | */ 11 | @Configuration 12 | public class WebSocketConfig { 13 | 14 | @Bean("serverEndpointExporter") 15 | public ServerEndpointExporter serverEndpointExporter() { 16 | return new ServerEndpointExporter(); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /websocket/src/main/java/me/lawrenceli/websocket/server/FanoutReceiver.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.server; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.amqp.rabbit.annotation.RabbitListener; 6 | import org.springframework.stereotype.Component; 7 | 8 | import java.util.List; 9 | 10 | /** 11 | * @author lawrence 12 | * @since 2021/3/24 13 | */ 14 | @Component 15 | public class FanoutReceiver { 16 | 17 | private static final Logger log = LoggerFactory.getLogger(FanoutReceiver.class); 18 | 19 | @RabbitListener(queues = "#{queueForWebSocket.name}") 20 | public void receiver(List clientsToReset) { 21 | log.info("队列接收到了主动断掉服务端 WebSocket 连接的消息: [{}]", clientsToReset); 22 | WebSocketEndpoint.disconnectSomeByServer(clientsToReset); 23 | } 24 | 25 | } -------------------------------------------------------------------------------- /common/src/main/java/me/lawrenceli/constant/GlobalConstant.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.constant; 2 | 3 | /** 4 | * @author lawrence 5 | * @since 2021/3/23 6 | */ 7 | public class GlobalConstant { 8 | 9 | public static final String REDIS_TOPIC_CHANNEL = "server-change"; 10 | 11 | public static final String HASH_RING_REDIS = "ring"; 12 | 13 | public static final String SERVER_UP_MESSAGE = "UP"; 14 | 15 | public static final String SERVER_DOWN_MESSAGE = "DOWN"; 16 | 17 | public static final Integer VIRTUAL_COUNT = 10; 18 | 19 | public static final String KEY_TO_BE_HASHED = "userId"; 20 | 21 | public static final String FANOUT_EXCHANGE_NAME = "websocket-cluster-exchange"; 22 | 23 | public static final String WEBSOCKET_ENDPOINT_PATH = "/connect"; // NOSONAR 24 | 25 | private GlobalConstant() { 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /websocket/src/main/java/me/lawrenceli/websocket/spring/BeanUtils.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.spring; 2 | 3 | import org.springframework.beans.BeansException; 4 | import org.springframework.context.ApplicationContext; 5 | import org.springframework.context.ApplicationContextAware; 6 | import org.springframework.stereotype.Component; 7 | 8 | import javax.annotation.Nonnull; 9 | 10 | /** 11 | * @author lawrence 12 | * @since 2021/3/24 13 | */ 14 | @Component 15 | public class BeanUtils implements ApplicationContextAware { 16 | 17 | private static ApplicationContext applicationContext; 18 | 19 | @Override 20 | public void setApplicationContext(@Nonnull ApplicationContext applicationContext) throws BeansException { 21 | BeanUtils.applicationContext = applicationContext; // NOSONAR 22 | } 23 | 24 | public static Object getBean(String beanName) { 25 | return applicationContext.getBean(beanName); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /gateway/src/test/java/me/lawrenceli/gateway/config/NamingServiceConfigTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.config; 2 | 3 | import com.alibaba.nacos.client.naming.NacosNamingService; 4 | import org.junit.jupiter.api.BeforeEach; 5 | import org.junit.jupiter.api.Test; 6 | 7 | import static org.junit.jupiter.api.Assertions.assertNotNull; 8 | 9 | /** 10 | * @author lawrence 11 | * @since 2021/8/15 12 | */ 13 | class NamingServiceConfigTest { 14 | 15 | private NamingServiceConfig namingServiceConfig; 16 | 17 | @BeforeEach 18 | void setUp() { 19 | WebSocketProperties webSocketProperties = new WebSocketProperties(); 20 | webSocketProperties.setNacosServerAddress("127.0.0.1"); 21 | webSocketProperties.setNacosNamespace("test"); 22 | namingServiceConfig = new NamingServiceConfig(webSocketProperties); 23 | } 24 | 25 | @Test 26 | void getNamingService() { 27 | NacosNamingService namingService = namingServiceConfig.getNamingService(); 28 | assertNotNull(namingService); 29 | } 30 | } -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/server/FanoutSender.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.server; 2 | 3 | import me.lawrenceli.constant.GlobalConstant; 4 | import me.lawrenceli.utils.JSON; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.amqp.rabbit.core.RabbitTemplate; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.List; 11 | 12 | /** 13 | * @author lawrence 14 | * @since 2021/3/24 15 | */ 16 | @Component 17 | public class FanoutSender { 18 | 19 | private static final Logger log = LoggerFactory.getLogger(FanoutSender.class); 20 | 21 | private final RabbitTemplate rabbitTemplate; 22 | 23 | public FanoutSender(RabbitTemplate rabbitTemplate) { 24 | this.rabbitTemplate = rabbitTemplate; 25 | } 26 | 27 | public void send(List clientsToReset) { 28 | log.info("开始向所有 WebSocket 实例发送广播: [{}]", JSON.toJSONString(clientsToReset)); 29 | rabbitTemplate.convertAndSend(GlobalConstant.FANOUT_EXCHANGE_NAME, "", clientsToReset); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /gateway/src/main/resources/application.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 7000 3 | 4 | logging: 5 | level: 6 | me.lawrenceli: debug 7 | 8 | spring: 9 | application: 10 | name: 11 | gateway-server 12 | redis: 13 | host: 'docker.for.mac.host.internal' 14 | port: 6379 15 | rabbitmq: 16 | host: 'docker.for.mac.host.internal' 17 | port: 5672 18 | username: admin 19 | password: admin 20 | cloud: 21 | nacos: 22 | discovery: 23 | server-addr: docker.for.mac.host.internal:8848 24 | namespace: 6e78ac6b-71bb-4611-994a-9b48140bf1ef 25 | gateway: 26 | discovery: 27 | locator: 28 | enabled: true 29 | lower-case-service-id: true 30 | routes: 31 | - id: path-route 32 | uri: lb://websocket-server 33 | predicates: 34 | - Path=/websocket/** 35 | filters: 36 | - StripPrefix=1 37 | 38 | websocket: 39 | service: 40 | name: 'websocket-server' 41 | docker: 42 | network: 'compose-network' 43 | host: 'tcp://docker.for.mac.host.internal:6666' 44 | image: 45 | name: 'websocket:1.0.0' 46 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/server/ServiceNode.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.server; 2 | 3 | import me.lawrenceli.hashring.Node; 4 | 5 | import java.io.Serializable; 6 | import java.util.Objects; 7 | 8 | /** 9 | * 真实节点 10 | * 可以添加更多特性字段来区分不同的实例，如 DataCenter, Port 等等 11 | * 12 | * @author lawrence 13 | * @since 2021/3/23 14 | */ 15 | public class ServiceNode implements Node, Serializable { 16 | 17 | private static final long serialVersionUID = 5410221835105700427L; 18 | 19 | // 仅用 IP 来作为划分实例的依据 20 | private final String ip; 21 | 22 | public ServiceNode(String ip) { 23 | this.ip = ip; 24 | } 25 | 26 | @Override 27 | public String getKey() { 28 | return ip; 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | return getKey(); 34 | } 35 | 36 | @Override 37 | public boolean equals(Object o) { 38 | if (this == o) return true; 39 | if (o == null || getClass() != o.getClass()) return false; 40 | ServiceNode that = (ServiceNode) o; 41 | return Objects.equals(ip, that.ip); 42 | } 43 | 44 | @Override 45 | public int hashCode() { 46 | return Objects.hash(ip); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /common/src/main/java/me/lawrenceli/utils/JSON.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.utils; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | /** 9 | * @author lawrence 10 | * @since 2021/3/23 11 | */ 12 | public class JSON { 13 | 14 | private static final Logger logger = LoggerFactory.getLogger(JSON.class); 15 | 16 | public static String toJSONString(T object) { 17 | ObjectMapper mapper = new ObjectMapper(); 18 | String json = ""; 19 | try { 20 | json = mapper.writeValueAsString(object); 21 | } catch (JsonProcessingException e) { 22 | logger.error("将对象 {} 序列化为 JSON 失败", object); 23 | } 24 | return json; 25 | } 26 | 27 | public static T parseJSON(String json, Class clazz) { 28 | ObjectMapper mapper = new ObjectMapper(); 29 | T t = null; 30 | try { 31 | t = mapper.readValue(json, clazz); 32 | } catch (JsonProcessingException e) { 33 | logger.error("将字符 {} 反序列化为对象失败", json); 34 | } 35 | return t; 36 | } 37 | 38 | /** 39 | * 工具类私有构造 40 | */ 41 | private JSON() { 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/config/CorsConfig.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.config; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.web.cors.CorsConfiguration; 8 | import org.springframework.web.cors.reactive.CorsWebFilter; 9 | import org.springframework.web.cors.reactive.UrlBasedCorsConfigurationSource; 10 | import org.springframework.web.util.pattern.PathPatternParser; 11 | 12 | /** 13 | * @author lawrence 14 | * @since 2021/3/20 15 | */ 16 | @Configuration 17 | public class CorsConfig { 18 | 19 | private static final Logger logger = LoggerFactory.getLogger(CorsConfig.class); 20 | 21 | @Bean 22 | public CorsWebFilter corsFilter() { 23 | logger.debug("配置 CORS"); 24 | CorsConfiguration config = new CorsConfiguration(); 25 | config.addAllowedMethod("*"); 26 | config.addAllowedOrigin("*"); 27 | config.addAllowedHeader("*"); 28 | UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(new PathPatternParser()); 29 | source.registerCorsConfiguration("/**", config); 30 | return new CorsWebFilter(source); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/me/lawrenceli/hashring/VirtualNode.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.hashring; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * 真实节点的副本：虚拟节点 7 | * 8 | * @author lawrence 9 | * @since 2021/3/23 10 | */ 11 | public class VirtualNode implements Node, Serializable { 12 | 13 | private static final long serialVersionUID = -1066173071509622053L; 14 | 15 | // 真实节点 16 | final T physicalNode; // NOSONAR 17 | 18 | // 虚拟节点作为真实节点的副本，给它加个 -index 区分 19 | final Integer replicaIndex; 20 | 21 | public VirtualNode(T physicalNode, Integer replicaIndex) { 22 | this.physicalNode = physicalNode; 23 | this.replicaIndex = replicaIndex; 24 | } 25 | 26 | @Override 27 | public String getKey() { 28 | return physicalNode.getKey() + "-" + replicaIndex; 29 | } 30 | 31 | /** 32 | * 是否作为某个真实节点的虚拟副本节点 33 | * 34 | * @param anyPhysicalNode 任何真实节点 35 | * @return 是/否 36 | */ 37 | public boolean isVirtualOf(T anyPhysicalNode) { 38 | return anyPhysicalNode.getKey().equals(this.physicalNode.getKey()); 39 | } 40 | 41 | /** 42 | * 获取当前虚拟节点的真实节点 43 | * 44 | * @return 真实节点 45 | */ 46 | public T getPhysicalNode() { 47 | return this.physicalNode; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /gateway/src/test/java/me/lawrenceli/gateway/server/ServiceNodeTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.server; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import static org.junit.jupiter.api.Assertions.assertEquals; 6 | import static org.junit.jupiter.api.Assertions.assertNotEquals; 7 | 8 | class ServiceNodeTest { 9 | @Test 10 | void testConstructor() { 11 | assertEquals("127.0.0.1", (new ServiceNode("127.0.0.1")).getKey()); 12 | } 13 | 14 | @Test 15 | void testEquals() { 16 | assertNotEquals(null, new ServiceNode("127.0.0.1")); 17 | } 18 | 19 | @Test 20 | void testEquals2() { 21 | ServiceNode serviceNode = new ServiceNode("127.0.0.1"); 22 | assertEquals(serviceNode, serviceNode); 23 | int expectedHashCodeResult = serviceNode.hashCode(); 24 | assertEquals(expectedHashCodeResult, serviceNode.hashCode()); 25 | } 26 | 27 | @Test 28 | void testEquals3() { 29 | ServiceNode serviceNode = new ServiceNode("127.0.0.1"); 30 | ServiceNode serviceNode1 = new ServiceNode("127.0.0.1"); 31 | assertEquals(serviceNode, serviceNode1); 32 | int expectedHashCodeResult = serviceNode.hashCode(); 33 | assertEquals(expectedHashCodeResult, serviceNode1.hashCode()); 34 | } 35 | 36 | @Test 37 | void testEquals4() { 38 | ServiceNode serviceNode = new ServiceNode("Ip"); 39 | assertNotEquals(serviceNode, new ServiceNode("127.0.0.1")); 40 | } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/config/RedisConfig.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.config; 2 | 3 | import me.lawrenceli.constant.GlobalConstant; 4 | import me.lawrenceli.gateway.server.RedisSubscriber; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.data.redis.connection.RedisConnectionFactory; 8 | import org.springframework.data.redis.listener.PatternTopic; 9 | import org.springframework.data.redis.listener.RedisMessageListenerContainer; 10 | import org.springframework.data.redis.listener.adapter.MessageListenerAdapter; 11 | 12 | /** 13 | * Redis pub/sub 配置 14 | * 15 | * @author lawrence 16 | * @since 2021/3/23 17 | */ 18 | @Configuration 19 | public class RedisConfig { 20 | 21 | @Bean 22 | MessageListenerAdapter listenerAdapter(RedisSubscriber redisSubscriber) { 23 | // 该构造的第二个参数是 消费方的方法名字符串 24 | return new MessageListenerAdapter(redisSubscriber); 25 | } 26 | 27 | @Bean 28 | RedisMessageListenerContainer container(RedisConnectionFactory connectionFactory, 29 | MessageListenerAdapter listenerAdapter) { 30 | RedisMessageListenerContainer container = new RedisMessageListenerContainer(); 31 | container.setConnectionFactory(connectionFactory); 32 | container.addMessageListener(listenerAdapter, new PatternTopic(GlobalConstant.REDIS_TOPIC_CHANNEL)); 33 | return container; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /.github/workflows/sonar.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | types: [ opened, synchronize, reopened ] 8 | jobs: 9 | build: 10 | name: Build 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | with: 15 | fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis 16 | - name: Set up JDK 11 17 | uses: actions/setup-java@v1 18 | with: 19 | java-version: 11 20 | - name: Cache SonarCloud packages 21 | uses: actions/cache@v1 22 | with: 23 | path: ~/.sonar/cache 24 | key: ${{ runner.os }}-sonar 25 | restore-keys: ${{ runner.os }}-sonar 26 | - name: Cache Maven packages 27 | uses: actions/cache@v1 28 | with: 29 | path: ~/.m2 30 | key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} 31 | restore-keys: ${{ runner.os }}-m2 32 | - name: Build and analyze 33 | env: 34 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any 35 | SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} 36 | run: | 37 | mvn -v 38 | if [ "${SONAR_TOKEN}" != "" ]; then 39 | mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=Lonor_websocket-cluster 40 | bash <(curl -s https://codecov.io/bash) 41 | else 42 | mvn -B verify 43 | fi -------------------------------------------------------------------------------- /websocket/src/test/java/me/lawrenceli/websocket/event/ServerDownEventHandlerTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.event; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.junit.jupiter.api.extension.ExtendWith; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.boot.test.mock.mockito.MockBean; 7 | import org.springframework.boot.web.reactive.context.AnnotationConfigReactiveWebApplicationContext; 8 | import org.springframework.context.event.ContextClosedEvent; 9 | import org.springframework.data.redis.core.StringRedisTemplate; 10 | import org.springframework.test.context.ContextConfiguration; 11 | import org.springframework.test.context.junit.jupiter.SpringExtension; 12 | 13 | import static org.mockito.Mockito.any; 14 | import static org.mockito.Mockito.doNothing; 15 | import static org.mockito.Mockito.verify; 16 | 17 | @ContextConfiguration(classes = {ServerDownEventHandler.class}) 18 | @ExtendWith(SpringExtension.class) 19 | class ServerDownEventHandlerTest { 20 | @Autowired 21 | private ServerDownEventHandler serverDownEventHandler; 22 | 23 | @MockBean 24 | private StringRedisTemplate stringRedisTemplate; 25 | 26 | @Test 27 | void testOnApplicationEvent() { 28 | doNothing().when(this.stringRedisTemplate).convertAndSend(any(), any()); 29 | this.serverDownEventHandler 30 | .onApplicationEvent(new ContextClosedEvent(new AnnotationConfigReactiveWebApplicationContext())); 31 | verify(this.stringRedisTemplate).convertAndSend(any(), any()); 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /websocket/src/main/java/me/lawrenceli/websocket/config/MQConfig.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.config; 2 | 3 | import me.lawrenceli.constant.GlobalConstant; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.amqp.core.AnonymousQueue; 7 | import org.springframework.amqp.core.Binding; 8 | import org.springframework.amqp.core.BindingBuilder; 9 | import org.springframework.amqp.core.FanoutExchange; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.context.annotation.Configuration; 12 | 13 | /** 14 | * @author lawrence 15 | * @since 2021/3/24 16 | */ 17 | @Configuration 18 | public class MQConfig { 19 | 20 | private static final Logger log = LoggerFactory.getLogger(MQConfig.class); 21 | 22 | @Bean 23 | public FanoutExchange fanoutExchange() { 24 | log.info("创建广播交换机 [{}]", GlobalConstant.FANOUT_EXCHANGE_NAME); 25 | return new FanoutExchange(GlobalConstant.FANOUT_EXCHANGE_NAME); 26 | } 27 | 28 | @Bean 29 | public AnonymousQueue queueForWebSocket() { 30 | log.info("创建用于 WebSocket 的匿名队列"); 31 | return new AnonymousQueue(); 32 | } 33 | 34 | /** 35 | * @param fanoutExchange 交换机 36 | * @param queueForWebSocket 队列 37 | * @return Binding 38 | */ 39 | @Bean 40 | public Binding bindingSingle(FanoutExchange fanoutExchange, AnonymousQueue queueForWebSocket) { 41 | log.info("把队列 [{}] 绑定到广播交换器 [{}]", queueForWebSocket.getName(), fanoutExchange.getName()); 42 | return BindingBuilder.bind(queueForWebSocket).to(fanoutExchange); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/config/NamingServiceConfig.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.config; 2 | 3 | import com.alibaba.nacos.api.exception.NacosException; 4 | import com.alibaba.nacos.client.naming.NacosNamingService; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | 10 | import java.util.Properties; 11 | 12 | /** 13 | * 该 Bean 并没有使用，而是用的 Docker API 来获取实例信息。 14 | * 15 | * @author lawrence 16 | * @since 2021/3/21 17 | */ 18 | @Configuration 19 | public class NamingServiceConfig { 20 | 21 | private static final Logger logger = LoggerFactory.getLogger(NamingServiceConfig.class); 22 | 23 | private final WebSocketProperties webSocketProperties; 24 | 25 | public NamingServiceConfig(WebSocketProperties webSocketProperties) { 26 | this.webSocketProperties = webSocketProperties; 27 | } 28 | 29 | @Bean("namingService") 30 | public NacosNamingService getNamingService() { 31 | logger.info("注入 Nacos ({}) 名称服务", webSocketProperties.getNacosServerAddress()); 32 | NacosNamingService namingService = null; 33 | Properties properties = new Properties(); 34 | properties.put("namespace", webSocketProperties.getNacosNamespace()); 35 | properties.put("serverAddr", webSocketProperties.getNacosServerAddress()); 36 | try { 37 | namingService = new NacosNamingService(properties); 38 | } catch (NacosException e) { 39 | logger.error("NacosNamingService 创建异常: {}", e.toString()); 40 | } 41 | return namingService; 42 | } 43 | } -------------------------------------------------------------------------------- /websocket/src/test/java/me/lawrenceli/websocket/event/ServerUpEventHandlerTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.event; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.junit.jupiter.api.extension.ExtendWith; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.context.event.ApplicationReadyEvent; 8 | import org.springframework.boot.test.mock.mockito.MockBean; 9 | import org.springframework.boot.web.reactive.context.AnnotationConfigReactiveWebApplicationContext; 10 | import org.springframework.data.redis.core.StringRedisTemplate; 11 | import org.springframework.test.context.ContextConfiguration; 12 | import org.springframework.test.context.junit.jupiter.SpringExtension; 13 | 14 | import static org.mockito.Mockito.any; 15 | import static org.mockito.Mockito.doNothing; 16 | import static org.mockito.Mockito.verify; 17 | 18 | @ContextConfiguration(classes = {ServerUpEventHandler.class}) 19 | @ExtendWith(SpringExtension.class) 20 | class ServerUpEventHandlerTest { 21 | @Autowired 22 | private ServerUpEventHandler serverUpEventHandler; 23 | 24 | @MockBean 25 | private StringRedisTemplate stringRedisTemplate; 26 | 27 | @Test 28 | void testOnApplicationEvent() { 29 | doNothing().when(this.stringRedisTemplate).convertAndSend(any(), any()); 30 | SpringApplication application = new SpringApplication(Object.class); 31 | this.serverUpEventHandler.onApplicationEvent(new ApplicationReadyEvent(application, new String[]{"Args"}, 32 | new AnnotationConfigReactiveWebApplicationContext())); 33 | verify(this.stringRedisTemplate).convertAndSend(any(), any()); 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /websocket/src/main/java/me/lawrenceli/websocket/controller/WebSocketController.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.controller; 2 | 3 | import me.lawrenceli.model.MessageType; 4 | import me.lawrenceli.model.WebSocketMessage; 5 | import me.lawrenceli.websocket.server.WebSocketEndpoint; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.http.ResponseEntity; 9 | import org.springframework.web.bind.annotation.GetMapping; 10 | import org.springframework.web.bind.annotation.RequestMapping; 11 | import org.springframework.web.bind.annotation.RequestParam; 12 | import org.springframework.web.bind.annotation.RestController; 13 | 14 | /** 15 | * @author lawrence 16 | * @since 2021/3/19 17 | */ 18 | @RestController 19 | @RequestMapping("/") 20 | public class WebSocketController { 21 | 22 | private static final Logger logger = LoggerFactory.getLogger(WebSocketController.class); 23 | 24 | final WebSocketEndpoint webSocketEndpoint; 25 | 26 | public WebSocketController(WebSocketEndpoint webSocketEndpoint) { 27 | this.webSocketEndpoint = webSocketEndpoint; 28 | } 29 | 30 | @GetMapping("/send") 31 | public ResponseEntity send(@RequestParam String userId, @RequestParam String message) { 32 | WebSocketMessage webSocketMessage = WebSocketMessage.toUserOrServerMessage(MessageType.FOR_USER, message, WebSocketEndpoint.sessionMap.size()); 33 | return ResponseEntity.ok(webSocketEndpoint.sendMessageToUser(userId, webSocketMessage)); 34 | } 35 | 36 | @GetMapping("/count") 37 | public ResponseEntity count() { 38 | logger.info("当前 session 连接: {}", WebSocketEndpoint.sessionMap); 39 | return ResponseEntity.ok(WebSocketEndpoint.sessionMap.size()); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /websocket/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | springcloud-websocket-cluster 7 | me.lawrenceli 8 | 1.0.0 9 | 10 | 4.0.0 11 | websocket 12 | 13 | 14 | 8 15 | 8 16 | 17 | 18 | 19 | 20 | me.lawrenceli 21 | common 22 | 1.0.0 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-websocket 31 | 32 | 33 | 34 | 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-maven-plugin 39 | 40 | 41 | 42 | repackage 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | me.lawrenceli 8 | springcloud-websocket-cluster 9 | 1.0.0 10 | 11 | gateway 12 | websocket 13 | common 14 | 15 | pom 16 | 17 | 18 | Hoxton.SR10 19 | 2.3.9.RELEASE 20 | 8 21 | 8 22 | lonor 23 | https://sonarcloud.io 24 | 25 | 26 | 27 | 28 | 29 | org.springframework.cloud 30 | spring-cloud-dependencies 31 | ${spring-cloud.version} 32 | pom 33 | import 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-dependencies 38 | ${spring-boot.version} 39 | pom 40 | import 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /websocket/src/main/java/me/lawrenceli/websocket/event/ServerDownEventHandler.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.event; 2 | 3 | import me.lawrenceli.constant.GlobalConstant; 4 | import me.lawrenceli.model.MessageType; 5 | import me.lawrenceli.model.WebSocketMessage; 6 | import me.lawrenceli.utils.JSON; 7 | import me.lawrenceli.websocket.server.WebSocketEndpoint; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.context.ApplicationListener; 11 | import org.springframework.context.event.ContextClosedEvent; 12 | import org.springframework.data.redis.core.StringRedisTemplate; 13 | import org.springframework.stereotype.Component; 14 | 15 | /** 16 | * 服务下线事件处理 17 | * 18 | * @author lawrence 19 | * @since 2021/3/23 20 | */ 21 | @Component 22 | public class ServerDownEventHandler implements ApplicationListener { 23 | 24 | private static final Logger logger = LoggerFactory.getLogger(ServerDownEventHandler.class); 25 | 26 | final StringRedisTemplate stringRedisTemplate; 27 | 28 | public ServerDownEventHandler(StringRedisTemplate stringRedisTemplate) { 29 | this.stringRedisTemplate = stringRedisTemplate; 30 | } 31 | 32 | @Override 33 | public void onApplicationEvent(ContextClosedEvent contextClosedEvent) { 34 | logger.debug("当前 WebSocket 实例 - 准备下线 {}", contextClosedEvent.getApplicationContext().getDisplayName()); 35 | logger.info("Redis 发布服务下线消息，通知网关移除相关节点"); 36 | stringRedisTemplate.convertAndSend(GlobalConstant.REDIS_TOPIC_CHANNEL, 37 | JSON.toJSONString(WebSocketMessage.toUserOrServerMessage( 38 | MessageType.FOR_SERVER, GlobalConstant.SERVER_DOWN_MESSAGE, WebSocketEndpoint.sessionMap.size())) 39 | ); 40 | logger.info("服务实例开始主动断开所有 WebSocket 连接..."); 41 | WebSocketEndpoint.disconnectAllByServer(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/discovery/DiscoveryController.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.discovery; 2 | 3 | import com.alibaba.nacos.api.exception.NacosException; 4 | import com.alibaba.nacos.api.naming.NamingService; 5 | import com.alibaba.nacos.api.naming.pojo.Instance; 6 | import me.lawrenceli.gateway.config.WebSocketProperties; 7 | import org.springframework.http.ResponseEntity; 8 | import org.springframework.web.bind.annotation.GetMapping; 9 | import org.springframework.web.bind.annotation.RequestMapping; 10 | import org.springframework.web.bind.annotation.RestController; 11 | 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | /** 17 | * 服务发现的端点 18 | * 19 | * @author lawrence 20 | * @since 2021/4/1 21 | */ 22 | @RestController 23 | @RequestMapping("/discovery") 24 | public class DiscoveryController { 25 | 26 | final NamingService namingService; 27 | final WebSocketProperties webSocketProperties; 28 | 29 | public DiscoveryController(NamingService namingService, 30 | WebSocketProperties webSocketProperties) { 31 | this.namingService = namingService; 32 | this.webSocketProperties = webSocketProperties; 33 | } 34 | 35 | /** 36 | * 前端以短轮询的方式请求，获取 WebSocket 服务运行状态 37 | * 38 | * @return false: 服务正在启动中或未注册到 Nacos，true: 服务正常，可接收请求 39 | * @throws NacosException Nacos 异常 40 | */ 41 | @GetMapping("/naming") 42 | public ResponseEntity> getServerStatus() throws NacosException { 43 | List allWebSocketInstances = namingService.getAllInstances(webSocketProperties.getService().getName()); 44 | HashMap ipAndStatus = new HashMap<>(4); 45 | for (Instance instance : allWebSocketInstances) { 46 | ipAndStatus.put(instance.getIp(), instance.isEnabled() && instance.isHealthy()); 47 | } 48 | return ResponseEntity.ok(ipAndStatus); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gateway/src/test/java/me/lawrenceli/gateway/filter/CustomLoadBalanceFilterTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.filter; 2 | 3 | import com.alibaba.cloud.nacos.NacosDiscoveryProperties; 4 | import com.alibaba.cloud.nacos.NacosServiceManager; 5 | import com.alibaba.cloud.nacos.discovery.NacosDiscoveryClient; 6 | import com.alibaba.cloud.nacos.discovery.NacosServiceDiscovery; 7 | import me.lawrenceli.gateway.config.WebSocketProperties; 8 | import me.lawrenceli.gateway.server.ServiceNode; 9 | import me.lawrenceli.hashring.ConsistentHashRouter; 10 | import org.junit.jupiter.api.Test; 11 | import org.springframework.cloud.gateway.config.LoadBalancerProperties; 12 | import org.springframework.cloud.loadbalancer.blocking.client.BlockingLoadBalancerClient; 13 | import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory; 14 | 15 | import java.util.ArrayList; 16 | 17 | import static org.junit.jupiter.api.Assertions.assertTrue; 18 | 19 | @SuppressWarnings("deprecation") 20 | class CustomLoadBalanceFilterTest { 21 | @Test 22 | void testConstructor() { 23 | BlockingLoadBalancerClient loadBalancer = new BlockingLoadBalancerClient(new LoadBalancerClientFactory()); 24 | 25 | LoadBalancerProperties loadBalancerProperties = new LoadBalancerProperties(); 26 | loadBalancerProperties.setUse404(true); 27 | ConsistentHashRouter consistentHashRouter = new ConsistentHashRouter<>(new ArrayList<>(), 3); 28 | 29 | WebSocketProperties webSocketProperties = new WebSocketProperties(); 30 | NacosDiscoveryProperties discoveryProperties = new NacosDiscoveryProperties(); 31 | assertTrue( 32 | (new CustomLoadBalanceFilter(loadBalancer, loadBalancerProperties, consistentHashRouter, webSocketProperties, 33 | new NacosDiscoveryClient( 34 | new NacosServiceDiscovery(discoveryProperties, new NacosServiceManager())))).webSocketProperties 35 | .isEmpty()); 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /common/src/test/java/me/lawrenceli/hashring/ConsistentHashRouterTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.hashring; 2 | 3 | import org.junit.jupiter.api.BeforeEach; 4 | import org.junit.jupiter.api.Test; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.SortedMap; 9 | 10 | import static me.lawrenceli.constant.GlobalConstant.VIRTUAL_COUNT; 11 | import static org.junit.jupiter.api.Assertions.assertEquals; 12 | import static org.junit.jupiter.api.Assertions.assertNull; 13 | import static org.junit.jupiter.api.Assertions.assertSame; 14 | 15 | /** 16 | * @author lawrence 17 | * @since 2021/8/14 18 | */ 19 | class ConsistentHashRouterTest { 20 | 21 | private ConsistentHashRouter consistentHashRouter; 22 | 23 | @BeforeEach 24 | void setUp() { 25 | List nodes = new ArrayList<>(); 26 | nodes.add(() -> "1"); 27 | consistentHashRouter = new ConsistentHashRouter<>(nodes, VIRTUAL_COUNT); 28 | } 29 | 30 | @Test 31 | void addNode() { 32 | consistentHashRouter.addNode(() -> "2", VIRTUAL_COUNT); 33 | assertSame(VIRTUAL_COUNT, consistentHashRouter.getVirtualNodeCountOf(() -> "2")); 34 | assertEquals(2 * VIRTUAL_COUNT, consistentHashRouter.getRing().size()); 35 | } 36 | 37 | @Test 38 | void getRing() { 39 | SortedMap> ring = consistentHashRouter.getRing(); 40 | assertEquals(VIRTUAL_COUNT, ring.size()); 41 | } 42 | 43 | @Test 44 | void routeNode() { 45 | Node routeNode = consistentHashRouter.routeNode("1"); 46 | assertEquals("1", routeNode.getKey()); 47 | } 48 | 49 | @Test 50 | void removeNode() { 51 | consistentHashRouter.removeNode(() -> "1"); 52 | Node node = consistentHashRouter.routeNode("1"); 53 | assertNull(node); 54 | } 55 | 56 | @Test 57 | void getVirtualNodeCountOf() { 58 | Integer virtualNodeCount = consistentHashRouter.getVirtualNodeCountOf(() -> "1"); 59 | assertEquals(VIRTUAL_COUNT, virtualNodeCount); 60 | } 61 | } -------------------------------------------------------------------------------- /common/src/test/java/me/lawrenceli/utils/JSONTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.utils; 2 | 3 | import me.lawrenceli.model.MessageType; 4 | import me.lawrenceli.model.WebSocketMessage; 5 | import org.junit.jupiter.api.BeforeEach; 6 | import org.junit.jupiter.api.Test; 7 | 8 | import java.lang.reflect.Field; 9 | import java.lang.reflect.Modifier; 10 | import java.util.Date; 11 | 12 | import static org.junit.jupiter.api.Assertions.*; 13 | 14 | /** 15 | * @author lawrence 16 | * @since 2021/8/15 17 | */ 18 | class JSONTest { 19 | 20 | private WebSocketMessage testMessage; 21 | 22 | @BeforeEach 23 | void setUp() { 24 | WebSocketMessage webSocketMessage = new WebSocketMessage(); 25 | webSocketMessage.setContent("content"); 26 | webSocketMessage.setTimestamp(new Date()); 27 | webSocketMessage.setType(MessageType.FOR_USER.code()); 28 | webSocketMessage.setServerIp("0.0.0.0"); 29 | webSocketMessage.setServerUserCount(1); 30 | testMessage = webSocketMessage; 31 | } 32 | 33 | @Test 34 | void toJSONString() { 35 | String message = JSON.toJSONString(testMessage); 36 | Field[] declaredFields = WebSocketMessage.class.getDeclaredFields(); 37 | for (Field field : declaredFields) { 38 | int modifier = field.getModifiers(); 39 | if (!Modifier.isStatic(modifier)) { 40 | assertTrue(message.contains(field.getName())); 41 | } 42 | } 43 | } 44 | 45 | @Test 46 | void parseJSON() { 47 | final String json = "{\"type\":1,\"content\":\"content\",\"serverUserCount\":1,\"serverIp\":\"0.0.0.0\",\"timestamp\":1628997609392}"; 48 | WebSocketMessage message = JSON.parseJSON(json, WebSocketMessage.class); 49 | assertEquals(message.getType(), testMessage.getType()); 50 | assertEquals(message.getContent(), testMessage.getContent()); 51 | assertEquals(message.getServerIp(), testMessage.getServerIp()); 52 | assertEquals(message.getServerUserCount(), testMessage.getServerUserCount()); 53 | } 54 | } -------------------------------------------------------------------------------- /websocket/src/main/java/me/lawrenceli/websocket/event/ServerUpEventHandler.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.event; 2 | 3 | import me.lawrenceli.constant.GlobalConstant; 4 | import me.lawrenceli.model.MessageType; 5 | import me.lawrenceli.model.WebSocketMessage; 6 | import me.lawrenceli.utils.JSON; 7 | import me.lawrenceli.websocket.server.WebSocketEndpoint; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.boot.context.event.ApplicationReadyEvent; 11 | import org.springframework.context.ApplicationListener; 12 | import org.springframework.data.redis.core.StringRedisTemplate; 13 | import org.springframework.stereotype.Component; 14 | 15 | import javax.annotation.Nonnull; 16 | 17 | /** 18 | * 服务上线事件处理 19 | * 20 | * @author lawrence 21 | * @since 2021/3/21 22 | */ 23 | @Component 24 | public class ServerUpEventHandler implements ApplicationListener { 25 | 26 | private static final Logger logger = LoggerFactory.getLogger(ServerUpEventHandler.class); 27 | 28 | final StringRedisTemplate stringRedisTemplate; 29 | 30 | public ServerUpEventHandler(StringRedisTemplate stringRedisTemplate) { 31 | this.stringRedisTemplate = stringRedisTemplate; 32 | } 33 | 34 | @Override 35 | public void onApplicationEvent(@Nonnull ApplicationReadyEvent applicationReadyEvent) { 36 | logger.debug("当前 WebSocket 实例 - 准备就绪，即将发布上线消息. {}", applicationReadyEvent); 37 | try { 38 | // Sleep 是为了确保该实例 100% 准备好了 39 | Thread.sleep(5000); 40 | } catch (InterruptedException ignore) { 41 | Thread.currentThread().interrupt(); 42 | } 43 | logger.info("WebSocket 实例通过 Redis 发布服务上线消息：通知网关更新哈希环"); 44 | // 这里消息内容复用面向客户端的实体类 45 | stringRedisTemplate.convertAndSend(GlobalConstant.REDIS_TOPIC_CHANNEL, 46 | JSON.toJSONString(WebSocketMessage.toUserOrServerMessage( 47 | MessageType.FOR_SERVER, GlobalConstant.SERVER_UP_MESSAGE, WebSocketEndpoint.sessionMap.size())) 48 | ); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /common/src/test/java/me/lawrenceli/hashring/VirtualNodeTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.hashring; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import static org.junit.jupiter.api.Assertions.assertEquals; 6 | import static org.junit.jupiter.api.Assertions.assertFalse; 7 | import static org.junit.jupiter.api.Assertions.assertSame; 8 | import static org.junit.jupiter.api.Assertions.assertTrue; 9 | import static org.mockito.Mockito.mock; 10 | import static org.mockito.Mockito.verify; 11 | import static org.mockito.Mockito.when; 12 | 13 | class VirtualNodeTest { 14 | @Test 15 | void testConstructor() { 16 | VirtualNode actualVirtualNode = new VirtualNode<>(mock(Node.class), 1); 17 | 18 | Node expectedPhysicalNode = actualVirtualNode.physicalNode; 19 | assertSame(expectedPhysicalNode, actualVirtualNode.getPhysicalNode()); 20 | assertEquals(1, actualVirtualNode.replicaIndex.intValue()); 21 | } 22 | 23 | @Test 24 | void testGetKey() { 25 | Node node = mock(Node.class); 26 | when(node.getKey()).thenReturn("Key"); 27 | assertEquals("Key-1", (new VirtualNode<>(node, 1)).getKey()); 28 | verify(node).getKey(); 29 | } 30 | 31 | @Test 32 | void testIsVirtualOf() { 33 | Node node = mock(Node.class); 34 | when(node.getKey()).thenReturn("Key"); 35 | VirtualNode virtualNode = new VirtualNode<>(node, 1); 36 | Node node1 = mock(Node.class); 37 | when(node1.getKey()).thenReturn("Key"); 38 | assertTrue(virtualNode.isVirtualOf(node1)); 39 | verify(node).getKey(); 40 | verify(node1).getKey(); 41 | } 42 | 43 | @Test 44 | void testIsVirtualOf2() { 45 | Node node = mock(Node.class); 46 | when(node.getKey()).thenReturn("foo"); 47 | VirtualNode virtualNode = new VirtualNode<>(node, 1); 48 | Node node1 = mock(Node.class); 49 | when(node1.getKey()).thenReturn("Key"); 50 | assertFalse(virtualNode.isVirtualOf(node1)); 51 | verify(node).getKey(); 52 | verify(node1).getKey(); 53 | } 54 | } 55 | 56 | -------------------------------------------------------------------------------- /gateway/src/test/java/me/lawrenceli/gateway/docker/DockerControllerTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.docker; 2 | 3 | import com.github.dockerjava.api.model.Container; 4 | import me.lawrenceli.gateway.config.WebSocketProperties; 5 | import org.junit.jupiter.api.Test; 6 | import org.springframework.http.HttpStatus; 7 | import org.springframework.http.ResponseEntity; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | import static org.junit.jupiter.api.Assertions.assertEquals; 13 | import static org.junit.jupiter.api.Assertions.assertTrue; 14 | import static org.mockito.Mockito.any; 15 | import static org.mockito.Mockito.doNothing; 16 | import static org.mockito.Mockito.mock; 17 | import static org.mockito.Mockito.verify; 18 | import static org.mockito.Mockito.when; 19 | 20 | class DockerControllerTest { 21 | @Test 22 | void testPs() { 23 | DockerService dockerService = mock(DockerService.class); 24 | when(dockerService.ps(any())).thenReturn(new ArrayList<>()); 25 | ResponseEntity> actualPsResult = (new DockerController(dockerService, new WebSocketProperties())) 26 | .ps("Container Name"); 27 | assertTrue(actualPsResult.hasBody()); 28 | assertEquals(HttpStatus.OK, actualPsResult.getStatusCode()); 29 | assertTrue(actualPsResult.getHeaders().isEmpty()); 30 | verify(dockerService).ps(any()); 31 | } 32 | 33 | @Test 34 | void testStopAndRemove() { 35 | DockerService dockerService = mock(DockerService.class); 36 | doNothing().when(dockerService).removeContainer(any()); 37 | doNothing().when(dockerService).stopContainer(any()); 38 | ResponseEntity actualStopAndRemoveResult = (new DockerController(dockerService, new WebSocketProperties())) 39 | .stopAndRemove("42"); 40 | assertEquals("42", actualStopAndRemoveResult.getBody()); 41 | assertEquals(HttpStatus.OK, actualStopAndRemoveResult.getStatusCode()); 42 | assertTrue(actualStopAndRemoveResult.getHeaders().isEmpty()); 43 | verify(dockerService).removeContainer(any()); 44 | verify(dockerService).stopContainer(any()); 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/docker/DockerController.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.docker; 2 | 3 | import com.github.dockerjava.api.model.Container; 4 | import me.lawrenceli.gateway.config.WebSocketProperties; 5 | import org.springframework.http.HttpStatus; 6 | import org.springframework.http.ResponseEntity; 7 | import org.springframework.web.bind.annotation.GetMapping; 8 | import org.springframework.web.bind.annotation.RequestMapping; 9 | import org.springframework.web.bind.annotation.RequestParam; 10 | import org.springframework.web.bind.annotation.RestController; 11 | 12 | import java.util.List; 13 | 14 | /** 15 | * @author lawrence 16 | * @since 2021/3/19 17 | */ 18 | @RestController 19 | @RequestMapping("/docker") 20 | public class DockerController { 21 | 22 | final DockerService dockerService; 23 | final WebSocketProperties webSocketProperties; 24 | 25 | public DockerController(DockerService dockerService, WebSocketProperties webSocketProperties) { 26 | this.dockerService = dockerService; 27 | this.webSocketProperties = webSocketProperties; 28 | } 29 | 30 | @GetMapping("/ps") 31 | public ResponseEntity> ps(@RequestParam(required = false) String containerName) { 32 | return ResponseEntity.ok(dockerService.ps(containerName)); 33 | } 34 | 35 | @GetMapping("/run") 36 | public ResponseEntity createAndRun() { 37 | List containerListOfWebSocketServices = dockerService.ps(webSocketProperties.getService().getName()); 38 | if (containerListOfWebSocketServices.size() < 5) { 39 | String imageName = webSocketProperties.getDocker().getImage().getName(); 40 | String id = dockerService.createContainer(imageName).getId(); 41 | dockerService.runContainer(id); 42 | return ResponseEntity.ok(id); 43 | } 44 | return ResponseEntity.status(HttpStatus.FORBIDDEN).body("WebSocket 实例数量达到上限"); 45 | } 46 | 47 | @GetMapping("/rm") 48 | public ResponseEntity stopAndRemove(@RequestParam String containerId) { 49 | dockerService.stopContainer(containerId); 50 | dockerService.removeContainer(containerId); 51 | return ResponseEntity.ok(containerId); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /common/src/test/java/me/lawrenceli/model/WebSocketMessageTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.model; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import java.util.ArrayList; 6 | import java.util.HashSet; 7 | import java.util.List; 8 | import java.util.Set; 9 | import java.util.regex.Matcher; 10 | import java.util.regex.Pattern; 11 | 12 | import static org.junit.jupiter.api.Assertions.assertEquals; 13 | import static org.junit.jupiter.api.Assertions.assertTrue; 14 | 15 | class WebSocketMessageTest { 16 | 17 | @Test 18 | void testToUserOrServerMessage() { 19 | WebSocketMessage actualToUserOrServerMessageResult = WebSocketMessage.toUserOrServerMessage(MessageType.FOR_USER, 20 | "Not all who wander are lost", 3); 21 | assertEquals("Not all who wander are lost", actualToUserOrServerMessageResult.getContent()); 22 | assertEquals(1, actualToUserOrServerMessageResult.getType().intValue()); 23 | assertEquals(3, actualToUserOrServerMessageResult.getServerUserCount().intValue()); 24 | assertTrue(WebSocketMessageTest.ipIsInner(actualToUserOrServerMessageResult.getServerIp())); 25 | } 26 | 27 | @Test 28 | void testToUserOrServerMessage2() { 29 | WebSocketMessage actualToUserOrServerMessageResult = WebSocketMessage.toUserOrServerMessage(MessageType.FOR_SERVER, 30 | "Not all who wander are lost", 3); 31 | assertEquals("Not all who wander are lost", actualToUserOrServerMessageResult.getContent()); 32 | assertEquals(2, actualToUserOrServerMessageResult.getType().intValue()); 33 | assertEquals(3, actualToUserOrServerMessageResult.getServerUserCount().intValue()); 34 | assertTrue(WebSocketMessageTest.ipIsInner(actualToUserOrServerMessageResult.getServerIp())); 35 | } 36 | 37 | /** 38 | * 私有 IP 39 | * A 类 10.0.0.0-10.255.255.255 40 | * B 类 172.16.0.0-172.31.255.255 41 | * C 类 192.168.0.0-192.168.255.255 42 | * 127 这个网段是环回地址 localhost 43 | */ 44 | static List ipFilterRegexList = new ArrayList<>(); 45 | 46 | static { 47 | Set ipFilter = new HashSet<>(); 48 | // A 类 49 | ipFilter.add("^10\\.(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|[0-9])" 50 | + "\\.(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|[0-9])" + "\\.(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|[0-9])$"); 51 | // B 类 52 | ipFilter.add("^172\\.(1[6789]|2[0-9]|3[01])\\" + ".(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|[0-9])\\" 53 | + ".(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|[0-9])$"); 54 | // C 类 55 | ipFilter.add("^192\\.168\\.(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|[0-9])\\" 56 | + ".(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|[0-9])$"); 57 | ipFilter.add("127.0.0.1"); 58 | ipFilter.add("0.0.0.0"); 59 | ipFilter.add("localhost"); 60 | for (String tmp : ipFilter) { 61 | ipFilterRegexList.add(Pattern.compile(tmp)); 62 | } 63 | } 64 | 65 | private static boolean ipIsInner(String ip) { 66 | boolean isInnerIp = false; 67 | for (Pattern tmp : ipFilterRegexList) { 68 | Matcher matcher = tmp.matcher(ip); 69 | if (matcher.find()) { 70 | isInnerIp = true; 71 | break; 72 | } 73 | } 74 | return isInnerIp; 75 | } 76 | } 77 | 78 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/filter/CustomLoadBalanceFilter.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.filter; 2 | 3 | import me.lawrenceli.gateway.config.WebSocketProperties; 4 | import me.lawrenceli.gateway.server.ServiceNode; 5 | import me.lawrenceli.hashring.ConsistentHashRouter; 6 | import org.springframework.beans.factory.config.BeanPostProcessor; 7 | import org.springframework.cloud.client.ServiceInstance; 8 | import org.springframework.cloud.client.discovery.DiscoveryClient; 9 | import org.springframework.cloud.client.loadbalancer.LoadBalancerClient; 10 | import org.springframework.cloud.gateway.config.LoadBalancerProperties; 11 | import org.springframework.cloud.gateway.filter.LoadBalancerClientFilter; 12 | import org.springframework.cloud.gateway.support.ServerWebExchangeUtils; 13 | import org.springframework.web.server.ServerWebExchange; 14 | 15 | import java.net.URI; 16 | import java.util.List; 17 | 18 | /** 19 | * 已废弃的自定义负载均衡过滤器 20 | * - 由于旧的 LoadBalancerClientFilter 已不推荐使用，此旧实现保留，简单易懂，原理不变。 21 | * 22 | * @author lawrence 23 | * @since 2021/3/24 24 | * @deprecated 推荐使用响应式网关的新过滤器 {@link CustomReactiveLoadBalanceFilter} 25 | */ 26 | @Deprecated 27 | public class CustomLoadBalanceFilter extends LoadBalancerClientFilter implements BeanPostProcessor { // NOSONAR 28 | 29 | final ConsistentHashRouter consistentHashRouter; 30 | final WebSocketProperties webSocketProperties; 31 | final DiscoveryClient discoveryClient; 32 | 33 | public CustomLoadBalanceFilter(LoadBalancerClient loadBalancer, 34 | LoadBalancerProperties properties, 35 | ConsistentHashRouter consistentHashRouter, 36 | WebSocketProperties webSocketProperties, 37 | DiscoveryClient discoveryClient) { 38 | super(loadBalancer, properties); 39 | this.consistentHashRouter = consistentHashRouter; 40 | this.webSocketProperties = webSocketProperties; 41 | this.discoveryClient = discoveryClient; 42 | } 43 | 44 | @Override 45 | protected ServiceInstance choose(ServerWebExchange exchange) { 46 | URI originalUrl = (URI) exchange.getAttributes().get(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR); 47 | String instancesId = originalUrl.getHost(); 48 | if (webSocketProperties.getService().getName().equals(instancesId)) { 49 | // 获取需要参与哈希的字段，此项目为 userId 50 | final String userId = WebSocketSessionLoadBalancer.getUserIdFromRequest(exchange); 51 | if (null != userId) { 52 | // 请求参数中有 userId，需要经过哈希环的路由 53 | ServiceNode serviceNode = consistentHashRouter.routeNode(userId); 54 | if (null != serviceNode) { 55 | // 获取当前注册中心的实例 56 | List instances = discoveryClient.getInstances(instancesId); 57 | for (ServiceInstance instance : instances) { 58 | // 如果 userId 映射后的真实节点的 IP 与某个实例 IP 一致，就转发 59 | if (instance.getHost().equals(serviceNode.getKey())) { 60 | return instance; 61 | } 62 | } 63 | } 64 | } 65 | } 66 | return super.choose(exchange); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /websocket/src/test/java/me/lawrenceli/websocket/controller/WebSocketControllerTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.controller; 2 | 3 | import me.lawrenceli.websocket.server.WebSocketEndpoint; 4 | import org.junit.jupiter.api.Test; 5 | import org.junit.jupiter.api.extension.ExtendWith; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.boot.test.mock.mockito.MockBean; 8 | import org.springframework.test.context.ContextConfiguration; 9 | import org.springframework.test.context.junit.jupiter.SpringExtension; 10 | import org.springframework.test.web.servlet.ResultActions; 11 | import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; 12 | import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; 13 | import org.springframework.test.web.servlet.result.ContentResultMatchers; 14 | import org.springframework.test.web.servlet.result.MockMvcResultMatchers; 15 | import org.springframework.test.web.servlet.setup.MockMvcBuilders; 16 | 17 | import static org.mockito.Mockito.any; 18 | import static org.mockito.Mockito.when; 19 | 20 | @ContextConfiguration(classes = {WebSocketController.class}) 21 | @ExtendWith(SpringExtension.class) 22 | class WebSocketControllerTest { 23 | @Autowired 24 | private WebSocketController webSocketController; 25 | 26 | @MockBean(name = "websocketEndpoint") 27 | private WebSocketEndpoint webSocketEndpoint; 28 | 29 | @Test 30 | void testCount() throws Exception { 31 | MockHttpServletRequestBuilder requestBuilder = MockMvcRequestBuilders.get("/count"); 32 | MockMvcBuilders.standaloneSetup(this.webSocketController) 33 | .build() 34 | .perform(requestBuilder) 35 | .andExpect(MockMvcResultMatchers.status().isOk()) 36 | .andExpect(MockMvcResultMatchers.content().contentType("application/json")) 37 | .andExpect(MockMvcResultMatchers.content().string("0")); 38 | } 39 | 40 | @Test 41 | void testCount2() throws Exception { 42 | MockHttpServletRequestBuilder requestBuilder = MockMvcRequestBuilders.get("/count", "Uri Vars"); 43 | MockMvcBuilders.standaloneSetup(this.webSocketController) 44 | .build() 45 | .perform(requestBuilder) 46 | .andExpect(MockMvcResultMatchers.status().isOk()) 47 | .andExpect(MockMvcResultMatchers.content().contentType("application/json")) 48 | .andExpect(MockMvcResultMatchers.content().string("0")); 49 | } 50 | 51 | @Test 52 | void testSend() throws Exception { 53 | when(this.webSocketEndpoint.sendMessageToUser(any(), any())) 54 | .thenReturn(true); 55 | MockHttpServletRequestBuilder requestBuilder = MockMvcRequestBuilders.get("/send") 56 | .param("message", "foo") 57 | .param("userId", "foo"); 58 | ResultActions resultActions = MockMvcBuilders.standaloneSetup(this.webSocketController) 59 | .build() 60 | .perform(requestBuilder) 61 | .andExpect(MockMvcResultMatchers.status().isOk()) 62 | .andExpect(MockMvcResultMatchers.content().contentType("application/json")); 63 | ContentResultMatchers contentResult = MockMvcResultMatchers.content(); 64 | resultActions.andExpect(contentResult.string(Boolean.TRUE.toString())); 65 | } 66 | } 67 | 68 | -------------------------------------------------------------------------------- /common/src/main/java/me/lawrenceli/model/WebSocketMessage.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.model; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.format.annotation.DateTimeFormat; 6 | 7 | import java.io.Serializable; 8 | import java.net.InetAddress; 9 | import java.net.UnknownHostException; 10 | import java.util.Date; 11 | 12 | /** 13 | * @author lawrence 14 | * @since 2021/3/23 15 | */ 16 | public class WebSocketMessage implements Serializable { 17 | 18 | private static final Logger logger = LoggerFactory.getLogger(WebSocketMessage.class); 19 | private static final long serialVersionUID = -6637831845278182827L; 20 | 21 | public static WebSocketMessage toUserOrServerMessage(final MessageType type, final String content, final Integer connectSize) { 22 | WebSocketMessage webSocketMessage = new WebSocketMessage() 23 | .setType(type.code()).setContent(content) 24 | .setServerUserCount(connectSize) 25 | .setTimestamp(new Date()); 26 | try { 27 | webSocketMessage = webSocketMessage.setServerIp(InetAddress.getLocalHost().getHostAddress()); 28 | } catch (UnknownHostException e) { 29 | String unknownIp = "0.0.0.0"; 30 | logger.error("获取实例 IP 失败，将被赋值为 {}: {}", unknownIp, e.getMessage()); 31 | webSocketMessage.setServerIp(unknownIp); 32 | } 33 | return webSocketMessage; 34 | } 35 | 36 | // 消息类型 37 | private Integer type; 38 | 39 | // 消息内容 40 | private String content; 41 | 42 | // 客户端所连接的服务端的连接数量，仅展示用 43 | private Integer serverUserCount; 44 | 45 | // 客户端所连接的服务端的局域网 IP，或唯一标识 46 | private String serverIp; 47 | 48 | // 时间戳 49 | @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") 50 | private Date timestamp; 51 | 52 | public Integer getType() { 53 | return type; 54 | } 55 | 56 | public WebSocketMessage setType(Integer type) { 57 | this.type = type; 58 | return this; 59 | } 60 | 61 | public String getContent() { 62 | return content; 63 | } 64 | 65 | public WebSocketMessage setContent(String content) { 66 | this.content = content; 67 | return this; 68 | } 69 | 70 | public Integer getServerUserCount() { 71 | return serverUserCount; 72 | } 73 | 74 | public WebSocketMessage setServerUserCount(Integer serverUserCount) { 75 | this.serverUserCount = serverUserCount; 76 | return this; 77 | } 78 | 79 | public String getServerIp() { 80 | return serverIp; 81 | } 82 | 83 | public WebSocketMessage setServerIp(String serverIp) { 84 | this.serverIp = serverIp; 85 | return this; 86 | } 87 | 88 | public Date getTimestamp() { 89 | return timestamp; 90 | } 91 | 92 | public WebSocketMessage setTimestamp(Date timestamp) { 93 | this.timestamp = timestamp; 94 | return this; 95 | } 96 | 97 | @Override 98 | public String toString() { 99 | return "WebSocketMessage{" + 100 | "type=" + type + 101 | ", content='" + content + '\'' + 102 | ", serverUserCount=" + serverUserCount + 103 | ", serverIp='" + serverIp + '\'' + 104 | ", timestamp=" + timestamp + 105 | '}'; 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /gateway/src/test/java/me/lawrenceli/gateway/docker/DockerServiceTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.docker; 2 | 3 | import com.github.dockerjava.api.DockerClient; 4 | import com.github.dockerjava.api.command.RemoveContainerCmd; 5 | import com.github.dockerjava.api.command.StartContainerCmd; 6 | import com.github.dockerjava.api.command.StopContainerCmd; 7 | import com.github.dockerjava.api.model.Container; 8 | import com.github.dockerjava.core.command.ListContainersCmdImpl; 9 | import com.google.common.collect.Lists; 10 | import me.lawrenceli.gateway.config.WebSocketProperties; 11 | import org.junit.jupiter.api.BeforeEach; 12 | import org.junit.jupiter.api.Test; 13 | import org.mockito.Answers; 14 | import org.mockito.InjectMocks; 15 | import org.mockito.Mock; 16 | import org.mockito.MockitoAnnotations; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | import static org.junit.jupiter.api.Assertions.assertEquals; 22 | import static org.junit.jupiter.api.Assertions.assertNotNull; 23 | import static org.mockito.Mockito.doNothing; 24 | import static org.mockito.Mockito.spy; 25 | import static org.mockito.Mockito.verify; 26 | import static org.mockito.Mockito.when; 27 | 28 | /** 29 | * @author lawrence 30 | * @since 2021/8/15 31 | */ 32 | class DockerServiceTest { 33 | 34 | static WebSocketProperties webSocketProperties; 35 | 36 | static { 37 | webSocketProperties = new WebSocketProperties(); 38 | WebSocketProperties.Docker docker = new WebSocketProperties.Docker(); 39 | docker.setHost("tcp://127.0.0.1:6666"); 40 | webSocketProperties.setDocker(docker); 41 | } 42 | 43 | @Mock(answer = Answers.RETURNS_DEEP_STUBS) 44 | private DockerClient dockerClient; 45 | 46 | @InjectMocks 47 | private DockerService dockerService = new DockerService(webSocketProperties); 48 | 49 | @BeforeEach 50 | void setUp() { 51 | MockitoAnnotations.initMocks(this); 52 | } 53 | 54 | @Test 55 | void ps() { 56 | String name = "test"; 57 | List containers = new ArrayList<>(); 58 | containers.add(new Container()); 59 | when(dockerClient.listContainersCmd().withNameFilter(Lists.newArrayList(name))) 60 | .thenReturn(new ListContainersCmdImpl(command -> containers)); 61 | List ps = dockerService.ps(name); 62 | assertNotNull(ps); 63 | assertEquals(1, ps.size()); 64 | } 65 | 66 | @Test 67 | void testRunContainer() { 68 | StartContainerCmd any = spy(StartContainerCmd.class); 69 | when(dockerClient.startContainerCmd("test")).thenReturn(any); 70 | doNothing().when(any).exec(); 71 | dockerService.runContainer("test"); 72 | verify(dockerClient).startContainerCmd("test"); 73 | 74 | } 75 | 76 | @Test 77 | void testStopContainer() { 78 | StopContainerCmd any = spy(StopContainerCmd.class); 79 | when(dockerClient.stopContainerCmd("test")).thenReturn(any); 80 | doNothing().when(any).exec(); 81 | dockerService.stopContainer("test"); 82 | verify(dockerClient).stopContainerCmd("test"); 83 | } 84 | 85 | 86 | @Test 87 | void testRemoveContainer() { 88 | RemoveContainerCmd any = spy(RemoveContainerCmd.class); 89 | when(dockerClient.removeContainerCmd("test")).thenReturn(any); 90 | doNothing().when(any).exec(); 91 | dockerService.removeContainer("test"); 92 | verify(dockerClient).removeContainerCmd("test"); 93 | } 94 | 95 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CI](https://github.com/Lonor/websocket-cluster/actions/workflows/sonar.yml/badge.svg)](https://github.com/Lonor/websocket-cluster/actions/workflows/sonar.yml) 2 | [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=Lonor_websocket-cluster&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=Lonor_websocket-cluster) 3 | [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=Lonor_websocket-cluster&metric=alert_status)](https://sonarcloud.io/dashboard?id=Lonor_websocket-cluster) 4 | [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=Lonor_websocket-cluster&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=Lonor_websocket-cluster) 5 | 6 | # 实战 Spring Cloud 的 WebSocket 集群 7 | 8 | 此项目是一个 WebSocket 集群的实践，基于 Spring Cloud。 9 | 10 | [English Doc](./README-en.md) 11 | 12 | ## 原理 13 | 14 | 我们利用一致性哈希算法，构造一个哈希环，网关监听 WebSocket 服务实例的上下线消息，根据实例的变动动态地更新哈希环。 每次有新服务上线时，添加其对应的虚拟节点，将需要变动的 WebSocket 15 | 客户端重新连接到新的实例上，这样的代价是最小的；当然也取决与虚拟节点的数量以及哈希算法的公平性。服务下线时，实现相对容易——只需要将当前实例的所有客户端断开就行，客户端始终会重新连接的。 16 | 同时，哈希环的核心作用体现在负载均衡上。网关做请求转发时，会经过我们重写的自定义负载均衡过滤器，根据业务上需要哈希的字段来实现真实节点的路由。 17 | 18 | ## 技术栈 19 | 20 | - Docker (开启 API 访问) 21 | - Redis 22 | - RabbitMQ 23 | - Nacos 24 | 25 | ## 本地开发 26 | 27 | 为 [docker-compose.yml](./docker-compose.yml) 创建一个专用网络： 28 | 29 | ```shell 30 | docker network create compose-network 31 | ``` 32 | 33 | 本地构建，并使用 docker compose 简单编排部署： 34 | 35 | ```shell 36 | mvn clean 37 | mvn install -pl gateway -am -amd 38 | mvn install -pl websocket -am -amd 39 | docker build -t websocket:1.0.0 websocket/. 40 | docker build -t gateway:1.0.0 gateway/. 41 | docker-compose up -d 42 | docker ps 43 | ``` 44 | 45 | 可以用 `docker-compose scale websocket-server=3` 命令来创建新的 Websocket 实例。实际上我给这个项目写了一个前端来展示。 46 | 47 | 别忘了开启 Docker 的 API 访问，用 `docker -H tcp://0.0.0.0:2375 ps` 来验证 API 是否开启成功。 尝试开启： 48 | 49 | ### Linux 上开启 Docker API 访问 50 | 51 | 在 `docker.service` 文件中，将 `-H tcp://0.0.0.0:2375` 添加到 `ExecStart` 开头的那一行。 52 | 53 | ```shell 54 | # cat /usr/lib/systemd/system/docker.service 55 | ExecStart=...... -H tcp://0.0.0.0:2375 56 | # after saved, restart the docker process 57 | systemctl daemon-reload 58 | systemctl restart docker 59 | ``` 60 | 61 | ### macOS 上访问 Docker API 62 | 63 | 最佳实践是用 `alpine/socat` 来暴露 TCP 套接字。参考 [socat 的用法](https://github.com/alpine-docker/socat#example). 64 | 65 | ```shell 66 | docker run -itd --name socat \ 67 | -p 0.0.0.0:6666:2375 \ 68 | -v /var/run/docker.sock:/var/run/docker.sock \ 69 | alpine/socat \ 70 | tcp-listen:2375,fork,reuseaddr unix-connect:/var/run/docker.sock 71 | ``` 72 | 73 | 注意，Docker 的 macOS 客户端提供了一个 `docker.for.mac.host.internal` 的主机名，可以在容器内访问宿主网络。 我将这个地址用在了 `application.yml` 配置文件中，用来作为 74 | redis、rabbitmq、nacos 服务端访问，因为他们都被我部署在容器里。如果要部署到服务器或自己本地开发，你可以把地址改掉。还有，我写了个 `Makefile` 75 | 用来帮自己在开发阶段更快地编译并重启服务，因为我并没有给这个项目配置一个持续集成流水线，请按需使用。 76 | 77 | 代码中，所有依赖注入都尽可能地在使用构造注入，并详细地打印了日志供分析。 78 | 79 | ## 前端 80 | 81 | 参见[此 React 项目](https://github.com/Lonor/websocket-cluster-front). 效果如图： 82 | 83 | ![Demo](./demo.gif) 84 | 85 | ## 部署 86 | 87 | 修改 `docker-compose.yml`, 添加如下 host 解析，可以方便地替换掉服务调用地址而不用更改任何代码或 `application.yaml`，注意后面的 IP 建议使用服务器内网 IP 地址。唯一必要的修改是 88 | Nacos 的 namespace. 89 | 90 | ```yml 91 | extra_hosts: 92 | - "docker.for.mac.host.internal:192.168.0.1" 93 | ``` 94 | 95 | 所有必要环境就绪后，直接开启服务。请仔细参考 Makefile 中的内容使用，如： 96 | 97 | ```shell 98 | make up 99 | ``` 100 | 101 | 注意 `make down` 操作会删除所有 none 容器镜像以及 redis 中的内容。 102 | 103 | ## 贡献 104 | 105 | 若有帮助，欢迎 star 收藏。有问题请提交 Issue。贡献请 fork 此项目后提交 Pull Request. 106 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/config/GatewayHashRingConfig.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.config; 2 | 3 | import me.lawrenceli.constant.GlobalConstant; 4 | import me.lawrenceli.gateway.filter.CustomReactiveLoadBalanceFilter; 5 | import me.lawrenceli.gateway.server.ServiceNode; 6 | import me.lawrenceli.hashring.ConsistentHashRouter; 7 | import me.lawrenceli.hashring.VirtualNode; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.cloud.client.discovery.DiscoveryClient; 11 | import org.springframework.cloud.client.loadbalancer.LoadBalancerClient; 12 | import org.springframework.cloud.gateway.config.LoadBalancerProperties; 13 | import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory; 14 | import org.springframework.context.annotation.Bean; 15 | import org.springframework.context.annotation.Configuration; 16 | import org.springframework.data.redis.core.RedisTemplate; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | /** 23 | * 初始化 24 | * 25 | * @author lawrence 26 | * @since 2021/3/23 27 | */ 28 | @Configuration 29 | public class GatewayHashRingConfig { 30 | 31 | private static final Logger logger = LoggerFactory.getLogger(GatewayHashRingConfig.class); 32 | 33 | final RedisTemplate redisTemplate; 34 | final WebSocketProperties webSocketProperties; 35 | private final LoadBalancerClientFactory clientFactory; 36 | 37 | public GatewayHashRingConfig(RedisTemplate redisTemplate, 38 | WebSocketProperties webSocketProperties, 39 | LoadBalancerClientFactory clientFactory) { 40 | this.redisTemplate = redisTemplate; 41 | this.webSocketProperties = webSocketProperties; 42 | this.clientFactory = clientFactory; 43 | } 44 | 45 | /** 46 | * @param client 负载均衡客户端 47 | * @param loadBalancerProperties 负载均衡配置 48 | * @param consistentHashRouter {@link #init() init方法}注入，此处未使用构造注入（会产生循环依赖） 49 | * @param discoveryClient 服务发现客户端 50 | * @return 注入自定义的 Reactive 过滤器 Bean 对象 51 | */ 52 | @Bean 53 | public CustomReactiveLoadBalanceFilter customReactiveLoadBalanceFilter(LoadBalancerClient client, 54 | LoadBalancerProperties loadBalancerProperties, 55 | ConsistentHashRouter consistentHashRouter, 56 | DiscoveryClient discoveryClient) { 57 | logger.debug("初始化 自定义响应式负载均衡器: {}, {}", client, loadBalancerProperties); 58 | return new CustomReactiveLoadBalanceFilter(clientFactory, loadBalancerProperties, 59 | consistentHashRouter, discoveryClient, webSocketProperties); 60 | } 61 | 62 | @Bean 63 | @SuppressWarnings("unchecked") 64 | public ConsistentHashRouter init() { 65 | // 先从 Redis 中获取哈希环(网关集群) 66 | final Map ring = redisTemplate.opsForHash().entries(GlobalConstant.HASH_RING_REDIS); 67 | // 获取环中的所有真实节点 68 | List serviceNodes = new ArrayList<>(); 69 | for (Object key : ring.keySet()) { 70 | Long hashKey = (Long) key; 71 | VirtualNode virtualNode = (VirtualNode) ring.get(hashKey); 72 | ServiceNode physicalNode = virtualNode.getPhysicalNode(); 73 | serviceNodes.add(physicalNode); 74 | } 75 | ConsistentHashRouter consistentHashRouter = new ConsistentHashRouter<>(serviceNodes, GlobalConstant.VIRTUAL_COUNT); 76 | logger.debug("初始化 ConsistentHashRouter: {}", consistentHashRouter); 77 | return consistentHashRouter; 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /websocket/src/test/java/me/lawrenceli/websocket/config/MQConfigTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.config; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.amqp.core.AnonymousQueue; 5 | import org.springframework.amqp.core.Binding; 6 | import org.springframework.amqp.core.FanoutExchange; 7 | 8 | import static org.junit.jupiter.api.Assertions.assertEquals; 9 | import static org.junit.jupiter.api.Assertions.assertFalse; 10 | import static org.junit.jupiter.api.Assertions.assertTrue; 11 | 12 | class MQConfigTest { 13 | @Test 14 | void testFanoutExchange() { 15 | // Diffblue Cover was unable to write a Spring test, 16 | // so wrote a non-Spring test instead. 17 | // Reason: R004 No meaningful assertions found. 18 | // Diffblue Cover was unable to create an assertion. 19 | // Make sure that fields modified by fanoutExchange() 20 | // have package-private, protected, or public getters. 21 | // See https://diff.blue/R004 to resolve this issue. 22 | 23 | FanoutExchange actualFanoutExchangeResult = (new MQConfig()).fanoutExchange(); 24 | assertTrue(actualFanoutExchangeResult.getArguments().isEmpty()); 25 | assertTrue(actualFanoutExchangeResult.shouldDeclare()); 26 | assertTrue(actualFanoutExchangeResult.isDurable()); 27 | assertFalse(actualFanoutExchangeResult.isAutoDelete()); 28 | assertEquals("websocket-cluster-exchange", actualFanoutExchangeResult.getName()); 29 | assertTrue(actualFanoutExchangeResult.getDeclaringAdmins().isEmpty()); 30 | } 31 | 32 | @Test 33 | void testQueueForWebSocket() { 34 | // Diffblue Cover was unable to write a Spring test, 35 | // so wrote a non-Spring test instead. 36 | // Reason: R004 No meaningful assertions found. 37 | // Diffblue Cover was unable to create an assertion. 38 | // Make sure that fields modified by queueForWebSocket() 39 | // have package-private, protected, or public getters. 40 | // See https://diff.blue/R004 to resolve this issue. 41 | 42 | AnonymousQueue actualQueueForWebSocketResult = (new MQConfig()).queueForWebSocket(); 43 | assertTrue(actualQueueForWebSocketResult.shouldDeclare()); 44 | assertTrue(actualQueueForWebSocketResult.isExclusive()); 45 | assertFalse(actualQueueForWebSocketResult.isDurable()); 46 | assertTrue(actualQueueForWebSocketResult.isAutoDelete()); 47 | assertTrue(actualQueueForWebSocketResult.getDeclaringAdmins().isEmpty()); 48 | assertEquals(1, actualQueueForWebSocketResult.getArguments().size()); 49 | } 50 | 51 | @Test 52 | void testBindingSingle() { 53 | // Diffblue Cover was unable to write a Spring test, 54 | // so wrote a non-Spring test instead. 55 | // Reason: R004 No meaningful assertions found. 56 | // Diffblue Cover was unable to create an assertion. 57 | // Make sure that fields modified by bindingSingle(FanoutExchange, AnonymousQueue) 58 | // have package-private, protected, or public getters. 59 | // See https://diff.blue/R004 to resolve this issue. 60 | 61 | MQConfig mqConfig = new MQConfig(); 62 | FanoutExchange fanoutExchange = new FanoutExchange("Name"); 63 | Binding actualBindingSingleResult = mqConfig.bindingSingle(fanoutExchange, new AnonymousQueue()); 64 | assertTrue(actualBindingSingleResult.getArguments().isEmpty()); 65 | assertTrue(actualBindingSingleResult.shouldDeclare()); 66 | assertEquals("", actualBindingSingleResult.getRoutingKey()); 67 | assertEquals("Name", actualBindingSingleResult.getExchange()); 68 | assertEquals(Binding.DestinationType.QUEUE, actualBindingSingleResult.getDestinationType()); 69 | assertTrue(actualBindingSingleResult.getDeclaringAdmins().isEmpty()); 70 | } 71 | } 72 | 73 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/docker/DockerService.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.docker; 2 | 3 | import com.github.dockerjava.api.DockerClient; 4 | import com.github.dockerjava.api.command.CreateContainerCmd; 5 | import com.github.dockerjava.api.command.CreateContainerResponse; 6 | import com.github.dockerjava.api.command.ListContainersCmd; 7 | import com.github.dockerjava.api.model.Container; 8 | import com.github.dockerjava.api.model.HostConfig; 9 | import com.github.dockerjava.core.DockerClientBuilder; 10 | import com.github.dockerjava.jaxrs.JerseyDockerHttpClient; 11 | import com.google.common.collect.Lists; 12 | import me.lawrenceli.gateway.config.WebSocketProperties; 13 | import me.lawrenceli.utils.StringPattern; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | import org.springframework.stereotype.Service; 17 | 18 | import java.net.URI; 19 | import java.net.URISyntaxException; 20 | import java.util.List; 21 | 22 | /** 23 | * Access Docker API by official Java SDK 24 | * 25 | * @author lawrence 26 | * @since 2021/3/19 27 | */ 28 | @Service 29 | public class DockerService { 30 | 31 | private static final Logger logger = LoggerFactory.getLogger(DockerService.class); 32 | 33 | final WebSocketProperties webSocketProperties; 34 | DockerClient dockerClient; 35 | 36 | public DockerService(WebSocketProperties webSocketProperties) { 37 | this.webSocketProperties = webSocketProperties; 38 | this.dockerClient = this.getDockerClient(); 39 | } 40 | 41 | private DockerClient getDockerClient() { 42 | logger.debug("初始化 Docker 客户端"); 43 | DockerClientBuilder dockerClientBuilder = DockerClientBuilder.getInstance(); 44 | try { 45 | dockerClientBuilder.withDockerHttpClient( 46 | new JerseyDockerHttpClient.Builder() 47 | .dockerHost(new URI(webSocketProperties.getDocker().getHost())) 48 | .build()); 49 | } catch (URISyntaxException e) { 50 | logger.error("Docker Host 不是合法的 URI"); 51 | } 52 | return dockerClientBuilder.build(); 53 | } 54 | 55 | public List ps(String containerNamePrefix) { 56 | String containerName = StringPattern.replacePatternBreaking(containerNamePrefix); 57 | logger.info("执行: docker ps|grep {}", containerName); 58 | ListContainersCmd listContainersCmd = dockerClient.listContainersCmd().withNameFilter(Lists.newArrayList(containerNamePrefix)); 59 | return listContainersCmd.exec(); 60 | } 61 | 62 | public CreateContainerResponse createContainer(String imageName) { 63 | imageName = StringPattern.replacePatternBreaking(imageName); 64 | logger.info("执行: 创建 {} 的容器", imageName); 65 | try (CreateContainerCmd containerCmd = dockerClient.createContainerCmd(imageName)) { 66 | return containerCmd.withHostConfig(HostConfig.newHostConfig().withNetworkMode(webSocketProperties.getDocker().getNetwork())) 67 | .withName(webSocketProperties.getService().getName() + '-' + System.currentTimeMillis()) 68 | .exec(); 69 | } 70 | } 71 | 72 | public void runContainer(String containerId) { 73 | containerId = StringPattern.replacePatternBreaking(containerId); 74 | logger.info("执行: 启动容器 {}", containerId); 75 | dockerClient.startContainerCmd(containerId).exec(); 76 | } 77 | 78 | public void stopContainer(String containerId) { 79 | containerId = StringPattern.replacePatternBreaking(containerId); 80 | logger.info("执行: 关闭容器 {}", containerId); 81 | dockerClient.stopContainerCmd(containerId).exec(); 82 | } 83 | 84 | public void removeContainer(String containerId) { 85 | containerId = StringPattern.replacePatternBreaking(containerId); 86 | logger.info("执行: 删除容器 {}", containerId); 87 | dockerClient.removeContainerCmd(containerId).exec(); 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /README-en.md: -------------------------------------------------------------------------------- 1 | # WebSocket Cluster (Spring Cloud) in action 2 | 3 | This is a Spring Cloud project for WebSocket cluster servers. 4 | 5 | [中文](README.md) 6 | 7 | ## Principle 8 | 9 | Using a consistent hashing algorithm, we construct a hash ring where the gateway listens to the up and down event of the 10 | WebSocket service instances and dynamically updates the hash ring according to the changes of the instances. Each time a 11 | new service comes online, the corresponding virtual node is added, and the WebSocket clients that need to change are 12 | reconnected to the new instance. Clients that need to be changed are reconnected to the new instance, which is the least 13 | expensive; of course, it also depends on the count of the virtual nodes and the fairness of the hashing algorithm. When 14 | the service goes offline, it is easier -- just disconnect all clients from the current instance, and the clients will 15 | always reconnect. At the same time, the core role of the hash ring is reflected in the load balancing. When the gateway 16 | does its request forwarding, it goes through our rewritten custom load balancing filter, which implements real node 17 | routing based on the fields that need to be hashed for business purposes. 18 | 19 | ## Middleware / Environment Preparation 20 | 21 | - Docker (with API accessible) 22 | - Redis 23 | - RabbitMQ 24 | - Nacos 25 | 26 | ## Development 27 | 28 | Create a docker-compose network for [docker-compose.yml](./docker-compose.yml): 29 | 30 | ```shell 31 | docker network create compose-network 32 | ``` 33 | 34 | Local build and deploy by docker-compose: 35 | 36 | ```shell 37 | mvn clean 38 | mvn install -pl gateway -am -amd 39 | mvn install -pl websocket -am -amd 40 | docker build -t websocket:1.0.0 websocket/. 41 | docker build -t gateway:1.0.0 gateway/. 42 | docker-compose up -d 43 | docker ps 44 | ``` 45 | 46 | Scale the websocket instance by: `docker-compose scale websocket-server=3`. Actually, I wrote a front-end web page to 47 | start a new instance for websocket service. 48 | 49 | Don't forget to enable the docker remote api (e.g., check out `docker -H tcp://0.0.0.0:2375 ps`): 50 | The following steps may help: 51 | 52 | ### Access docker API on Linux 53 | 54 | Append `-H tcp://0.0.0.0:2375` to the line started of `ExecStart` in the file named `docker.service` 55 | 56 | ```shell 57 | # cat /usr/lib/systemd/system/docker.service 58 | ExecStart=...... -H tcp://0.0.0.0:2375 59 | # after saved, restart the docker process 60 | systemctl daemon-reload 61 | systemctl restart docker 62 | ``` 63 | 64 | ### Access docker API on macOS 65 | 66 | The best practice is using the image `alpine/socat` to expose a tcp socket. ( 67 | see: [usage of socat](https://github.com/alpine-docker/socat#example)). 68 | 69 | ```shell 70 | docker run -itd --name socat \ 71 | -p 0.0.0.0:6666:2375 \ 72 | -v /var/run/docker.sock:/var/run/docker.sock \ 73 | alpine/socat \ 74 | tcp-listen:2375,fork,reuseaddr unix-connect:/var/run/docker.sock 75 | ``` 76 | 77 | For the record, the Docker-desktop client for macOS provides the `docker.for.mac.host.internal` hostname for accessing 78 | the host in the container of docker. I also add this domain to my `/etc/hosts` and point it to the localhost address. So 79 | I use this address in `application.yml` to set up configuration for the redis, the rabbitmq and the nacos servers(they 80 | are all deployed in the container). You can change this address if you deploy on your machine or other linux servers. 81 | BTW, the Makefile is only for self usage, which makes me build and restart the service faster during the development 82 | phase because I didn't prepare a CI/CD pipeline for this project. Please use it as needed. 83 | 84 | ## Front-end 85 | 86 | Check out [this react app](https://github.com/Lonor/websocket-cluster-front). It looks like: 87 | 88 | ![Demo](./demo.gif) 89 | 90 | ## Contribution 91 | 92 | If this project helps, please star it. Submit an issue if you have any question. You can contribute code by forking this 93 | project and submit your Pull Request. -------------------------------------------------------------------------------- /common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | springcloud-websocket-cluster 7 | me.lawrenceli 8 | 1.0.0 9 | 10 | 4.0.0 11 | common 12 | jar 13 | 14 | 15 | 8 16 | 8 17 | 2.3.9.RELEASE 18 | 2.2.9.RELEASE 19 | 20 | 21 | 22 | org.springframework.cloud 23 | spring-cloud-commons 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-starter-data-redis 28 | ${spring-boot.version} 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-starter-json 33 | 34 | 35 | org.springframework.boot 36 | spring-boot-starter-amqp 37 | 38 | 39 | org.springframework.boot 40 | spring-boot-starter-test 41 | 42 | 43 | com.alibaba.cloud 44 | spring-cloud-starter-alibaba-nacos-discovery 45 | ${nacos.version} 46 | 47 | 48 | javax.ws.rs 49 | jsr311-api 50 | 51 | 52 | org.springframework.cloud 53 | spring-cloud-starter-netflix-ribbon 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | org.apache.maven.plugins 63 | maven-compiler-plugin 64 | 3.11.0 65 | 66 | 8 67 | 8 68 | 69 | 70 | 71 | org.apache.maven.plugins 72 | maven-surefire-plugin 73 | 2.22.2 74 | 75 | 76 | org.jacoco 77 | jacoco-maven-plugin 78 | 0.8.8 79 | 80 | 81 | 82 | prepare-agent 83 | 84 | 85 | 86 | report 87 | prepare-package 88 | 89 | report 90 | 91 | 92 | 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /gateway/src/test/java/me/lawrenceli/gateway/server/RedisSubscriberTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.server; 2 | 3 | import me.lawrenceli.constant.GlobalConstant; 4 | import me.lawrenceli.hashring.ConsistentHashRouter; 5 | import me.lawrenceli.model.MessageType; 6 | import me.lawrenceli.model.WebSocketMessage; 7 | import me.lawrenceli.utils.JSON; 8 | import org.junit.jupiter.api.BeforeEach; 9 | import org.junit.jupiter.api.DisplayName; 10 | import org.junit.jupiter.api.Nested; 11 | import org.junit.jupiter.api.Test; 12 | import org.mockito.Answers; 13 | import org.mockito.InjectMocks; 14 | import org.mockito.Mock; 15 | import org.mockito.Mockito; 16 | import org.mockito.MockitoAnnotations; 17 | import org.springframework.data.redis.core.RedisTemplate; 18 | 19 | import java.util.ArrayList; 20 | import java.util.Date; 21 | import java.util.HashMap; 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | import static org.junit.jupiter.api.Assertions.assertEquals; 26 | import static org.mockito.ArgumentMatchers.anyList; 27 | 28 | /** 29 | * @author lawrence 30 | * @since 2021/8/19 31 | */ 32 | class RedisSubscriberTest { 33 | 34 | @InjectMocks 35 | RedisSubscriber redisSubscriber; 36 | 37 | @Mock(answer = Answers.RETURNS_DEEP_STUBS) 38 | RedisTemplate redisTemplate; 39 | 40 | @Mock 41 | FanoutSender fanoutSender; 42 | 43 | private ConsistentHashRouter consistentHashRouter; 44 | 45 | @BeforeEach 46 | void setUp() { 47 | List nodes = new ArrayList<>(); 48 | nodes.add(new ServiceNode("192.168.0.0")); 49 | nodes.add(new ServiceNode("192.168.0.1")); 50 | consistentHashRouter = new ConsistentHashRouter<>(nodes, GlobalConstant.VIRTUAL_COUNT); 51 | MockitoAnnotations.initMocks(this); 52 | redisSubscriber.setConsistentHashRouter(consistentHashRouter); 53 | } 54 | 55 | @Nested 56 | @DisplayName("Test handleMessage") 57 | class HandleMessage { 58 | @BeforeEach 59 | void setUp() { 60 | Mockito.when(redisTemplate.opsForHash().entries(GlobalConstant.KEY_TO_BE_HASHED)) 61 | .thenReturn(mockUserIdAndHashInRedis()); 62 | Mockito.doNothing().when(fanoutSender).send(anyList()); 63 | } 64 | 65 | @Test 66 | @DisplayName("Test When Server Up") 67 | void testHandleUpMessage() throws InterruptedException { 68 | WebSocketMessage webSocketServerUpMessage = generateServerChangeMessage("UP"); 69 | assertEquals(new ServiceNode("192.168.0.0"), consistentHashRouter.routeNode("100")); 70 | redisSubscriber.handleMessage(JSON.toJSONString(webSocketServerUpMessage)); 71 | 72 | assertEquals(3 * GlobalConstant.VIRTUAL_COUNT, consistentHashRouter.getRing().size()); 73 | assertEquals(new ServiceNode("192.168.0.2"), consistentHashRouter.routeNode("100")); 74 | } 75 | 76 | @Test 77 | @DisplayName("Test When Server Down") 78 | void testHandleDownMessage() throws InterruptedException { 79 | WebSocketMessage webSocketServerDownMessage = generateServerChangeMessage("DOWN"); 80 | redisSubscriber.handleMessage(JSON.toJSONString(webSocketServerDownMessage)); 81 | 82 | assertEquals(2 * GlobalConstant.VIRTUAL_COUNT, consistentHashRouter.getRing().size()); 83 | assertEquals(new ServiceNode("192.168.0.0"), consistentHashRouter.routeNode("100")); 84 | } 85 | } 86 | 87 | 88 | private Map mockUserIdAndHashInRedis() { 89 | ConsistentHashRouter.MD5Hash md5Hash = new ConsistentHashRouter.MD5Hash(); 90 | HashMap userIdAndHash = new HashMap<>(2); 91 | userIdAndHash.put("100", md5Hash.hash("100")); 92 | userIdAndHash.put("200", md5Hash.hash("200")); 93 | return userIdAndHash; 94 | } 95 | 96 | private WebSocketMessage generateServerChangeMessage(String upOrDown) { 97 | return new WebSocketMessage().setType(MessageType.FOR_SERVER.code()) 98 | .setServerUserCount(2) 99 | .setTimestamp(new Date()) 100 | .setContent(upOrDown).setServerIp("192.168.0.2"); 101 | } 102 | 103 | } -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/config/WebSocketProperties.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.config; 2 | 3 | import org.springframework.beans.factory.annotation.Value; 4 | import org.springframework.boot.context.properties.ConfigurationProperties; 5 | import org.springframework.context.annotation.Configuration; 6 | 7 | import java.io.Serializable; 8 | import java.util.Objects; 9 | import java.util.Properties; 10 | 11 | /** 12 | * @author lawrence 13 | * @since 2021/3/21 14 | */ 15 | @ConfigurationProperties(prefix = "websocket") 16 | @Configuration 17 | public class WebSocketProperties extends Properties { 18 | 19 | private static final long serialVersionUID = -7568982050310381466L; 20 | 21 | @Value("${spring.cloud.nacos.discovery.server-addr}") 22 | private String nacosServerAddress; 23 | 24 | @Value("${spring.cloud.nacos.discovery.namespace}") 25 | private String nacosNamespace; 26 | 27 | public String getNacosServerAddress() { 28 | return nacosServerAddress; 29 | } 30 | 31 | public void setNacosServerAddress(String nacosServerAddress) { 32 | this.nacosServerAddress = nacosServerAddress; 33 | } 34 | 35 | public String getNacosNamespace() { 36 | return nacosNamespace; 37 | } 38 | 39 | public void setNacosNamespace(String nacosNamespace) { 40 | this.nacosNamespace = nacosNamespace; 41 | } 42 | 43 | private Service service; 44 | 45 | private Docker docker; 46 | 47 | 48 | public Service getService() { 49 | return service; 50 | } 51 | 52 | public void setService(Service service) { 53 | this.service = service; 54 | } 55 | 56 | public Docker getDocker() { 57 | return docker; 58 | } 59 | 60 | public void setDocker(Docker docker) { 61 | this.docker = docker; 62 | } 63 | 64 | public static class Service implements Serializable { 65 | 66 | private static final long serialVersionUID = 4470917617306041628L; 67 | 68 | private String name; 69 | 70 | public String getName() { 71 | return name; 72 | } 73 | 74 | public void setName(String name) { 75 | this.name = name; 76 | } 77 | } 78 | 79 | 80 | public static class Docker implements Serializable { 81 | 82 | private static final long serialVersionUID = -2233645916767230952L; 83 | 84 | private String host; 85 | 86 | private String network; 87 | 88 | private Image image; 89 | 90 | public static class Image implements Serializable { 91 | 92 | private static final long serialVersionUID = 648412373970515185L; 93 | 94 | private String name; 95 | 96 | public String getName() { 97 | return name; 98 | } 99 | 100 | public void setName(String name) { 101 | this.name = name; 102 | } 103 | } 104 | 105 | public String getHost() { 106 | return host; 107 | } 108 | 109 | public void setHost(String host) { 110 | this.host = host; 111 | } 112 | 113 | public String getNetwork() { 114 | return network; 115 | } 116 | 117 | public void setNetwork(String network) { 118 | this.network = network; 119 | } 120 | 121 | public Image getImage() { 122 | return image; 123 | } 124 | 125 | public void setImage(Image image) { 126 | this.image = image; 127 | } 128 | } 129 | 130 | 131 | @Override 132 | public synchronized boolean equals(Object o) { 133 | if (this == o) return true; 134 | if (o == null || getClass() != o.getClass()) return false; 135 | if (!super.equals(o)) return false; 136 | WebSocketProperties that = (WebSocketProperties) o; 137 | return Objects.equals(nacosServerAddress, that.nacosServerAddress) && Objects.equals(nacosNamespace, that.nacosNamespace) && Objects.equals(service, that.service) && Objects.equals(docker, that.docker); 138 | } 139 | 140 | @Override 141 | public synchronized int hashCode() { 142 | return Objects.hash(super.hashCode(), nacosServerAddress, nacosNamespace, service, docker); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /gateway/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | springcloud-websocket-cluster 7 | me.lawrenceli 8 | 1.0.0 9 | 10 | 4.0.0 11 | gateway 12 | 13 | 14 | 8 15 | 8 16 | 2.2.5.RELEASE 17 | 3.2.14 18 | 2.3.9.RELEASE 19 | 20 | 21 | 22 | 23 | me.lawrenceli 24 | common 25 | 1.0.0 26 | 27 | 28 | org.springframework.cloud 29 | spring-cloud-starter-gateway 30 | 31 | 32 | org.springframework.boot 33 | spring-boot-starter-webflux 34 | 35 | 36 | org.springframework.cloud 37 | spring-cloud-loadbalancer 38 | 39 | 40 | 41 | com.github.docker-java 42 | docker-java 43 | ${docker.java.version} 44 | 45 | 46 | org.springframework.boot 47 | spring-boot-configuration-processor 48 | true 49 | 50 | 51 | 52 | 53 | 54 | 55 | org.apache.maven.plugins 56 | maven-compiler-plugin 57 | 3.11.0 58 | 59 | 8 60 | 8 61 | 62 | 63 | 64 | org.apache.maven.plugins 65 | maven-surefire-plugin 66 | 2.22.2 67 | 68 | 69 | org.jacoco 70 | jacoco-maven-plugin 71 | 0.8.8 72 | 73 | 74 | 75 | prepare-agent 76 | 77 | 78 | 79 | report 80 | prepare-package 81 | 82 | report 83 | 84 | 85 | 86 | 87 | 88 | org.springframework.boot 89 | spring-boot-maven-plugin 90 | 91 | 92 | 93 | org.springframework.boot 94 | spring-boot-configuration-processor 95 | 96 | 97 | 98 | 99 | 100 | 101 | repackage 102 | 103 | 104 | 105 | 106 | 107 | 108 | -------------------------------------------------------------------------------- /common/src/main/java/me/lawrenceli/hashring/ConsistentHashRouter.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.hashring; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.security.MessageDigest; 7 | import java.security.NoSuchAlgorithmException; 8 | import java.util.Collection; 9 | import java.util.Iterator; 10 | import java.util.SortedMap; 11 | import java.util.TreeMap; 12 | 13 | /** 14 | * T: 真实节点泛型 15 | * 16 | * @author lawrence 17 | * @since 2021/3/23 18 | */ 19 | public class ConsistentHashRouter { 20 | 21 | private static final Logger logger = LoggerFactory.getLogger(ConsistentHashRouter.class); 22 | 23 | private final HashAlgorithm hashAlgorithm; 24 | 25 | /** 26 | * 哈希环本体 27 | * K: 虚拟节点 key 的哈希， V: 虚拟节点 28 | */ 29 | private final SortedMap> ring = new TreeMap<>(); 30 | 31 | public SortedMap> getRing() { 32 | return ring; 33 | } 34 | 35 | public ConsistentHashRouter(Collection physicalNodes, Integer virtualNodeCount) { 36 | this(physicalNodes, virtualNodeCount, new MD5Hash()); 37 | } 38 | 39 | public ConsistentHashRouter(Collection physicalNodes, Integer virtualNodeCount, HashAlgorithm hashAlgorithmImpl) { 40 | this.hashAlgorithm = hashAlgorithmImpl; 41 | // 遍历真实节点： 42 | for (T physicalNode : physicalNodes) { 43 | // 给每个真实节点添加虚拟节点 44 | addNode(physicalNode, virtualNodeCount); 45 | } 46 | } 47 | 48 | /** 49 | * 核心：根据提供的 业务 key 路由到对应的真实节点 50 | * 51 | * @param businessKey 业务key，比如 userID，redis 中的 key 等等分散在不同服务的标识 52 | * @return 真实节点 53 | */ 54 | public T routeNode(String businessKey) { 55 | if (ring.isEmpty()) { 56 | logger.debug("哈希环为空"); 57 | return null; 58 | } 59 | Long hashOfBusinessKey = this.hashAlgorithm.hash(businessKey); 60 | // 截取哈希环中比当前业务值哈希大的部分环 map 61 | SortedMap> biggerTailMap = ring.tailMap(hashOfBusinessKey); 62 | // 获取路由到的虚拟节点的 hash 63 | Long nodeHash; 64 | if (biggerTailMap.isEmpty()) { 65 | // 没有，回到整个哈希环的环首 66 | nodeHash = ring.firstKey(); 67 | } else { 68 | // 存在，则为被截取后的 tailMap 的首个节点 key 69 | nodeHash = biggerTailMap.firstKey(); 70 | } 71 | VirtualNode virtualNode = ring.get(nodeHash); 72 | return virtualNode.getPhysicalNode(); 73 | } 74 | 75 | /** 76 | * 新增节点 77 | * 78 | * @param physicalNode 单个真实节点 79 | * @param virtualNodeCount 虚拟节点数量（需要限制为自然数） 80 | */ 81 | public void addNode(T physicalNode, Integer virtualNodeCount) { 82 | logger.info("【上线】哈希环新增一个真实节点: {}, 虚拟副本节点数量 {}", physicalNode, virtualNodeCount); 83 | // 先获取当前真实节点的虚拟节点数量 84 | Integer virtualNodeCountExistBefore = getVirtualNodeCountOf(physicalNode); 85 | for (int i = 0; i < virtualNodeCount; i++) { 86 | VirtualNode virtualNode = new VirtualNode<>(physicalNode, virtualNodeCountExistBefore + i); 87 | ring.put(this.hashAlgorithm.hash(virtualNode.getKey()), virtualNode); 88 | } 89 | } 90 | 91 | /** 92 | * 下线一个真实节点 93 | * 94 | * @param physicalNode 真实节点 95 | */ 96 | public void removeNode(T physicalNode) { 97 | logger.info("【下线】移除一个真实节点: {}", physicalNode); 98 | // 实现注意遍历删除可能存在的并发修改异常 99 | Iterator iterator = ring.keySet().iterator(); 100 | while (iterator.hasNext()) { 101 | Long nodeHashKey = iterator.next(); 102 | VirtualNode virtualNode = ring.get(nodeHashKey); 103 | if (virtualNode.isVirtualOf(physicalNode)) { 104 | logger.info("【下线：遍历】删除哈希环对应的虚拟节点: {}", nodeHashKey); 105 | iterator.remove(); 106 | } 107 | } 108 | } 109 | 110 | /** 111 | * 获取当前真实节点的虚拟节点的数量 112 | * 113 | * @param physicalNode 真实节点 114 | * @return 虚拟节点数量 115 | */ 116 | protected Integer getVirtualNodeCountOf(T physicalNode) { 117 | int countVirtualNode = 0; 118 | for (VirtualNode virtualNode : ring.values()) { 119 | if (virtualNode.isVirtualOf(physicalNode)) { 120 | countVirtualNode++; 121 | } 122 | } 123 | return countVirtualNode; 124 | } 125 | 126 | /** 127 | * 默认 hash 实现 128 | */ 129 | public static class MD5Hash implements HashAlgorithm { 130 | MessageDigest instance; 131 | 132 | public MD5Hash() { 133 | try { 134 | instance = MessageDigest.getInstance("MD5"); 135 | } catch (NoSuchAlgorithmException e) { 136 | logger.error("获取 MD5 加密实例失败"); 137 | } 138 | } 139 | 140 | @Override 141 | public long hash(String key) { 142 | instance.reset(); 143 | instance.update(key.getBytes()); 144 | byte[] digest = instance.digest(); 145 | long h = 0; 146 | for (int i = 0; i < 4; i++) { 147 | h <<= 8; 148 | h |= (digest[i]) & 0xFF; 149 | } 150 | return h; 151 | } 152 | } 153 | 154 | } 155 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/filter/CustomReactiveLoadBalanceFilter.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.filter; 2 | 3 | import me.lawrenceli.gateway.config.WebSocketProperties; 4 | import me.lawrenceli.gateway.server.ServiceNode; 5 | import me.lawrenceli.hashring.ConsistentHashRouter; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.beans.factory.config.BeanPostProcessor; 9 | import org.springframework.cloud.client.ServiceInstance; 10 | import org.springframework.cloud.client.discovery.DiscoveryClient; 11 | import org.springframework.cloud.client.loadbalancer.reactive.DefaultRequest; 12 | import org.springframework.cloud.client.loadbalancer.reactive.Request; 13 | import org.springframework.cloud.client.loadbalancer.reactive.Response; 14 | import org.springframework.cloud.gateway.config.LoadBalancerProperties; 15 | import org.springframework.cloud.gateway.filter.GatewayFilterChain; 16 | import org.springframework.cloud.gateway.filter.ReactiveLoadBalancerClientFilter; 17 | import org.springframework.cloud.gateway.support.DelegatingServiceInstance; 18 | import org.springframework.cloud.gateway.support.NotFoundException; 19 | import org.springframework.cloud.gateway.support.ServerWebExchangeUtils; 20 | import org.springframework.cloud.loadbalancer.core.ServiceInstanceListSupplier; 21 | import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory; 22 | import org.springframework.web.server.ServerWebExchange; 23 | import reactor.core.publisher.Mono; 24 | 25 | import java.net.URI; 26 | 27 | import static org.springframework.cloud.gateway.support.ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR; 28 | import static org.springframework.cloud.gateway.support.ServerWebExchangeUtils.GATEWAY_SCHEME_PREFIX_ATTR; 29 | import static org.springframework.cloud.gateway.support.ServerWebExchangeUtils.addOriginalRequestUrl; 30 | 31 | /** 32 | * 响应式网关的负载均衡过滤器 33 | * 34 | * @author lawrence 35 | * @since 2021/3/29 36 | */ 37 | public class CustomReactiveLoadBalanceFilter extends ReactiveLoadBalancerClientFilter implements BeanPostProcessor { 38 | 39 | private static final Logger logger = LoggerFactory.getLogger(CustomReactiveLoadBalanceFilter.class); 40 | 41 | final ConsistentHashRouter consistentHashRouter; 42 | final DiscoveryClient discoveryClient; 43 | final WebSocketProperties webSocketProperties; 44 | final LoadBalancerClientFactory clientFactory; 45 | final LoadBalancerProperties properties; 46 | 47 | public CustomReactiveLoadBalanceFilter(LoadBalancerClientFactory clientFactory, 48 | LoadBalancerProperties properties, 49 | ConsistentHashRouter consistentHashRouter, 50 | DiscoveryClient discoveryClient, 51 | WebSocketProperties webSocketProperties) { 52 | super(clientFactory, properties); 53 | this.clientFactory = clientFactory; 54 | this.properties = properties; 55 | this.consistentHashRouter = consistentHashRouter; 56 | this.discoveryClient = discoveryClient; 57 | this.webSocketProperties = webSocketProperties; 58 | } 59 | 60 | @Override 61 | public Mono filter(ServerWebExchange exchange, GatewayFilterChain chain) { 62 | URI url = exchange.getAttribute(GATEWAY_REQUEST_URL_ATTR); 63 | String schemePrefix = exchange.getAttribute(GATEWAY_SCHEME_PREFIX_ATTR); 64 | if (url == null 65 | || (!"lb".equals(url.getScheme()) && !"lb".equals(schemePrefix))) { 66 | return chain.filter(exchange); 67 | } 68 | // preserve the original url 69 | addOriginalRequestUrl(exchange, url); 70 | 71 | if (logger.isTraceEnabled()) { 72 | logger.trace("{} url before: {}", ReactiveLoadBalancerClientFilter.class.getSimpleName(), url); 73 | } 74 | 75 | return chooseInstance(exchange).doOnNext(response -> { 76 | if (!response.hasServer()) { 77 | throw NotFoundException.create(properties.isUse404(), 78 | "Unable to find instance for " + url.getHost()); 79 | } 80 | 81 | ServiceInstance retrievedInstance = response.getServer(); 82 | 83 | URI uri = exchange.getRequest().getURI(); 84 | 85 | // if the `lb:` mechanism was used, use `` as the default, 86 | // if the loadbalancer doesn't provide one. 87 | String overrideScheme = retrievedInstance.isSecure() ? "https" : "http"; 88 | if (schemePrefix != null) { 89 | overrideScheme = url.getScheme(); 90 | } 91 | 92 | DelegatingServiceInstance serviceInstance = new DelegatingServiceInstance( 93 | retrievedInstance, overrideScheme); 94 | 95 | URI requestUrl = reconstructURI(serviceInstance, uri); 96 | 97 | if (logger.isTraceEnabled()) { 98 | logger.trace("LoadBalancerClientFilter url chosen: {}", requestUrl); 99 | } 100 | exchange.getAttributes().put(GATEWAY_REQUEST_URL_ATTR, requestUrl); 101 | }).then(chain.filter(exchange)); 102 | } 103 | 104 | @SuppressWarnings("deprecation") 105 | private Mono> chooseInstance(ServerWebExchange exchange) { 106 | URI uri = exchange.getAttribute(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR); 107 | assert uri != null; 108 | WebSocketSessionLoadBalancer loadBalancer = new WebSocketSessionLoadBalancer( 109 | clientFactory.getLazyProvider(uri.getHost(), ServiceInstanceListSupplier.class), 110 | consistentHashRouter, 111 | discoveryClient, 112 | webSocketProperties); 113 | return loadBalancer.choose(this.createRequest(exchange)); 114 | } 115 | 116 | @SuppressWarnings("deprecation") 117 | private Request createRequest(ServerWebExchange exchange) { 118 | // 其实返回的 Request 对象里至少需要包含的就是负载均衡时 choose 的依据 119 | return new DefaultRequest<>(exchange); 120 | } 121 | 122 | } 123 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/server/RedisSubscriber.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.server; 2 | 3 | import me.lawrenceli.constant.GlobalConstant; 4 | import me.lawrenceli.gateway.config.RedisConfig; 5 | import me.lawrenceli.gateway.config.WebSocketProperties; 6 | import me.lawrenceli.hashring.ConsistentHashRouter; 7 | import me.lawrenceli.hashring.VirtualNode; 8 | import me.lawrenceli.model.WebSocketMessage; 9 | import me.lawrenceli.utils.JSON; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import org.springframework.cloud.client.discovery.DiscoveryClient; 13 | import org.springframework.data.redis.core.RedisTemplate; 14 | import org.springframework.stereotype.Component; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import java.util.Map; 19 | import java.util.SortedMap; 20 | import java.util.concurrent.ConcurrentHashMap; 21 | 22 | /** 23 | * 使用 Redis pub/sub 订阅消息 24 | * 25 | * @author lawrence 26 | * @since 2021/3/23 27 | */ 28 | @Component 29 | public class RedisSubscriber { 30 | 31 | private static final Logger logger = LoggerFactory.getLogger(RedisSubscriber.class); 32 | 33 | final DiscoveryClient discoveryClient; // spring cloud native interface 34 | final WebSocketProperties webSocketProperties; 35 | final RedisTemplate redisTemplate; 36 | final FanoutSender fanoutSender; 37 | private ConsistentHashRouter consistentHashRouter; 38 | 39 | public RedisSubscriber(DiscoveryClient discoveryClient, 40 | WebSocketProperties webSocketProperties, 41 | ConsistentHashRouter consistentHashRouter, 42 | RedisTemplate redisTemplate, FanoutSender fanoutSender) { 43 | this.discoveryClient = discoveryClient; 44 | this.webSocketProperties = webSocketProperties; 45 | this.consistentHashRouter = consistentHashRouter; 46 | this.redisTemplate = redisTemplate; 47 | this.fanoutSender = fanoutSender; 48 | } 49 | 50 | public void setConsistentHashRouter(ConsistentHashRouter consistentHashRouter) { 51 | this.consistentHashRouter = consistentHashRouter; 52 | } 53 | 54 | /** 55 | * Redis 订阅者消费, 默认方法名为 handleMessage, 56 | * 57 | * @param webSocketMessageJSON WebSocket 实例上下线事件的消息： WebSocketMessage 对象的序列化字符串 58 | * @see WebSocket 集群方案 59 | * @see RedisConfig 60 | */ 61 | @SuppressWarnings("unused") 62 | public void handleMessage(String webSocketMessageJSON) throws InterruptedException { 63 | WebSocketMessage webSocketMessage = JSON.parseJSON(webSocketMessageJSON, WebSocketMessage.class); 64 | logger.info("【Redis 订阅】网关收到 WebSocket 实例变化消息: {}", webSocketMessage); 65 | String upOrDown = webSocketMessage.getContent(); 66 | // 实例的标识：IP 67 | String serverIp = webSocketMessage.getServerIp(); 68 | logger.info("【哈希环】该实例上线之前为 {}, 稍后将更新...", JSON.toJSONString(consistentHashRouter.getRing())); // NOSONAR 69 | if (GlobalConstant.SERVER_UP_MESSAGE.equalsIgnoreCase(upOrDown)) { 70 | // 实例上线, 但 Nacos 可能尚未发现服务，此处再等 Nacos 获取到最新服务列表 71 | Thread.sleep(3000); 72 | // 一个服务上线了，应当告知原本哈希到其他节点但现在路由到此节点的所有客户端断开连接 73 | // 为了确定是哪些客户端需要重连，可以遍历所有 userId 和哈希，筛选出节点添加前后匹配到不同真实节点的所有 userId 74 | // 因此，每次 WebSocket 有新连接(onOpen)的时候都有必要将 userId(+hash) 保存在 redis 中，然后在节点变动时取出 75 | Map userIdAndHashInRedis = redisTemplate.opsForHash().entries(GlobalConstant.KEY_TO_BE_HASHED); 76 | logger.debug("Redis 中 userId hash : {}", userIdAndHashInRedis); 77 | Map oldUserAndServer = new ConcurrentHashMap<>(); 78 | for (Object userIdObj : userIdAndHashInRedis.keySet()) { 79 | String userId = (String) userIdObj; 80 | Long oldHashObj = Long.valueOf(userIdAndHashInRedis.get(userId).toString()); 81 | ServiceNode oldServiceNode = consistentHashRouter.routeNode(userId); 82 | logger.debug("【遍历】当前客户端 [{}] 的旧节点 [{}]", userId, oldServiceNode); 83 | // 如果 WebSocket 实例上线之前就有了客户端的连接，重连间隙可能只有几秒，极有可能此时哈希环是空的 84 | // https://github.com/Lonor/websocket-cluster/issues/2 85 | if (null != oldServiceNode) { 86 | oldUserAndServer.put(userId, oldServiceNode); 87 | } 88 | } 89 | // 向 Hash 环添加 node 90 | ServiceNode serviceNode = new ServiceNode(serverIp); 91 | consistentHashRouter.addNode(serviceNode, GlobalConstant.VIRTUAL_COUNT); 92 | // 添加了 node 之后就可能有部分 userId 路由到的真实服务节点发生变动 93 | List userIdClientsToReset = new ArrayList<>(); 94 | for (Map.Entry entry : oldUserAndServer.entrySet()) { 95 | ServiceNode newServiceNode = consistentHashRouter.routeNode(entry.getKey()); 96 | logger.debug("【遍历】当前客户端 [{}] 的新节点 [{}]", entry.getKey(), newServiceNode); 97 | // 同一 userId 路由到的真实服务节点前后可能会不一样, 把这些 userId 筛选出来 98 | if (!newServiceNode.getKey().equals(entry.getValue().getKey())) { 99 | userIdClientsToReset.add(entry.getKey()); 100 | logger.info("【哈希环更新】客户端在哈希环的映射服务节点发生了变动: [{}]: [{}] -> [{}]", entry.getKey(), entry.getValue(), newServiceNode); 101 | } 102 | } 103 | // 通知部分客户端断开连接, 可以发一个全局广播让客户端断开，也可以服务端主动断开，其实就是这些客户端都要自动连接上新的实例 104 | fanoutSender.send(userIdClientsToReset); 105 | } 106 | if (GlobalConstant.SERVER_DOWN_MESSAGE.equalsIgnoreCase(upOrDown)) { 107 | // 实例下线, 服务端已经立刻主动断连，网关也要移除掉对应节点 108 | ServiceNode serviceNode = new ServiceNode(serverIp); 109 | consistentHashRouter.removeNode(serviceNode); 110 | } 111 | // 将最新的哈希环放到 Redis 112 | SortedMap> ring = consistentHashRouter.getRing(); 113 | redisTemplate.opsForHash().putAll(GlobalConstant.HASH_RING_REDIS, ring); 114 | logger.info("【哈希环】实例上线之后为 {}", JSON.toJSONString(ring)); // NOSONAR 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /websocket/src/main/java/me/lawrenceli/websocket/server/WebSocketEndpoint.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.server; 2 | 3 | import me.lawrenceli.constant.GlobalConstant; 4 | import me.lawrenceli.hashring.ConsistentHashRouter; 5 | import me.lawrenceli.model.MessageType; 6 | import me.lawrenceli.model.WebSocketMessage; 7 | import me.lawrenceli.utils.JSON; 8 | import me.lawrenceli.websocket.spring.BeanUtils; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.springframework.data.redis.core.RedisTemplate; 12 | import org.springframework.stereotype.Component; 13 | 14 | import javax.websocket.OnClose; 15 | import javax.websocket.OnError; 16 | import javax.websocket.OnOpen; 17 | import javax.websocket.Session; 18 | import javax.websocket.server.PathParam; 19 | import javax.websocket.server.ServerEndpoint; 20 | import java.io.IOException; 21 | import java.util.List; 22 | import java.util.Map; 23 | import java.util.concurrent.ConcurrentHashMap; 24 | import java.util.concurrent.ConcurrentMap; 25 | 26 | /** 27 | * @author lawrence 28 | * @since 2021/3/19 29 | */ 30 | @Component("websocketEndpoint") 31 | @ServerEndpoint(GlobalConstant.WEBSOCKET_ENDPOINT_PATH + "/{userId}") 32 | public class WebSocketEndpoint { 33 | private static final Logger logger = LoggerFactory.getLogger(WebSocketEndpoint.class); 34 | 35 | /** 36 | * Websocket session 内存 37 | * key: userId 38 | * value: websocket session 39 | */ 40 | public static final ConcurrentMap sessionMap = new ConcurrentHashMap<>(); // NOSONAR 41 | 42 | // Do not make DI with `@autowired` or constructor in this class with `@ServerEndpoint` annotated. 43 | 44 | /** 45 | * 服务端发送消息给客户端 46 | * 47 | * @param session WebSocket 连接 48 | * @param message 消息内容 49 | * @return 是否发送成功 50 | */ 51 | protected Boolean sendMessageBySession(Session session, String message) { 52 | if (session != null) { 53 | try { 54 | session.getBasicRemote().sendText(message); 55 | } catch (IOException ignore) { 56 | logger.error("WebSocket 通信时发生未知的 IO 异常 [message{}, session{}]", message, session); 57 | return false; 58 | } 59 | return true; 60 | } else { 61 | logger.debug("当前用户未连接，客户端提示失败"); 62 | return false; 63 | } 64 | } 65 | 66 | public Boolean sendMessageToUser(String userId, WebSocketMessage webSocketMessage) { 67 | Session session = sessionMap.get(userId); 68 | try { 69 | logger.debug("向 {} - session {} 发送消息: {}", userId, session.getId(), webSocketMessage); // NOSONAR 70 | return this.sendMessageBySession(session, JSON.toJSONString(webSocketMessage)); 71 | } catch (Exception e) { 72 | logger.error("服务端发给用户 {} 发送消息异常: {}", userId, e.getMessage()); 73 | return false; 74 | } 75 | } 76 | 77 | @OnOpen 78 | public void onOpen(Session session, @PathParam(value = "userId") String userId) { 79 | sessionMap.put(userId, session); 80 | // 将 userId，hash(userId) 保存到全局 redis，是为了有新的实例上线时筛选部分变动的客户端主动重新连接到新的实例上 81 | RedisTemplate redisTemplate = (RedisTemplate) BeanUtils.getBean("redisTemplate"); 82 | redisTemplate.opsForHash().put(GlobalConstant.KEY_TO_BE_HASHED, userId, String.valueOf(new ConsistentHashRouter.MD5Hash().hash(userId))); 83 | logger.info("客户端用户 {} 连接, 当前实例连接数: [{}] ", userId, sessionMap.size()); 84 | // 发一条消息告诉客户端情况 85 | WebSocketMessage webSocketMessage = WebSocketMessage.toUserOrServerMessage(MessageType.FOR_SERVER, "已连接", sessionMap.size()); 86 | this.sendMessageToUser(userId, webSocketMessage); 87 | } 88 | 89 | @OnError 90 | public void onError(Throwable throwable) { 91 | logger.error("WebSocket 出现错误: {} ", throwable.getMessage()); 92 | } 93 | 94 | @OnClose 95 | public void onClose(@PathParam(value = "userId") String userId) { 96 | sessionMap.remove(userId); 97 | RedisTemplate redisTemplate = (RedisTemplate) BeanUtils.getBean("redisTemplate"); 98 | redisTemplate.opsForHash().delete(GlobalConstant.KEY_TO_BE_HASHED, userId, new ConsistentHashRouter.MD5Hash().hash(userId)); 99 | logger.info("客户端用户 {} 断开连接, 从内存和 Redis 中移除", userId); 100 | // 关闭连接时 sessionMap 的 size 即连接数需要立刻更新到前端服务列表的 userCount 中 101 | // 为了不批量发送，定义一个是否发出去的标记，确保最多发送一次 102 | boolean sent = false; 103 | if (sessionMap.size() > 0) { 104 | // 获取不是当前 session 的其他客户端 105 | for (String anyClientUserId : sessionMap.keySet()) { 106 | if (!sent) { 107 | // 若未发送，则尝试发送，成功后修改标记 108 | logger.debug("通知客户端更新服务连接数"); 109 | sent = sendMessageToUser(anyClientUserId, WebSocketMessage.toUserOrServerMessage(MessageType.FOR_SERVER, "更新服务列表", sessionMap.size())); 110 | } 111 | } 112 | } 113 | } 114 | 115 | public static void disconnectAllByServer() { 116 | for (Map.Entry sessionEntry : sessionMap.entrySet()) { 117 | Session session = sessionEntry.getValue(); 118 | try { 119 | session.close(); 120 | logger.info("当前服务端已主动断开 {} 的连接", sessionEntry.getKey()); 121 | } catch (IOException e) { 122 | logger.error("服务端主动断开 {} 连接发生异常: {}", sessionEntry.getKey(), e.getMessage()); 123 | e.printStackTrace(); 124 | } 125 | } 126 | } 127 | 128 | public static void disconnectSomeByServer(List userIds) { 129 | for (String userId : userIds) { 130 | if (sessionMap.containsKey(userId)) { 131 | Session session = sessionMap.get(userId); 132 | logger.info("【MQ 通知重连】当前服务端已主动断开 {} 的连接", userId); 133 | try { 134 | session.close(); 135 | } catch (IOException e) { 136 | logger.error("服务端主动断开 {} 连接发生异常: {}", userId, e.getMessage()); 137 | e.printStackTrace(); 138 | } 139 | } 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /gateway/src/test/java/me/lawrenceli/gateway/discovery/DiscoveryControllerTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.discovery; 2 | 3 | import com.alibaba.nacos.api.exception.NacosException; 4 | import com.alibaba.nacos.api.naming.pojo.Instance; 5 | import com.alibaba.nacos.client.naming.NacosNamingService; 6 | import me.lawrenceli.gateway.config.WebSocketProperties; 7 | import org.junit.jupiter.api.Test; 8 | import org.springframework.http.HttpStatus; 9 | import org.springframework.http.ResponseEntity; 10 | 11 | import java.util.ArrayList; 12 | import java.util.Map; 13 | import java.util.Objects; 14 | 15 | import static org.junit.jupiter.api.Assertions.assertEquals; 16 | import static org.junit.jupiter.api.Assertions.assertTrue; 17 | import static org.mockito.Mockito.any; 18 | import static org.mockito.Mockito.mock; 19 | import static org.mockito.Mockito.verify; 20 | import static org.mockito.Mockito.when; 21 | 22 | class DiscoveryControllerTest { 23 | @Test 24 | void testGetServerStatus() throws NacosException { 25 | // Diffblue Cover was unable to write a Spring test, 26 | // so wrote a non-Spring test instead. 27 | // Reason: R005 Unable to load class. 28 | // Class: javax.servlet.ServletContext 29 | // Please check that the class is available on your test runtime classpath. 30 | // See https://diff.blue/R005 to resolve this issue. 31 | 32 | NacosNamingService nacosNamingService = mock(NacosNamingService.class); 33 | when(nacosNamingService.getAllInstances(any())).thenReturn(new ArrayList<>()); 34 | 35 | WebSocketProperties.Service service = new WebSocketProperties.Service(); 36 | service.setName("42"); 37 | 38 | WebSocketProperties webSocketProperties = new WebSocketProperties(); 39 | webSocketProperties.setService(service); 40 | ResponseEntity> actualServerStatus = (new DiscoveryController(nacosNamingService, 41 | webSocketProperties)).getServerStatus(); 42 | assertTrue(actualServerStatus.hasBody()); 43 | assertEquals(HttpStatus.OK, actualServerStatus.getStatusCode()); 44 | assertTrue(actualServerStatus.getHeaders().isEmpty()); 45 | verify(nacosNamingService).getAllInstances(any()); 46 | } 47 | 48 | @Test 49 | void testGetServerStatus2() throws NacosException { 50 | ArrayList instanceList = new ArrayList<>(); 51 | instanceList.add(new Instance()); 52 | NacosNamingService nacosNamingService = mock(NacosNamingService.class); 53 | when(nacosNamingService.getAllInstances(any())).thenReturn(instanceList); 54 | 55 | WebSocketProperties.Service service = new WebSocketProperties.Service(); 56 | service.setName("42"); 57 | 58 | WebSocketProperties webSocketProperties = new WebSocketProperties(); 59 | webSocketProperties.setService(service); 60 | ResponseEntity> actualServerStatus = (new DiscoveryController(nacosNamingService, 61 | webSocketProperties)).getServerStatus(); 62 | assertEquals(1, Objects.requireNonNull(actualServerStatus.getBody()).size()); 63 | assertTrue(actualServerStatus.hasBody()); 64 | assertEquals(HttpStatus.OK, actualServerStatus.getStatusCode()); 65 | assertTrue(actualServerStatus.getHeaders().isEmpty()); 66 | verify(nacosNamingService).getAllInstances(any()); 67 | } 68 | 69 | @Test 70 | void testGetServerStatus3() throws NacosException { 71 | Instance instance = mock(Instance.class); 72 | when(instance.isHealthy()).thenReturn(false); 73 | when(instance.isEnabled()).thenReturn(true); 74 | when(instance.getIp()).thenReturn("127.0.0.1"); 75 | 76 | ArrayList instanceList = new ArrayList<>(); 77 | instanceList.add(instance); 78 | NacosNamingService nacosNamingService = mock(NacosNamingService.class); 79 | when(nacosNamingService.getAllInstances(any())).thenReturn(instanceList); 80 | 81 | WebSocketProperties.Service service = new WebSocketProperties.Service(); 82 | service.setName("42"); 83 | 84 | WebSocketProperties webSocketProperties = new WebSocketProperties(); 85 | webSocketProperties.setService(service); 86 | ResponseEntity> actualServerStatus = (new DiscoveryController(nacosNamingService, 87 | webSocketProperties)).getServerStatus(); 88 | assertEquals(1, Objects.requireNonNull(actualServerStatus.getBody()).size()); 89 | assertTrue(actualServerStatus.hasBody()); 90 | assertEquals(HttpStatus.OK, actualServerStatus.getStatusCode()); 91 | assertTrue(actualServerStatus.getHeaders().isEmpty()); 92 | verify(nacosNamingService).getAllInstances(any()); 93 | verify(instance).isEnabled(); 94 | verify(instance).isHealthy(); 95 | verify(instance).getIp(); 96 | } 97 | 98 | @Test 99 | void testGetServerStatus4() throws NacosException { 100 | Instance instance = mock(Instance.class); 101 | when(instance.isHealthy()).thenReturn(true); 102 | when(instance.isEnabled()).thenReturn(false); 103 | when(instance.getIp()).thenReturn("127.0.0.1"); 104 | 105 | ArrayList instanceList = new ArrayList<>(); 106 | instanceList.add(instance); 107 | NacosNamingService nacosNamingService = mock(NacosNamingService.class); 108 | when(nacosNamingService.getAllInstances(any())).thenReturn(instanceList); 109 | 110 | WebSocketProperties.Service service = new WebSocketProperties.Service(); 111 | service.setName("42"); 112 | 113 | WebSocketProperties webSocketProperties = new WebSocketProperties(); 114 | webSocketProperties.setService(service); 115 | ResponseEntity> actualServerStatus = (new DiscoveryController(nacosNamingService, 116 | webSocketProperties)).getServerStatus(); 117 | assertEquals(1, Objects.requireNonNull(actualServerStatus.getBody()).size()); 118 | assertTrue(actualServerStatus.hasBody()); 119 | assertEquals(HttpStatus.OK, actualServerStatus.getStatusCode()); 120 | assertTrue(actualServerStatus.getHeaders().isEmpty()); 121 | verify(nacosNamingService).getAllInstances(any()); 122 | verify(instance).isEnabled(); 123 | verify(instance).getIp(); 124 | } 125 | } 126 | 127 | -------------------------------------------------------------------------------- /websocket/src/test/java/me/lawrenceli/websocket/server/WebSocketEndpointTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.websocket.server; 2 | 3 | import com.sun.security.auth.UserPrincipal; 4 | import me.lawrenceli.model.WebSocketMessage; 5 | import org.apache.tomcat.websocket.WsRemoteEndpointImplClient; 6 | import org.apache.tomcat.websocket.WsSession; 7 | import org.apache.tomcat.websocket.WsWebSocketContainer; 8 | import org.apache.tomcat.websocket.pojo.PojoEndpointServer; 9 | import org.junit.jupiter.api.Test; 10 | import org.junit.jupiter.api.extension.ExtendWith; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.junit.jupiter.SpringExtension; 14 | import org.springframework.web.socket.server.standard.ServerEndpointRegistration; 15 | 16 | import javax.websocket.DeploymentException; 17 | import javax.websocket.Extension; 18 | import java.io.IOException; 19 | import java.net.URI; 20 | import java.nio.file.Paths; 21 | import java.time.LocalDate; 22 | import java.time.LocalDateTime; 23 | import java.time.ZoneId; 24 | import java.util.ArrayList; 25 | import java.util.Date; 26 | import java.util.HashMap; 27 | import java.util.List; 28 | 29 | import static org.junit.jupiter.api.Assertions.assertFalse; 30 | import static org.junit.jupiter.api.Assertions.assertTrue; 31 | import static org.mockito.Mockito.any; 32 | import static org.mockito.Mockito.anyLong; 33 | import static org.mockito.Mockito.doNothing; 34 | import static org.mockito.Mockito.doThrow; 35 | import static org.mockito.Mockito.mock; 36 | import static org.mockito.Mockito.verify; 37 | 38 | @ContextConfiguration(classes = {WebSocketEndpoint.class}) 39 | @ExtendWith(SpringExtension.class) 40 | class WebSocketEndpointTest { 41 | @Autowired 42 | private WebSocketEndpoint webSocketEndpoint; 43 | 44 | @Test 45 | void testSendMessageBySession() throws IOException, DeploymentException { 46 | WsRemoteEndpointImplClient wsRemoteEndpointImplClient = mock(WsRemoteEndpointImplClient.class); 47 | doNothing().when(wsRemoteEndpointImplClient).sendString(any()); 48 | doNothing().when(wsRemoteEndpointImplClient).setSendTimeout(anyLong()); 49 | PojoEndpointServer localEndpoint = new PojoEndpointServer(); 50 | WsWebSocketContainer wsWebSocketContainer = new WsWebSocketContainer(); 51 | URI requestUri = Paths.get(System.getProperty("java.io.tmpdir"), "test.txt").toUri(); 52 | HashMap> requestParameterMap = new HashMap<>(); 53 | UserPrincipal userPrincipal = new UserPrincipal("userPrincipal"); 54 | ArrayList negotiatedExtensions = new ArrayList<>(); 55 | HashMap pathParameters = new HashMap<>(); 56 | assertTrue( 57 | this.webSocketEndpoint.sendMessageBySession( 58 | new WsSession(localEndpoint, wsRemoteEndpointImplClient, wsWebSocketContainer, requestUri, 59 | requestParameterMap, "Query String", userPrincipal, "42", negotiatedExtensions, "Sub Protocol", 60 | pathParameters, true, new ServerEndpointRegistration("Path", new PojoEndpointServer())), 61 | "Not all who wander are lost")); 62 | verify(wsRemoteEndpointImplClient).sendString(any()); 63 | verify(wsRemoteEndpointImplClient).setSendTimeout(anyLong()); 64 | } 65 | 66 | @Test 67 | void testSendMessageBySession2() { 68 | assertFalse(this.webSocketEndpoint.sendMessageBySession(null, "Not all who wander are lost")); 69 | } 70 | 71 | @Test 72 | void testSendMessageBySession3() throws IOException, DeploymentException { 73 | WsRemoteEndpointImplClient wsRemoteEndpointImplClient = mock(WsRemoteEndpointImplClient.class); 74 | doThrow(new IOException("foo")).when(wsRemoteEndpointImplClient).sendString(any()); 75 | doNothing().when(wsRemoteEndpointImplClient).setSendTimeout(anyLong()); 76 | PojoEndpointServer localEndpoint = new PojoEndpointServer(); 77 | WsWebSocketContainer wsWebSocketContainer = new WsWebSocketContainer(); 78 | URI requestUri = Paths.get(System.getProperty("java.io.tmpdir"), "test.txt").toUri(); 79 | HashMap> requestParameterMap = new HashMap<>(); 80 | UserPrincipal userPrincipal = new UserPrincipal("userPrincipal"); 81 | ArrayList negotiatedExtensions = new ArrayList<>(); 82 | HashMap pathParameters = new HashMap<>(); 83 | assertFalse( 84 | this.webSocketEndpoint.sendMessageBySession( 85 | new WsSession(localEndpoint, wsRemoteEndpointImplClient, wsWebSocketContainer, requestUri, 86 | requestParameterMap, "Query String", userPrincipal, "42", negotiatedExtensions, "Sub Protocol", 87 | pathParameters, true, new ServerEndpointRegistration("Path", new PojoEndpointServer())), 88 | "Not all who wander are lost")); 89 | verify(wsRemoteEndpointImplClient).sendString(any()); 90 | verify(wsRemoteEndpointImplClient).setSendTimeout(anyLong()); 91 | } 92 | 93 | @Test 94 | void testSendMessageToUser() { 95 | WebSocketMessage webSocketMessage = new WebSocketMessage(); 96 | webSocketMessage.setContent("Not all who wander are lost"); 97 | webSocketMessage.setServerIp("Server Ip"); 98 | webSocketMessage.setServerUserCount(3); 99 | LocalDateTime atStartOfDayResult = LocalDate.of(1970, 1, 1).atStartOfDay(); 100 | webSocketMessage.setTimestamp(Date.from(atStartOfDayResult.atZone(ZoneId.of("UTC")).toInstant())); 101 | webSocketMessage.setType(1); 102 | assertFalse(this.webSocketEndpoint.sendMessageToUser("42", webSocketMessage)); 103 | } 104 | 105 | @Test 106 | void testSendMessageToUser2() { 107 | WebSocketMessage webSocketMessage = new WebSocketMessage(); 108 | webSocketMessage.setContent("Not all who wander are lost"); 109 | webSocketMessage.setServerIp("Server Ip"); 110 | webSocketMessage.setServerUserCount(3); 111 | LocalDateTime atStartOfDayResult = LocalDate.of(1970, 1, 1).atStartOfDay(); 112 | webSocketMessage.setTimestamp(Date.from(atStartOfDayResult.atZone(ZoneId.of("UTC")).toInstant())); 113 | webSocketMessage.setType(1); 114 | assertFalse(this.webSocketEndpoint.sendMessageToUser("向 {} - session {} 发送消息: {}", webSocketMessage)); 115 | } 116 | 117 | } 118 | 119 | -------------------------------------------------------------------------------- /gateway/src/main/java/me/lawrenceli/gateway/filter/WebSocketSessionLoadBalancer.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.gateway.filter; 2 | 3 | import me.lawrenceli.constant.GlobalConstant; 4 | import me.lawrenceli.gateway.config.WebSocketProperties; 5 | import me.lawrenceli.gateway.server.ServiceNode; 6 | import me.lawrenceli.hashring.ConsistentHashRouter; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.beans.factory.ObjectProvider; 10 | import org.springframework.cloud.client.ServiceInstance; 11 | import org.springframework.cloud.client.discovery.DiscoveryClient; 12 | import org.springframework.cloud.client.loadbalancer.reactive.DefaultResponse; 13 | import org.springframework.cloud.client.loadbalancer.reactive.Request; 14 | import org.springframework.cloud.client.loadbalancer.reactive.Response; 15 | import org.springframework.cloud.gateway.support.ServerWebExchangeUtils; 16 | import org.springframework.cloud.loadbalancer.core.NoopServiceInstanceListSupplier; 17 | import org.springframework.cloud.loadbalancer.core.ReactorServiceInstanceLoadBalancer; 18 | import org.springframework.cloud.loadbalancer.core.RoundRobinLoadBalancer; 19 | import org.springframework.cloud.loadbalancer.core.ServiceInstanceListSupplier; 20 | import org.springframework.http.server.PathContainer; 21 | import org.springframework.util.MultiValueMap; 22 | import org.springframework.web.server.ServerWebExchange; 23 | import reactor.core.publisher.Mono; 24 | 25 | import java.net.URI; 26 | import java.util.List; 27 | 28 | /** 29 | * WebSocket 负载均衡器 30 | * 31 | * @author lawrence 32 | * @since 2021/3/29 33 | */ 34 | public class WebSocketSessionLoadBalancer implements ReactorServiceInstanceLoadBalancer { 35 | 36 | private static final Logger logger = LoggerFactory.getLogger(WebSocketSessionLoadBalancer.class); 37 | 38 | final ConsistentHashRouter consistentHashRouter; 39 | final DiscoveryClient discoveryClient; 40 | final WebSocketProperties webSocketProperties; 41 | private final ObjectProvider serviceInstanceListSupplierProvider; 42 | 43 | public WebSocketSessionLoadBalancer(ObjectProvider serviceInstanceListSupplierProvider, 44 | ConsistentHashRouter consistentHashRouter, 45 | DiscoveryClient discoveryClient, 46 | WebSocketProperties webSocketProperties) { 47 | this.serviceInstanceListSupplierProvider = serviceInstanceListSupplierProvider; 48 | this.consistentHashRouter = consistentHashRouter; 49 | this.discoveryClient = discoveryClient; 50 | this.webSocketProperties = webSocketProperties; 51 | } 52 | 53 | @Override 54 | @SuppressWarnings("deprecation") 55 | public Mono> choose(Request request) { 56 | ServerWebExchange exchange = (ServerWebExchange) request.getContext(); 57 | URI originalUrl = (URI) exchange.getAttributes().get(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR); 58 | String instancesId = originalUrl.getHost(); 59 | if (webSocketProperties.getService().getName().equals(instancesId)) { 60 | // 获取需要参与哈希的字段，此项目为 userId 61 | final String userIdFromRequest = getUserIdFromRequest(exchange); 62 | if (null != userIdFromRequest && null != this.serviceInstanceListSupplierProvider) { 63 | // 请求参数中有 userId，需要经过哈希环的路由 64 | ServiceInstanceListSupplier supplier = serviceInstanceListSupplierProvider.getIfAvailable(NoopServiceInstanceListSupplier::new); 65 | return (supplier.get()).next().map(list -> getServiceInstanceByUserId(userIdFromRequest, instancesId)); 66 | } 67 | } 68 | return choose(); 69 | } 70 | 71 | @Override 72 | @SuppressWarnings("deprecation") 73 | public Mono> choose() { 74 | logger.debug("【RoundRobin】无 userId 的请求，对于 WebSocket 服务也没有意义，轮询转发..."); 75 | // Round-Robin: 该术语来源于含义为「带子」的法语词 ruban，久而被讹用并成为惯用语。在 17、18 世纪时法国农民希望以请愿的方式抗议国王时， 76 | // 通常君主的反应是将请愿书中最前面的两至三人逮捕并处决，所以很自然地没有人希望自己的名字被列在前面。为了对付这种专制的报复， 77 | // 人们在请愿书底部把名字签成一个圈（如同一条环状的带子），这样就找不出打头的人，于是只能对所有参与者进行同样的惩罚。 78 | RoundRobinLoadBalancer roundRobinLoadBalancer = new RoundRobinLoadBalancer(serviceInstanceListSupplierProvider, webSocketProperties.getService().getName()); 79 | return roundRobinLoadBalancer.choose(); 80 | } 81 | 82 | /** 83 | * 哈希环的使用，根据 userId 来查询对应的节点 84 | * 85 | * @param userId 用户ID，关联了 WebSocket Session 86 | * @param instancesId 服务名 87 | * @return 服务实例的 Response 88 | */ 89 | @SuppressWarnings("deprecation") 90 | private Response getServiceInstanceByUserId(final String userId, String instancesId) { 91 | ServiceNode serviceNode = consistentHashRouter.routeNode(userId); 92 | if (null != serviceNode) { 93 | // 获取当前注册中心的实例 94 | List instances = discoveryClient.getInstances(instancesId); 95 | for (ServiceInstance instance : instances) { 96 | // 如果 userId 映射后的真实节点的 IP 与某个实例 IP 一致，就转发 97 | if (instance.getHost().equals(serviceNode.getKey())) { 98 | logger.debug("当前客户端[{}]匹配到真实节点 {}", userId, serviceNode.getKey()); 99 | return new DefaultResponse(instance); 100 | } 101 | } 102 | } 103 | logger.warn("网关监测到当前无哈希环, 即无 WebSocket 服务实例，尝试取第一个实例，可能为 null"); 104 | return new DefaultResponse(discoveryClient.getInstances(instancesId).get(0)); 105 | } 106 | 107 | /** 108 | * 从 WS/HTTP 请求 中获取待哈希字段 userId 109 | * 110 | * @param exchange 请求上下文 111 | * @return userId，可能为空 112 | */ 113 | protected static String getUserIdFromRequest(ServerWebExchange exchange) { 114 | URI originalUrl = (URI) exchange.getAttributes().get(ServerWebExchangeUtils.GATEWAY_REQUEST_URL_ATTR); 115 | String userId = null; 116 | if (originalUrl.getPath().startsWith(GlobalConstant.WEBSOCKET_ENDPOINT_PATH)) { 117 | // ws: "lb://websocket-server/connect/1" 获取这里面的最后一个路径参数 userId: 1 118 | List elements = exchange.getRequest().getPath().elements(); 119 | PathContainer.Element lastElement = elements.get(elements.size() - 1); 120 | userId = lastElement.value(); 121 | logger.debug("【网关负载均衡】WebSocket 获取到 userId: {}", userId); 122 | } else { 123 | // 前提：websocket http 服务 userId 放在 query 中 124 | // rest: "lb://websocket-server/send?userId=1&message=text" 125 | MultiValueMap queryParams = exchange.getRequest().getQueryParams(); 126 | List userIds = queryParams.get(GlobalConstant.KEY_TO_BE_HASHED); 127 | if (null != userIds && !userIds.isEmpty()) { 128 | userId = userIds.get(0); 129 | logger.debug("【网关负载均衡】HTTP 获取到 userId: {}", userId); 130 | } 131 | } 132 | return userId; 133 | } 134 | 135 | } 136 | -------------------------------------------------------------------------------- /common/src/test/java/me/lawrenceli/utils/StringPatternTest.java: -------------------------------------------------------------------------------- 1 | package me.lawrenceli.utils; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import static org.junit.jupiter.api.Assertions.assertEquals; 6 | 7 | class StringPatternTest { 8 | 9 | @Test 10 | void testReplacePatternBreaking() { // NOSONAR 11 | assertEquals("String", StringPattern.replacePatternBreaking("String")); 12 | assertEquals("_", StringPattern.replacePatternBreaking("\t")); 13 | assertEquals("[___]", StringPattern.replacePatternBreaking("[\n\r\t]")); 14 | assertEquals("__", StringPattern.replacePatternBreaking("\t\t")); 15 | assertEquals("_[___]", StringPattern.replacePatternBreaking("\t[\n\r\t]")); 16 | assertEquals("__", StringPattern.replacePatternBreaking("\t_")); 17 | assertEquals("_String", StringPattern.replacePatternBreaking("\tString")); 18 | assertEquals("_42", StringPattern.replacePatternBreaking("\t42")); 19 | assertEquals("[___]_", StringPattern.replacePatternBreaking("[\n\r\t]\t")); 20 | assertEquals("[___][___]", StringPattern.replacePatternBreaking("[\n\r\t][\n\r\t]")); 21 | assertEquals("[___]_", StringPattern.replacePatternBreaking("[\n\r\t]_")); 22 | assertEquals("[___]String", StringPattern.replacePatternBreaking("[\n\r\t]String")); 23 | assertEquals("[___]42", StringPattern.replacePatternBreaking("[\n\r\t]42")); 24 | assertEquals("__", StringPattern.replacePatternBreaking("_\t")); 25 | assertEquals("_[___]", StringPattern.replacePatternBreaking("_[\n\r\t]")); 26 | assertEquals("String_", StringPattern.replacePatternBreaking("String\t")); 27 | assertEquals("String[___]", StringPattern.replacePatternBreaking("String[\n\r\t]")); 28 | assertEquals("42_", StringPattern.replacePatternBreaking("42\t")); 29 | assertEquals("42[___]", StringPattern.replacePatternBreaking("42[\n\r\t]")); 30 | assertEquals("___", StringPattern.replacePatternBreaking("\t\t\t")); 31 | assertEquals("__[___]", StringPattern.replacePatternBreaking("\t\t[\n\r\t]")); 32 | assertEquals("___", StringPattern.replacePatternBreaking("\t\t_")); 33 | assertEquals("__String", StringPattern.replacePatternBreaking("\t\tString")); 34 | assertEquals("__42", StringPattern.replacePatternBreaking("\t\t42")); 35 | assertEquals("_[___]_", StringPattern.replacePatternBreaking("\t[\n\r\t]\t")); 36 | assertEquals("_[___][___]", StringPattern.replacePatternBreaking("\t[\n\r\t][\n\r\t]")); 37 | assertEquals("_[___]_", StringPattern.replacePatternBreaking("\t[\n\r\t]_")); 38 | assertEquals("_[___]String", StringPattern.replacePatternBreaking("\t[\n\r\t]String")); 39 | assertEquals("_[___]42", StringPattern.replacePatternBreaking("\t[\n\r\t]42")); 40 | assertEquals("___", StringPattern.replacePatternBreaking("\t_\t")); 41 | assertEquals("__[___]", StringPattern.replacePatternBreaking("\t_[\n\r\t]")); 42 | assertEquals("___", StringPattern.replacePatternBreaking("\t__")); 43 | assertEquals("__String", StringPattern.replacePatternBreaking("\t_String")); 44 | assertEquals("__42", StringPattern.replacePatternBreaking("\t_42")); 45 | assertEquals("_String_", StringPattern.replacePatternBreaking("\tString\t")); 46 | assertEquals("_String[___]", StringPattern.replacePatternBreaking("\tString[\n\r\t]")); 47 | assertEquals("_String_", StringPattern.replacePatternBreaking("\tString_")); 48 | assertEquals("_StringString", StringPattern.replacePatternBreaking("\tStringString")); 49 | assertEquals("_String42", StringPattern.replacePatternBreaking("\tString42")); 50 | assertEquals("_42_", StringPattern.replacePatternBreaking("\t42\t")); 51 | assertEquals("_42[___]", StringPattern.replacePatternBreaking("\t42[\n\r\t]")); 52 | assertEquals("_42_", StringPattern.replacePatternBreaking("\t42_")); 53 | assertEquals("_42String", StringPattern.replacePatternBreaking("\t42String")); 54 | assertEquals("_4242", StringPattern.replacePatternBreaking("\t4242")); 55 | assertEquals("[___]__", StringPattern.replacePatternBreaking("[\n\r\t]\t\t")); 56 | assertEquals("[___]_[___]", StringPattern.replacePatternBreaking("[\n\r\t]\t[\n\r\t]")); 57 | assertEquals("[___]__", StringPattern.replacePatternBreaking("[\n\r\t]\t_")); 58 | assertEquals("[___]_String", StringPattern.replacePatternBreaking("[\n\r\t]\tString")); 59 | assertEquals("[___]_42", StringPattern.replacePatternBreaking("[\n\r\t]\t42")); 60 | assertEquals("[___][___]_", StringPattern.replacePatternBreaking("[\n\r\t][\n\r\t]\t")); 61 | assertEquals("[___][___][___]", StringPattern.replacePatternBreaking("[\n\r\t][\n\r\t][\n\r\t]")); 62 | assertEquals("[___][___]_", StringPattern.replacePatternBreaking("[\n\r\t][\n\r\t]_")); 63 | assertEquals("[___][___]String", StringPattern.replacePatternBreaking("[\n\r\t][\n\r\t]String")); 64 | assertEquals("[___][___]42", StringPattern.replacePatternBreaking("[\n\r\t][\n\r\t]42")); 65 | assertEquals("[___]__", StringPattern.replacePatternBreaking("[\n\r\t]_\t")); 66 | assertEquals("[___]_[___]", StringPattern.replacePatternBreaking("[\n\r\t]_[\n\r\t]")); 67 | assertEquals("[___]__", StringPattern.replacePatternBreaking("[\n\r\t]__")); 68 | assertEquals("[___]_String", StringPattern.replacePatternBreaking("[\n\r\t]_String")); 69 | assertEquals("[___]_42", StringPattern.replacePatternBreaking("[\n\r\t]_42")); 70 | assertEquals("[___]String_", StringPattern.replacePatternBreaking("[\n\r\t]String\t")); 71 | assertEquals("[___]String[___]", StringPattern.replacePatternBreaking("[\n\r\t]String[\n\r\t]")); 72 | assertEquals("[___]String_", StringPattern.replacePatternBreaking("[\n\r\t]String_")); 73 | assertEquals("[___]StringString", StringPattern.replacePatternBreaking("[\n\r\t]StringString")); 74 | assertEquals("[___]String42", StringPattern.replacePatternBreaking("[\n\r\t]String42")); 75 | assertEquals("[___]42_", StringPattern.replacePatternBreaking("[\n\r\t]42\t")); 76 | assertEquals("[___]42[___]", StringPattern.replacePatternBreaking("[\n\r\t]42[\n\r\t]")); 77 | assertEquals("[___]42_", StringPattern.replacePatternBreaking("[\n\r\t]42_")); 78 | assertEquals("[___]42String", StringPattern.replacePatternBreaking("[\n\r\t]42String")); 79 | assertEquals("[___]4242", StringPattern.replacePatternBreaking("[\n\r\t]4242")); 80 | assertEquals("___", StringPattern.replacePatternBreaking("_\t\t")); 81 | assertEquals("__[___]", StringPattern.replacePatternBreaking("_\t[\n\r\t]")); 82 | assertEquals("___", StringPattern.replacePatternBreaking("_\t_")); 83 | assertEquals("__String", StringPattern.replacePatternBreaking("_\tString")); 84 | assertEquals("__42", StringPattern.replacePatternBreaking("_\t42")); 85 | assertEquals("_[___]_", StringPattern.replacePatternBreaking("_[\n\r\t]\t")); 86 | assertEquals("_[___][___]", StringPattern.replacePatternBreaking("_[\n\r\t][\n\r\t]")); 87 | assertEquals("_[___]_", StringPattern.replacePatternBreaking("_[\n\r\t]_")); 88 | assertEquals("_[___]String", StringPattern.replacePatternBreaking("_[\n\r\t]String")); 89 | assertEquals("_[___]42", StringPattern.replacePatternBreaking("_[\n\r\t]42")); 90 | assertEquals("___", StringPattern.replacePatternBreaking("__\t")); 91 | assertEquals("__[___]", StringPattern.replacePatternBreaking("__[\n\r\t]")); 92 | assertEquals("_String_", StringPattern.replacePatternBreaking("_String\t")); 93 | assertEquals("_String[___]", StringPattern.replacePatternBreaking("_String[\n\r\t]")); 94 | assertEquals("_42_", StringPattern.replacePatternBreaking("_42\t")); 95 | assertEquals("_42[___]", StringPattern.replacePatternBreaking("_42[\n\r\t]")); 96 | assertEquals("String__", StringPattern.replacePatternBreaking("String\t\t")); 97 | assertEquals("String_[___]", StringPattern.replacePatternBreaking("String\t[\n\r\t]")); 98 | assertEquals("String__", StringPattern.replacePatternBreaking("String\t_")); 99 | assertEquals("String_String", StringPattern.replacePatternBreaking("String\tString")); 100 | assertEquals("String_42", StringPattern.replacePatternBreaking("String\t42")); 101 | assertEquals("String[___]_", StringPattern.replacePatternBreaking("String[\n\r\t]\t")); 102 | assertEquals("String[___][___]", StringPattern.replacePatternBreaking("String[\n\r\t][\n\r\t]")); 103 | assertEquals("String[___]_", StringPattern.replacePatternBreaking("String[\n\r\t]_")); 104 | assertEquals("String[___]String", StringPattern.replacePatternBreaking("String[\n\r\t]String")); 105 | assertEquals("String[___]42", StringPattern.replacePatternBreaking("String[\n\r\t]42")); 106 | assertEquals("String__", StringPattern.replacePatternBreaking("String_\t")); 107 | assertEquals("String_[___]", StringPattern.replacePatternBreaking("String_[\n\r\t]")); 108 | assertEquals("StringString_", StringPattern.replacePatternBreaking("StringString\t")); 109 | assertEquals("StringString[___]", StringPattern.replacePatternBreaking("StringString[\n\r\t]")); 110 | assertEquals("String42_", StringPattern.replacePatternBreaking("String42\t")); 111 | assertEquals("String42[___]", StringPattern.replacePatternBreaking("String42[\n\r\t]")); 112 | assertEquals("42__", StringPattern.replacePatternBreaking("42\t\t")); 113 | assertEquals("42_[___]", StringPattern.replacePatternBreaking("42\t[\n\r\t]")); 114 | assertEquals("42__", StringPattern.replacePatternBreaking("42\t_")); 115 | assertEquals("42_String", StringPattern.replacePatternBreaking("42\tString")); 116 | assertEquals("42_42", StringPattern.replacePatternBreaking("42\t42")); 117 | assertEquals("42[___]_", StringPattern.replacePatternBreaking("42[\n\r\t]\t")); 118 | assertEquals("42[___][___]", StringPattern.replacePatternBreaking("42[\n\r\t][\n\r\t]")); 119 | assertEquals("42[___]_", StringPattern.replacePatternBreaking("42[\n\r\t]_")); 120 | assertEquals("42[___]String", StringPattern.replacePatternBreaking("42[\n\r\t]String")); 121 | assertEquals("42[___]42", StringPattern.replacePatternBreaking("42[\n\r\t]42")); 122 | assertEquals("42__", StringPattern.replacePatternBreaking("42_\t")); 123 | assertEquals("42_[___]", StringPattern.replacePatternBreaking("42_[\n\r\t]")); 124 | assertEquals("42String_", StringPattern.replacePatternBreaking("42String\t")); 125 | assertEquals("42String[___]", StringPattern.replacePatternBreaking("42String[\n\r\t]")); 126 | assertEquals("4242_", StringPattern.replacePatternBreaking("4242\t")); 127 | assertEquals("4242[___]", StringPattern.replacePatternBreaking("4242[\n\r\t]")); 128 | } 129 | } 130 | 131 | --------------------------------------------------------------------------------