├── resources └── certs │ ├── ca-cert.srl │ ├── client.jks │ ├── server.jks │ ├── truststore.jks │ ├── server-file │ ├── client-file │ ├── ca-cert │ ├── server-cert.pem │ ├── server-signed │ ├── client-cert.pem │ ├── client-signed │ ├── truststore.pem │ ├── server-key.pem │ ├── client-key.pem │ ├── ca-key │ ├── server.pem │ └── client.pem ├── logos └── large-logo.png ├── CODE-OF-CONDUCT.md ├── GOVERNANCE.md ├── .gitignore ├── server ├── core │ ├── flags.go │ ├── server_test_test.go │ ├── histogram_test.go │ ├── nats_test.go │ ├── stats_test.go │ ├── monitoring_test.go │ ├── histogram.go │ ├── nats2stan_test.go │ ├── server_config_test.go │ ├── connector.go │ ├── nats2nats.go │ ├── nats2stan.go │ ├── stan2nats.go │ ├── stan2stan.go │ ├── stats.go │ ├── nats2nats_test.go │ ├── nats.go │ ├── monitoring.go │ ├── stan2stan_test.go │ ├── stan2nats_test.go │ └── server_test.go ├── logging │ ├── nats_test.go │ ├── logging.go │ └── nats.go └── conf │ ├── utils.go │ ├── utils_test.go │ ├── load.go │ ├── load_test.go │ ├── conf_test.go │ ├── conf.go │ └── parse.go ├── go.mod ├── MAINTAINERS.md ├── .travis.yml ├── Dockerfile ├── nats-replicator.cfg ├── .goreleaser.yml ├── docs ├── buildandrun.md ├── monitoring.md └── config.md ├── README.md ├── main.go ├── performance └── nats_nats_bench │ └── main.go └── LICENSE /resources/certs/ca-cert.srl: -------------------------------------------------------------------------------- 1 | 97D65EDD0B698E2B 2 | -------------------------------------------------------------------------------- /logos/large-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apis/nats-replicator/master/logos/large-logo.png -------------------------------------------------------------------------------- /resources/certs/client.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apis/nats-replicator/master/resources/certs/client.jks -------------------------------------------------------------------------------- /resources/certs/server.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apis/nats-replicator/master/resources/certs/server.jks -------------------------------------------------------------------------------- /resources/certs/truststore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apis/nats-replicator/master/resources/certs/truststore.jks -------------------------------------------------------------------------------- /CODE-OF-CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Community Code of Conduct 2 | 3 | NATS follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). 4 | -------------------------------------------------------------------------------- /GOVERNANCE.md: -------------------------------------------------------------------------------- 1 | # NATS Server Governance 2 | 3 | This repository is part of the NATS project and is subject to the [NATS Governance](https://github.com/nats-io/nats-general/blob/master/GOVERNANCE.md). 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | .vscode -------------------------------------------------------------------------------- /server/core/flags.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | // Flags defines the various flags you can call the account server with. These are used in main 4 | // and passed down to the server code to process. 5 | type Flags struct { 6 | ConfigFile string 7 | 8 | Debug bool 9 | Verbose bool 10 | DebugAndVerbose bool 11 | } 12 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nats-io/nats-replicator 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/nats-io/nats-server/v2 v2.1.9 7 | github.com/nats-io/nats-streaming-server v0.20.0 8 | github.com/nats-io/nats.go v1.10.0 9 | github.com/nats-io/nuid v1.0.1 10 | github.com/nats-io/stan.go v0.8.2 11 | github.com/stretchr/testify v1.7.0 12 | ) 13 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # Maintainership 2 | 3 | Maintainership is on a per project basis. 4 | 5 | ## Core-maintainers 6 | 7 | - Derek Collison [@derekcollison](https://github.com/derekcollison) 8 | - Stephen Asbury [@sasbury](https://github.com/sasbury) 9 | 10 | ## Maintainers 11 | 12 | - Ivan Kozlovic [@kozlovic](https://github.com/kozlovic) 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.12.x 4 | 5 | git: 6 | depth: false 7 | 8 | env: 9 | - GO111MODULE=off 10 | 11 | install: 12 | - go get github.com/mattn/goveralls 13 | 14 | before_script: 15 | - EXCLUDE_VENDOR=$(go list ./... | grep -v "/vendor/") 16 | - $(exit $(go fmt $EXCLUDE_VENDOR | wc -l)) 17 | - go vet $EXCLUDE_VENDOR 18 | 19 | script: 20 | - go test -race -coverpkg=./... -coverprofile=./coverage.out ./... 21 | 22 | after_success: 23 | - git reset --hard 24 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.12.4 AS builder 2 | 3 | WORKDIR /src/nats-replicator 4 | 5 | LABEL maintainer "Stephen Asbury " 6 | 7 | COPY . . 8 | 9 | RUN go mod download 10 | RUN CGO_ENABLED=0 go build -v -a -tags netgo -installsuffix netgo -o /nats-replicator 11 | 12 | FROM alpine:3.9 13 | 14 | RUN mkdir -p /nats/bin && mkdir /nats/conf 15 | 16 | COPY --from=builder /nats-replicator /nats/bin/nats-replicator 17 | 18 | RUN ln -ns /nats/bin/nats-replicator /bin/nats-replicator 19 | 20 | ENTRYPOINT ["/bin/nats-replicator"] 21 | -------------------------------------------------------------------------------- /nats-replicator.cfg: -------------------------------------------------------------------------------- 1 | reconnectinterval: 5000, 2 | 3 | logging: { 4 | Time: true, 5 | Debug: true, 6 | Trace: true, 7 | Colors: true, 8 | PID: false, 9 | }, 10 | 11 | monitoring: { 12 | httpport: 19090, 13 | } 14 | 15 | nats: [ 16 | { 17 | Name: "nats-1", 18 | Servers: ["localhost:14222"], 19 | ConnectTimeout: 5000, 20 | MaxReconnects: 5, 21 | ReconnectWait: 5000, 22 | }, 23 | { 24 | Name: "nats-2", 25 | Servers: ["localhost:24222"], 26 | ConnectTimeout: 5000, 27 | MaxReconnects: 5, 28 | ReconnectWait: 5000, 29 | }, 30 | ], 31 | 32 | connect: [ 33 | { 34 | type: NATSToNATS, 35 | id: "hub-to-device", 36 | IncomingSubject: "simplicity.command.>", 37 | IncomingConnection: "nats-2", 38 | OutgoingSubject: "simplicity.command", 39 | OutgoingConnection: "nats-1", 40 | }, 41 | { 42 | type: NATSToNATS, 43 | id: "device-to-hub", 44 | IncomingSubject: "simplicity.response.>", 45 | IncomingConnection: "nats-1", 46 | OutgoingSubject: "simplicity.response", 47 | OutgoingConnection: "nats-2", 48 | }, 49 | ], 50 | -------------------------------------------------------------------------------- /server/logging/nats_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package logging 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func TestNATSForCoverage(t *testing.T) { 26 | logger := NewNATSLogger(Config{}) 27 | logger.Debugf("test") 28 | logger.Tracef("test") 29 | logger.Noticef("test") 30 | logger.Errorf("test") 31 | logger.Warnf("test") 32 | 33 | require.False(t, logger.TraceEnabled()) 34 | // skip fatal 35 | logger.Close() 36 | } 37 | -------------------------------------------------------------------------------- /resources/certs/server-file: -------------------------------------------------------------------------------- 1 | -----BEGIN NEW CERTIFICATE REQUEST----- 2 | MIIC3jCCAcYCAQAwaTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWEx 3 | EzARBgNVBAcTCkxvc0FuZ2VsZXMxDTALBgNVBAoTBE5vbmUxDTALBgNVBAsTBE5v 4 | bmUxEjAQBgNVBAMTCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC 5 | AQoCggEBAKC5cmzu67GvoxFpQqB4Zs0MFH35xdW3QuAs0u93ueDI8P36r4o0pou9 6 | XDvST5Hk1RTHmdKMzrKrJCaskJu6BwLc+do5RrjgFzqzNR5CuzUQSj+MOfHgixFY 7 | zvZcp6LW06X6oYapphwW3ewRRR2t0hOPAdqoTxAZ85Kh2bQjFha7gYXAVxrgQfyc 8 | DQA7mSFz+UeDhkFcF1epLqdpNrV0Fkv0E9gChQhJnZcGbO0vqkdWBjbRlGsnzt21 9 | 8+JFK9oBMh/iH3naSKTBbxGqYTpy9zq3zUJke8IdgOnliN00MrQiTcqI/tv76zYI 10 | +aIcs9gQ3vxNvmzR/K2nEeAW49v84gcCAwEAAaAwMC4GCSqGSIb3DQEJDjEhMB8w 11 | HQYDVR0OBBYEFA/O/DlRDLrA6quCD2TTnYNWJmRhMA0GCSqGSIb3DQEBCwUAA4IB 12 | AQAQLBhFXg6/2lTPIqMbXHIoGSWT+tJpzpvs8E7TEsMGa3+/LEO7KtX5484ofbHU 13 | jAaCLgycVLK1y0xgN+cNwucXsvwAZ4TdfNlc8ZFHXIJGtA80UI9MOY9PfvWouD7s 14 | LT6+rKeM55htsQfRipzRCvObiq9HigrYH41DjPybf8pSIykI1ZaPlNiUR62rKlXb 15 | /ZpPk9E9gTBPnEuzTMNrjQnMluOmsXGD/n3J1HTXdzgOJRCH3DeAUEn1GcvokMk6 16 | 8w/hNp92Srgegou2N/puPblr6RWk8hYczCETyO5Gu840dU5iDQMnLRmBbZitDhs1 17 | 3/rWyQD+wn/p6fpcfKvmnzs+ 18 | -----END NEW CERTIFICATE REQUEST----- 19 | -------------------------------------------------------------------------------- /resources/certs/client-file: -------------------------------------------------------------------------------- 1 | -----BEGIN NEW CERTIFICATE REQUEST----- 2 | MIIC6TCCAdECAQAwdDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWEx 3 | EzARBgNVBAcTCkxvc0FuZ2VsZXMxDTALBgNVBAoTBE5vbmUxDTALBgNVBAsTBE5v 4 | bmUxHTAbBgNVBAMTFG5hdC5rYWZrYS5jbGllbnQuc3NsMIIBIjANBgkqhkiG9w0B 5 | AQEFAAOCAQ8AMIIBCgKCAQEAgafEGfn5YmpqsYk4vUE3sDBZsCmWrlVjLgILtv8y 6 | 5WF7opEp+qlmwo/o3qtUxmFf1LSTbd4fEzoLIPrm8PZpZLDQo8HzSWRTGDIPGJEf 7 | VNxhN6OhaWkWYCwaweAvapnHiYQFksGcf+EaUjtCGRrprngV8BuFQDF/KhWv2fuK 8 | xRDBDqNhRF8dDg2A4/kxelYpmmBjHNuIwWMGHm15ezf4AlKGLmrGHhCcWi8m7CZo 9 | 5rwhSk7XkfelNmm+3U4pkCRn569FHAYwE72ulXTIEL8nC3zw9qrh+OgxHLYZ0DoO 10 | Cg51PvL7J6O6oJAvhs3GNgyfj5Cn0RoPEq1Wb438TkN80QIDAQABoDAwLgYJKoZI 11 | hvcNAQkOMSEwHzAdBgNVHQ4EFgQUaPH9qmK7WbGtHQbrA9CBZICJ/SAwDQYJKoZI 12 | hvcNAQELBQADggEBAART7S9NMiOBUf4hNHBgMJw/DyaW//MJYfFZbZYrhPGtZ3YS 13 | rrFRO5RRdkRqc/fkT7u7qI+TwOiD3OHI/7rM0jt7DJ9z29XZlVMmlvXjgYxYHlDI 14 | XWIPjpK0QlsCDJRtZZzvl7n0cdOjnDOnC8diZA1g/vwMAcyQ5hf7m6OLOQQl/Hlz 15 | prZ5L6YvZilrxlS3pvaUKZgstOQ02vrMy3P+mcgZNJxVXMTx2yYEWlsDI9bqopLY 16 | sHcOiZ/kUtpQJaQM1tB1x3shPZa8Zfx/K3uXGMvWoaJmsbZ1PCGG1C0HQcnodLos 17 | qp0LxvJCsOUKaqevElZw0x7U4yU359Q0+7/5PoI= 18 | -----END NEW CERTIFICATE REQUEST----- 19 | -------------------------------------------------------------------------------- /server/core/server_test_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/nats-io/nats-replicator/server/conf" 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func TestFullTestEnvironment(t *testing.T) { 26 | connect := []conf.ConnectorConfig{} 27 | tbs, err := StartTestEnvironment(connect) 28 | require.NoError(t, err) 29 | tbs.Close() 30 | } 31 | 32 | func TestFullTLSTestEnvironment(t *testing.T) { 33 | connect := []conf.ConnectorConfig{} 34 | tbs, err := StartTLSTestEnvironment(connect) 35 | require.NoError(t, err) 36 | tbs.Close() 37 | } 38 | -------------------------------------------------------------------------------- /resources/certs/ca-cert: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDRjCCAi4CCQCZ2pXVwpC9gzANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV 3 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 4 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 5 | NDMwMDA1MzI4WhcNMjEwNDI5MDA1MzI4WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE 6 | CAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5vbmUxDTALBgNV 7 | BAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwggEiMA0GCSqGSIb3DQEB 8 | AQUAA4IBDwAwggEKAoIBAQCvFAzhfXlw7eOmPTsCD1KISIOTBc9A7QvSspm3I2ka 9 | YKbsJgPJGuDVDosj0j74m77r56vXmNkCD5NNM3HuNxE5nAtaq848P+hbpxBgDp3n 10 | ZLO0BW+CSkfM1V6Qg4wp8jTAa6KMsK21ih1tKzS1VGnUJ7UcJhyU57joMmsiYl3E 11 | XHBnx0WKKgGQKdrYPv600Qdy77WoFWzawHqrroZnpH8Eb/YXfQEw7Hc+kedl0b5Y 12 | Rag6CEaB0B9nii321juhaAoMTpp8/hSca4zO3a9Jwz1KLlCFXjlHdyw4kv4uNGQk 13 | vIGwYFPJLbGi5SAaJwj9yS8v1e8TzK7YtJRsPUJYB+0nAgMBAAEwDQYJKoZIhvcN 14 | AQELBQADggEBADXRM+LVI+sXR64+8kKpDoFkR98RgIaXgMVN52M9O2fwU1tO5EDH 15 | MnqqRc5zkTjmLM6RDhGQnwTl+d+EcrFqteFU7qgj0EHBUE07rVgtinrJD/0INFzZ 16 | 6vMe+ZJVOJ6V6SjJBXQM0q2ASENH1zZCYfQgZPCBYdRhq3YzvgkCyt6OxIWE8APW 17 | Jlmmw4o/N5hbGwrDfTA3CyJU3Xy1zAsNSjRSQH1sHUHvVALZz9fHPamgpyFRiTmG 18 | 7qypC5FLAMo0mI8569I/ns1qIqk68aHJtynrxRIoPWc2eFK0J8v0oJ2LmaoK3NSa 19 | bByzkeofOaoaR2EzAKJoc3mkkX1+eDKmU34= 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /resources/certs/server-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDSjCCAjICCQCX1l7dC2mOKjANBgkqhkiG9w0BAQUFADBlMQswCQYDVQQGEwJV 3 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 4 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 5 | NDMwMDA1MzMwWhcNMjEwNDI5MDA1MzMwWjBpMQswCQYDVQQGEwJVUzETMBEGA1UE 6 | CBMKQ2FsaWZvcm5pYTETMBEGA1UEBxMKTG9zQW5nZWxlczENMAsGA1UEChMETm9u 7 | ZTENMAsGA1UECxMETm9uZTESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG 8 | 9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoLlybO7rsa+jEWlCoHhmzQwUffnF1bdC4CzS 9 | 73e54Mjw/fqvijSmi71cO9JPkeTVFMeZ0ozOsqskJqyQm7oHAtz52jlGuOAXOrM1 10 | HkK7NRBKP4w58eCLEVjO9lynotbTpfqhhqmmHBbd7BFFHa3SE48B2qhPEBnzkqHZ 11 | tCMWFruBhcBXGuBB/JwNADuZIXP5R4OGQVwXV6kup2k2tXQWS/QT2AKFCEmdlwZs 12 | 7S+qR1YGNtGUayfO3bXz4kUr2gEyH+IfedpIpMFvEaphOnL3OrfNQmR7wh2A6eWI 13 | 3TQytCJNyoj+2/vrNgj5ohyz2BDe/E2+bNH8racR4Bbj2/ziBwIDAQABMA0GCSqG 14 | SIb3DQEBBQUAA4IBAQBQN5Yt2iEIxjKBP+Xs7iA+tAV6YrCF4xlE1j9ufo+FaelN 15 | FzlNda7ZFDzPsiipGIU8wi3fAEOWVBgDKpVKihY7AXaEfjgxCg1NwSiDyzyljGCR 16 | /I9eE1wqWv4fMu5AmpaaFu763kL4L9SeMvu7J/tHYt2vxFF5f+ZSpu+ulp85Q+aD 17 | dyK4WpTT0dQKMNFGt000+C7t3GuinAt0yN9U/GpbBXSQAzd9+cA78PgmXTdalGvs 18 | fgHc3rHvVLInC+z/DGpnaeH2NDz+zBRTCB0PRHDghWbG8iXkzJcEXo6ZTMQ6wwVU 19 | PlgVgE7ZiCXXhD7kowoJQ5HZTuA4SR0pr5mSi3Au 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /resources/certs/server-signed: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDSjCCAjICCQCX1l7dC2mOKjANBgkqhkiG9w0BAQUFADBlMQswCQYDVQQGEwJV 3 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 4 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 5 | NDMwMDA1MzMwWhcNMjEwNDI5MDA1MzMwWjBpMQswCQYDVQQGEwJVUzETMBEGA1UE 6 | CBMKQ2FsaWZvcm5pYTETMBEGA1UEBxMKTG9zQW5nZWxlczENMAsGA1UEChMETm9u 7 | ZTENMAsGA1UECxMETm9uZTESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG 8 | 9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoLlybO7rsa+jEWlCoHhmzQwUffnF1bdC4CzS 9 | 73e54Mjw/fqvijSmi71cO9JPkeTVFMeZ0ozOsqskJqyQm7oHAtz52jlGuOAXOrM1 10 | HkK7NRBKP4w58eCLEVjO9lynotbTpfqhhqmmHBbd7BFFHa3SE48B2qhPEBnzkqHZ 11 | tCMWFruBhcBXGuBB/JwNADuZIXP5R4OGQVwXV6kup2k2tXQWS/QT2AKFCEmdlwZs 12 | 7S+qR1YGNtGUayfO3bXz4kUr2gEyH+IfedpIpMFvEaphOnL3OrfNQmR7wh2A6eWI 13 | 3TQytCJNyoj+2/vrNgj5ohyz2BDe/E2+bNH8racR4Bbj2/ziBwIDAQABMA0GCSqG 14 | SIb3DQEBBQUAA4IBAQBQN5Yt2iEIxjKBP+Xs7iA+tAV6YrCF4xlE1j9ufo+FaelN 15 | FzlNda7ZFDzPsiipGIU8wi3fAEOWVBgDKpVKihY7AXaEfjgxCg1NwSiDyzyljGCR 16 | /I9eE1wqWv4fMu5AmpaaFu763kL4L9SeMvu7J/tHYt2vxFF5f+ZSpu+ulp85Q+aD 17 | dyK4WpTT0dQKMNFGt000+C7t3GuinAt0yN9U/GpbBXSQAzd9+cA78PgmXTdalGvs 18 | fgHc3rHvVLInC+z/DGpnaeH2NDz+zBRTCB0PRHDghWbG8iXkzJcEXo6ZTMQ6wwVU 19 | PlgVgE7ZiCXXhD7kowoJQ5HZTuA4SR0pr5mSi3Au 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /server/logging/logging.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package logging 18 | 19 | // Config defines logging flags for the NATS logger 20 | type Config struct { 21 | Hide bool 22 | Time bool 23 | Debug bool 24 | Trace bool 25 | Colors bool 26 | PID bool 27 | } 28 | 29 | // Logger interface 30 | type Logger interface { 31 | Debugf(format string, v ...interface{}) 32 | Errorf(format string, v ...interface{}) 33 | Fatalf(format string, v ...interface{}) 34 | Noticef(format string, v ...interface{}) 35 | Tracef(format string, v ...interface{}) 36 | Warnf(format string, v ...interface{}) 37 | 38 | TraceEnabled() bool 39 | 40 | Close() error 41 | } 42 | -------------------------------------------------------------------------------- /resources/certs/client-cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDVTCCAj0CCQCX1l7dC2mOKzANBgkqhkiG9w0BAQUFADBlMQswCQYDVQQGEwJV 3 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 4 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 5 | NDMwMDA1MzMzWhcNMjEwNDI5MDA1MzMzWjB0MQswCQYDVQQGEwJVUzETMBEGA1UE 6 | CBMKQ2FsaWZvcm5pYTETMBEGA1UEBxMKTG9zQW5nZWxlczENMAsGA1UEChMETm9u 7 | ZTENMAsGA1UECxMETm9uZTEdMBsGA1UEAxMUbmF0LmthZmthLmNsaWVudC5zc2ww 8 | ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCBp8QZ+fliamqxiTi9QTew 9 | MFmwKZauVWMuAgu2/zLlYXuikSn6qWbCj+jeq1TGYV/UtJNt3h8TOgsg+ubw9mlk 10 | sNCjwfNJZFMYMg8YkR9U3GE3o6FpaRZgLBrB4C9qmceJhAWSwZx/4RpSO0IZGumu 11 | eBXwG4VAMX8qFa/Z+4rFEMEOo2FEXx0ODYDj+TF6VimaYGMc24jBYwYebXl7N/gC 12 | UoYuasYeEJxaLybsJmjmvCFKTteR96U2ab7dTimQJGfnr0UcBjATva6VdMgQvycL 13 | fPD2quH46DEcthnQOg4KDnU+8vsno7qgkC+GzcY2DJ+PkKfRGg8SrVZvjfxOQ3zR 14 | AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAD/v1U5zKnXh3e0MwYcsMGCNUUoGdAVa 15 | OLJuER9yq2I7CpYEVUlh1kGRkPs8u3Ww1NrYmASB9feaAPJ+U72zttkdKlXht6W+ 16 | b5rbQBCNPVd7yYfhl4kelnvSP1jhCT82KCHWo/6g4ku42d1oKy+hPyBErWfuC2MV 17 | tbls0br3LLlRa8316mlCL4ek4AGXvb60cB5g5B/PrgBsGBqtqTf1Yc7i+DhUOSZU 18 | Wlm5Yk3Z+AaztxfYhHAlubhzHFppA9H8kKjnbN0kTzPzaZhnrrF7Ro25ViLvc0cW 19 | rPnMY0+vNSAUd0zRMymuA0PGRSeEqVFbni6EGKuHXXM+wPh38JThACY= 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /resources/certs/client-signed: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDVTCCAj0CCQCX1l7dC2mOKzANBgkqhkiG9w0BAQUFADBlMQswCQYDVQQGEwJV 3 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 4 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 5 | NDMwMDA1MzMzWhcNMjEwNDI5MDA1MzMzWjB0MQswCQYDVQQGEwJVUzETMBEGA1UE 6 | CBMKQ2FsaWZvcm5pYTETMBEGA1UEBxMKTG9zQW5nZWxlczENMAsGA1UEChMETm9u 7 | ZTENMAsGA1UECxMETm9uZTEdMBsGA1UEAxMUbmF0LmthZmthLmNsaWVudC5zc2ww 8 | ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCBp8QZ+fliamqxiTi9QTew 9 | MFmwKZauVWMuAgu2/zLlYXuikSn6qWbCj+jeq1TGYV/UtJNt3h8TOgsg+ubw9mlk 10 | sNCjwfNJZFMYMg8YkR9U3GE3o6FpaRZgLBrB4C9qmceJhAWSwZx/4RpSO0IZGumu 11 | eBXwG4VAMX8qFa/Z+4rFEMEOo2FEXx0ODYDj+TF6VimaYGMc24jBYwYebXl7N/gC 12 | UoYuasYeEJxaLybsJmjmvCFKTteR96U2ab7dTimQJGfnr0UcBjATva6VdMgQvycL 13 | fPD2quH46DEcthnQOg4KDnU+8vsno7qgkC+GzcY2DJ+PkKfRGg8SrVZvjfxOQ3zR 14 | AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAD/v1U5zKnXh3e0MwYcsMGCNUUoGdAVa 15 | OLJuER9yq2I7CpYEVUlh1kGRkPs8u3Ww1NrYmASB9feaAPJ+U72zttkdKlXht6W+ 16 | b5rbQBCNPVd7yYfhl4kelnvSP1jhCT82KCHWo/6g4ku42d1oKy+hPyBErWfuC2MV 17 | tbls0br3LLlRa8316mlCL4ek4AGXvb60cB5g5B/PrgBsGBqtqTf1Yc7i+DhUOSZU 18 | Wlm5Yk3Z+AaztxfYhHAlubhzHFppA9H8kKjnbN0kTzPzaZhnrrF7Ro25ViLvc0cW 19 | rPnMY0+vNSAUd0zRMymuA0PGRSeEqVFbni6EGKuHXXM+wPh38JThACY= 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | --- 2 | project_name: nats-replicator 3 | 4 | release: 5 | draft: true 6 | github: 7 | owner: nats-io 8 | name: nats-replicator 9 | 10 | name_template: 'Release {{.Tag}}' 11 | 12 | builds: 13 | - goos: 14 | - linux 15 | - darwin 16 | - windows 17 | goarch: 18 | - 386 19 | - amd64 20 | - arm 21 | - arm64 22 | goarm: 23 | - 6 24 | - 7 25 | ignore: 26 | - goos: darwin 27 | goarch: 386 28 | 29 | main: . 30 | binary: nats-replicator 31 | 32 | lang: go 33 | 34 | archive: 35 | wrap_in_directory: true 36 | name_template: '{{ .ProjectName }}-v{{ .Version }}-{{ .Os }}-{{ .Arch }}{{ if .Arm 37 | }}v{{ .Arm }}{{ end }}' 38 | format: zip 39 | files: 40 | - LICENSE 41 | - README.md 42 | 43 | nfpm: 44 | formats: 45 | - deb 46 | name_template: '{{ .ProjectName }}-v{{ .Version }}-{{ .Os }}-{{ .Arch }}{{ if .Arm 47 | }}v{{ .Arm }}{{ end }}' 48 | bindir: /usr/local/bin 49 | description: NATS Replicator 50 | vendor: nats.io 51 | maintainer: Stephen Asbury 52 | homepage: https://nats.io 53 | license: Apache 2.0 54 | 55 | snapshot: 56 | name_template: SNAPSHOT-{{ .Commit }} 57 | 58 | checksum: 59 | name_template: '{{ .ProjectName }}-v{{ .Version }}-checksums.txt' 60 | 61 | dist: dist 62 | 63 | github_urls: 64 | download: https://github.com 65 | -------------------------------------------------------------------------------- /resources/certs/truststore.pem: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: caroot 3 | 2.16.840.1.113894.746875.1.1: 4 | subject=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 5 | issuer=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 6 | -----BEGIN CERTIFICATE----- 7 | MIIDRjCCAi4CCQCZ2pXVwpC9gzANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV 8 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 9 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 10 | NDMwMDA1MzI4WhcNMjEwNDI5MDA1MzI4WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE 11 | CAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5vbmUxDTALBgNV 12 | BAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwggEiMA0GCSqGSIb3DQEB 13 | AQUAA4IBDwAwggEKAoIBAQCvFAzhfXlw7eOmPTsCD1KISIOTBc9A7QvSspm3I2ka 14 | YKbsJgPJGuDVDosj0j74m77r56vXmNkCD5NNM3HuNxE5nAtaq848P+hbpxBgDp3n 15 | ZLO0BW+CSkfM1V6Qg4wp8jTAa6KMsK21ih1tKzS1VGnUJ7UcJhyU57joMmsiYl3E 16 | XHBnx0WKKgGQKdrYPv600Qdy77WoFWzawHqrroZnpH8Eb/YXfQEw7Hc+kedl0b5Y 17 | Rag6CEaB0B9nii321juhaAoMTpp8/hSca4zO3a9Jwz1KLlCFXjlHdyw4kv4uNGQk 18 | vIGwYFPJLbGi5SAaJwj9yS8v1e8TzK7YtJRsPUJYB+0nAgMBAAEwDQYJKoZIhvcN 19 | AQELBQADggEBADXRM+LVI+sXR64+8kKpDoFkR98RgIaXgMVN52M9O2fwU1tO5EDH 20 | MnqqRc5zkTjmLM6RDhGQnwTl+d+EcrFqteFU7qgj0EHBUE07rVgtinrJD/0INFzZ 21 | 6vMe+ZJVOJ6V6SjJBXQM0q2ASENH1zZCYfQgZPCBYdRhq3YzvgkCyt6OxIWE8APW 22 | Jlmmw4o/N5hbGwrDfTA3CyJU3Xy1zAsNSjRSQH1sHUHvVALZz9fHPamgpyFRiTmG 23 | 7qypC5FLAMo0mI8569I/ns1qIqk68aHJtynrxRIoPWc2eFK0J8v0oJ2LmaoK3NSa 24 | bByzkeofOaoaR2EzAKJoc3mkkX1+eDKmU34= 25 | -----END CERTIFICATE----- 26 | -------------------------------------------------------------------------------- /docs/buildandrun.md: -------------------------------------------------------------------------------- 1 | # Build and Run the NATS-Replicator 2 | 3 | ## Running the server 4 | 5 | The server will compile to an executable named `nats-replicator`. A [configuration](config.md) file is required to get any real behavior out of the server. 6 | 7 | To specify the [configuration](config.md) file, use the `-c` flag: 8 | 9 | ```bash 10 | % nats-replicator -c 11 | ``` 12 | 13 | You can use the `-D`, `-V` or `-DV` flags to turn on debug or verbose logging. The `-DV` option will turn on all logging, depending on the config file settings, these settings will override the ones in the config file. 14 | 15 | 16 | 17 | ## Building the Server 18 | 19 | This project uses go modules and provides a make file. You should be able to simply: 20 | 21 | ```bash 22 | % git clone https://github.com/nats-io/nats-replicator.git 23 | % cd nats-replicator 24 | % make 25 | ``` 26 | 27 | Use `make test` to run the tests, and `make install` to install. The nats and nats streaming servers are imported as go modules. 28 | 29 | ## Docker 30 | 31 | You can build the docker image using: 32 | 33 | ```bash 34 | % docker build . -t "nats-io/nats-replicator:0.5" 35 | ``` 36 | 37 | Then run it with: 38 | 39 | ```bash 40 | % docker run -v :/conf/replicator.conf "nats-io/nats-replicator:0.5" -c /conf/replicator.conf 41 | ``` 42 | 43 | Be sure to include your monitoring port, for example, if port 9090 is used for monitoring, you can run with: 44 | 45 | ```bash 46 | % docker run -v :/conf/replicator.conf -p 9090:9090 "nats-io/nats-replicator:0.5" -c /conf/replicator.conf 47 | ``` 48 | -------------------------------------------------------------------------------- /server/core/histogram_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | // Based on https://github.com/VividCortex/gohistogram MIT license 19 | 20 | import ( 21 | "math" 22 | "testing" 23 | 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | func approx(x, y float64) bool { 28 | return math.Abs(x-y) < 0.2 29 | } 30 | 31 | func TestHistogram(t *testing.T) { 32 | h := NewHistogram(160) 33 | for i := 0; i < 15000; i++ { 34 | h.Add(float64(i % 100)) 35 | } 36 | 37 | firstQ := h.Quantile(0.25) 38 | median := h.Quantile(0.5) 39 | thirdQ := h.Quantile(0.75) 40 | 41 | require.Equal(t, float64(15000), h.Count()) 42 | require.True(t, approx(firstQ, 24)) 43 | require.True(t, approx(median, 49)) 44 | require.True(t, approx(thirdQ, 74)) 45 | 46 | require.True(t, approx(h.Mean(), 49.5)) 47 | 48 | h.Scale(0.5) 49 | median = h.Quantile(0.5) 50 | require.True(t, approx(median, 24.5)) 51 | } 52 | 53 | func TestHistogramTrim(t *testing.T) { 54 | h := NewHistogram(10) 55 | for i := 0; i < 150; i++ { 56 | h.Add(float64(i % 100)) 57 | } 58 | require.Equal(t, 10, len(h.Bins)) 59 | } 60 | -------------------------------------------------------------------------------- /resources/certs/server-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCguXJs7uuxr6MR 3 | aUKgeGbNDBR9+cXVt0LgLNLvd7ngyPD9+q+KNKaLvVw70k+R5NUUx5nSjM6yqyQm 4 | rJCbugcC3PnaOUa44Bc6szUeQrs1EEo/jDnx4IsRWM72XKei1tOl+qGGqaYcFt3s 5 | EUUdrdITjwHaqE8QGfOSodm0IxYWu4GFwFca4EH8nA0AO5khc/lHg4ZBXBdXqS6n 6 | aTa1dBZL9BPYAoUISZ2XBmztL6pHVgY20ZRrJ87dtfPiRSvaATIf4h952kikwW8R 7 | qmE6cvc6t81CZHvCHYDp5YjdNDK0Ik3KiP7b++s2CPmiHLPYEN78Tb5s0fytpxHg 8 | FuPb/OIHAgMBAAECggEANelen3mY57YxbaKwLaGVYgw+R29j0+cv8IA4lQjE+ciN 9 | RbQz85jrkI4JBSvrctWeP+UgDMccgkPz0WEq9IF23pf+2xtBRLwuLot0Mt1RbGA1 10 | d5Fy0lZub3gpm+vCc1W6Er+6NLBtPpjRAeFdHLJ0eAkNJyozswPxdusBWnI2c5MT 11 | g6iW3Cnl5T4VW/yYSM5ktJSfCBqtlPdrCjWzaB3g+VKXiI0hhtL5DUr/Jjx5GUKb 12 | h3UN3Us6QUHNT1oEQMNaeYQdHQxEtz7VPAeUmLqDlM4/Bo1Bd81Olb9uy3xIt1u+ 13 | 0Ns8IOtbtP81Qn+VRpLHNI/AKoVL/K28IM/I1IPe6QKBgQDVQ9F7T//1MoSDHbji 14 | FIeEBJc0yghDq3VmYSIir5jHXnh5Ay263nnVIvg8Y/f5/Zlgr6bJeR3EvWuvfI4A 15 | H6LrW5MAeB3InNygNKeJFM1lOdvo0sx4mETIZpEBGmYivFWZNhUrS8EhYfLrKBdp 16 | hE5d31yn0K+t8GuZpK9MIbRMowKBgQDA7mBZJWMBvrzrRi8NNbibYIKbLD3VGPhP 17 | M7fMUxLhATcS0tIKXV8am7gOKvbg1zVLhS4/aBxIDO9SaqNADT3CgxCc9g00iiQJ 18 | avVl9e82ROSmz55BCNudrnPogvbRnHiopeWIK2X/PdmvGsNwPh9AJrlwNMaqD/nQ 19 | uv4FHucnTQKBgE4tRiVF9jYUBq9pvdRiYirq4+LeDJmByM922+SLKh4ra04w9BqE 20 | Y3TWFNlObCCf7hGbUjCYzWjJZyg1KHizIC3Wq9SIM6LOnbG8m42Mqp5oz0xRudKb 21 | PtuXfaBB5R7mmxpG0QvGAU4TcdDyKWLstS2EK5r4zO2eLFNuIzRtRoKxAoGAd758 22 | BlhyDs83qj8xTN2e6rEH3w+igPSyiVXsKeEVwplieUhoHQ6+zGEB56k6+WoZJfpP 23 | LgOMKhv9HgYZtNODFYsLcKA2qfljTIUaMmJmSiSQVghejLbWuBNi1VkToB2hterh 24 | f5aQA897oHbX/n5QHxzp036uHzczMh4dM0hu57ECgYEAktJYUJFW+nxRbDguauoa 25 | dIh9NbKFqpcBhPtcKsEKBtcuNxuFmlMZHEBwCE66mqpnjDVByU3qi6nu2L1ZoVQ4 26 | QKWxjzkgnWJAhnSQO5QQLHeUYqBWJ2XMGLyoupgDFaahQxS2/+6Zu6/6XsorOAKj 27 | iNobI1GsQun0U1CqgHnzwCw= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /resources/certs/client-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCBp8QZ+fliamqx 3 | iTi9QTewMFmwKZauVWMuAgu2/zLlYXuikSn6qWbCj+jeq1TGYV/UtJNt3h8TOgsg 4 | +ubw9mlksNCjwfNJZFMYMg8YkR9U3GE3o6FpaRZgLBrB4C9qmceJhAWSwZx/4RpS 5 | O0IZGumueBXwG4VAMX8qFa/Z+4rFEMEOo2FEXx0ODYDj+TF6VimaYGMc24jBYwYe 6 | bXl7N/gCUoYuasYeEJxaLybsJmjmvCFKTteR96U2ab7dTimQJGfnr0UcBjATva6V 7 | dMgQvycLfPD2quH46DEcthnQOg4KDnU+8vsno7qgkC+GzcY2DJ+PkKfRGg8SrVZv 8 | jfxOQ3zRAgMBAAECggEAGTrdRTTIm6cbpfqO6P0U6hsuiMI/CSOijiRlxTF55PsI 9 | RQU8hwOJ9Jhud04dzBDMxiZxffK4V1R3L+7DG+bHcYmawBMQ1ZpJNS8gkuZCk4/i 10 | 9yHHWizR5tsOReXrNDJCivb+4qT8YEIcjh3r6di72nCRzEx+rJQ1K6pWsvNo+JnV 11 | gZpQMSpIid0WpQsII2Gq+0ScPrw7RVWweoduqYgjfr1rM9taZZSzyOcQww34e1P0 12 | WDfjcIZkOdSGXBuxe0mRnuK4q6uE8dVMDlig5FaS6noNSePVerRCAn9o3K0NeKpa 13 | Z7A1l+n4kX9vKzR91Dwp25pufhHEOFFw72xOaOUEcQKBgQDu4PFrVTYfeY7OVEDA 14 | PbuO10Ag4jyal/QuemipZzLRplCk7FH8R55YEZtVMMcDHAP2VVeOoOBte77h1FdJ 15 | 3mdctZ6Ai7Q7piVP7HR7cuClxPgvrA9VgxPsPDTVlbklfU/ZgXxZdw49K0Dn3Jl3 16 | Bi9cjfvwtTIsios6YLIAO/2ShQKBgQCK8r4Hcu85b9kJ//WF4mBiLPudJ+wxo2bR 17 | DVjq9SaXk9qoNAOV42RUB3s8V0peo79hfmLJWFAm1706H8l+0aeWN4nr66Gx/0Pm 18 | 3g/ztf0IMJlGa+MqDFO5B0JEm7PAjG1J0YZX+gW2Zd954F65cU4DjtBP64sEoqMo 19 | i+8Jt6wA3QKBgQC1vKVvh1C5+X1QNIFewJilP9YmLnj1FI5Nngtqdn0PS30nPDPT 20 | v5kUX7DRy15dWz8Ydi068eJM6Ux7+1S9elshCXwhSChDCVBx0e98zBVliMlZW4n/ 21 | AM5zeAqqRFKr1v6c+Apm9lD68KFcxVRsXWRDAjKfGvulL3JlY5OI4hs1EQKBgQCH 22 | 9onufhAKkyC9AUK+NMr9pmi72nHsDKmyTK5Ck4qk7iAbUXJkvDLTatKzM/No8jB2 23 | dRazUQB1UcwvUSV5PCwR+Ny0B0mdBFzsT0UqxF0KI4wIdc++uHtAZhL5Uaat9nuG 24 | rUkZU2U9myf3eY8XRQECyD+cxK6u5XpkVbGdP6ZG3QKBgQCYFaGrWc7uPOj0iGac 25 | qU3dh03WfCScv7jo5uiQCAIP2Jwkc3dtIiMR7s+RFZzdMoAsCGpqP5EwEWbgC5UB 26 | wB6zSWIaYwp/v21jqvsZ5+Q+9ZV3iY8f56MpzShgTOGkZM1J7I9rsQACapC8t/aX 27 | XDttH8MprVIwasd6gqTnngXUXQ== 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /resources/certs/ca-key: -------------------------------------------------------------------------------- 1 | -----BEGIN ENCRYPTED PRIVATE KEY----- 2 | MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIyM7x4XHLFoACAggA 3 | MB0GCWCGSAFlAwQBKgQQy/pCehLas29GlAQ2ae25YwSCBNAHvfwcv8mOSgRvfzmf 4 | k0lPFPbom+VWMNU5qZQSBJIlEFiwGVSZ6lUwNwHItfdDZ6CJBM7BV3MCOjKd85lb 5 | QPRuCADoRgh6nLzoctMU0SYX7rNHcTiiYvpg8CDQrfBtK8g4BZ0Oojg7AZJI4I8f 6 | oX1xrvSzWs6ZHLm6X0FoFNlwvbGUOchvCx1LvULur90DjnPyofdDGifqgMkUx5hL 7 | beMVmmMape5Lekv/HDjCzfAImqIbnXtJ4iQp6UHwFmncLMORpxKf6j7GZ6ss2uwr 8 | i+1aurk/XeNKwQ/4dXsTPhgNMkaJQRAwKPq4ebTD1vvqdylaat90BIuLIeN8UkNW 9 | WSseMRpEWelcrp19et49TLPmOwTnIK4CH/TP5QfDCoDoLl2arjSixVYfTB5HeCpG 10 | dpVtBX/NgAdvyhfah2q6vKxIVttHDZTt5pB33qlzHY6CexRDW11kKNiua1hwZhsu 11 | SoqtT3Jo0U5zJVjAyOnf9u7+ckFLSiXrZTI8eJzKD/xhcVRBiefQFdbdz00S9Whp 12 | AqbcRh518rrq6SV7XJr9PnkmvoaIl2H2K7rPi17gu1PEDlCUvBu4khgE+cVNXoUF 13 | M+l3xXlen2Vz2LNLS1fR3XTLZdUDG2plgipIwcpMDrPhWHbqJwEul0S8Jivd2Rcs 14 | uoIhd4diDtosQZfMTrqnPa09pr+naMhZpmCFEZ8412CN/J07RfSHd9zt9mBi06Vb 15 | 4wTJJRbJd0wd8IX3xTqHqZ9GlEXTCGDql45JkcvmgckMO0FFfI4ygzAsm9N+23q4 16 | 2jho0UULq3c1GNAtVGmQe2CQ+t++Y1qSxktsqDi61NgO1+ictRs+armkQ3DWRd2y 17 | CKyEdoJCEM9Jv587GqD48gymuotdem+B9B/WQxYdTJI61WFZnjObEBtT/VTXR20W 18 | D/jGucNLHb/0brv6MSPwbWgyJ1gO7JTbYdam0iwuqTdaHDarXQisD7V1L7dPdt4O 19 | 7yVepouTIoZ1RxEVjE9zXydP1OgzaZSl8xMyabtT56chjki5uXrIcDjcRqjU/cwr 20 | 2UrqTNBkg20dkeeHWU73mBzAk14OF3bZF3P+yAwXMkEBDyaFdeQ1+mJCJwqd/Nxl 21 | MNwr98VhhLgXUvyY5RrcmrJp0tM2A75RNIjTqO+ybElR5J7EYLSc71wG3xFZ6hcV 22 | A9EmBZg4BQHIildE8MymW2bDeTFZMzdcviAdMIV7xb0/daBSokuMnyHLZbU2CUGe 23 | tCsTxZSeR6JYoGuSaj/EtZWGY42qH0X1++b2bjRtubdqJVLuVH0KqQ+rspwmXENk 24 | p5ZrBQM2yz+Z4PK65f6lHP/5XaCKMSI6WFITxrU3nQqHvngfyQy5O09qq7L6sWcf 25 | ODQwYuc7XtUMsvY9ciORaym6VMhzadoXcUnNGOcEmbLz7Vbdv7s0OGjAB5rcbpaF 26 | i5AR3ExDUniUgG7G3EieX6SNmUwhcWI09ujux3vArE7lleEmUnA4rsksqrKoNual 27 | oq8y2xG6wVQy6AFJfe3hK0Yq+AqpH1psRDwmfPxMSnfdsVRRPwXS4JHGJK1n2l8y 28 | Pg3XilyOo0nu3y/24xmYXv1WLL5iIkNxKb7fdN0lEm4PibXQrenS56nW002ZIDpe 29 | lZ2pu7P/on0qDzBZuU/uwozOWw== 30 | -----END ENCRYPTED PRIVATE KEY----- 31 | -------------------------------------------------------------------------------- /server/conf/utils.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package conf 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "os" 23 | "path/filepath" 24 | ) 25 | 26 | // validatePathExists checks that the provided path exists and is a dir if requested 27 | func validatePathExists(path string, dir bool) (string, error) { 28 | if path == "" { 29 | return "", errors.New("path is not specified") 30 | } 31 | 32 | abs, err := filepath.Abs(path) 33 | if err != nil { 34 | return "", fmt.Errorf("error parsing path [%s]: %v", abs, err) 35 | } 36 | 37 | var finfo os.FileInfo 38 | if finfo, err = os.Stat(abs); os.IsNotExist(err) { 39 | return "", fmt.Errorf("the path [%s] doesn't exist", abs) 40 | } 41 | 42 | mode := finfo.Mode() 43 | if dir && mode.IsRegular() { 44 | return "", fmt.Errorf("the path [%s] is not a directory", abs) 45 | } 46 | 47 | if !dir && mode.IsDir() { 48 | return "", fmt.Errorf("the path [%s] is not a file", abs) 49 | } 50 | 51 | return abs, nil 52 | } 53 | 54 | // ValidateDirPath checks that the provided path exists and is a dir 55 | func ValidateDirPath(path string) (string, error) { 56 | return validatePathExists(path, true) 57 | } 58 | 59 | // ValidateFilePath checks that the provided path exists and is not a dir 60 | func ValidateFilePath(path string) (string, error) { 61 | return validatePathExists(path, false) 62 | } 63 | -------------------------------------------------------------------------------- /server/conf/utils_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package conf 18 | 19 | import ( 20 | "io/ioutil" 21 | "os" 22 | "path/filepath" 23 | "testing" 24 | 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestFilePath(t *testing.T) { 29 | file, err := ioutil.TempFile(os.TempDir(), "prefix") 30 | require.NoError(t, err) 31 | 32 | path, err := ValidateFilePath(file.Name()) 33 | require.NoError(t, err) 34 | require.NotEqual(t, "", path) 35 | 36 | _, err = ValidateDirPath(file.Name()) 37 | require.Error(t, err) 38 | } 39 | 40 | func TestDirPath(t *testing.T) { 41 | path, err := ioutil.TempDir(os.TempDir(), "prefix") 42 | require.NoError(t, err) 43 | 44 | abspath, err := ValidateDirPath(path) 45 | require.NoError(t, err) 46 | require.NotEqual(t, "", abspath) 47 | 48 | _, err = ValidateFilePath(path) 49 | require.Error(t, err) 50 | } 51 | 52 | func TestPathDoesntExist(t *testing.T) { 53 | path, err := ioutil.TempDir(os.TempDir(), "prefix") 54 | require.NoError(t, err) 55 | 56 | path = filepath.Join(path, "foo") 57 | 58 | _, err = ValidateDirPath(path) 59 | require.Error(t, err) 60 | } 61 | 62 | func TestEmptyPath(t *testing.T) { 63 | _, err := ValidateFilePath("") 64 | require.Error(t, err) 65 | 66 | _, err = ValidateDirPath("") 67 | require.Error(t, err) 68 | } 69 | 70 | func TestBadPath(t *testing.T) { 71 | _, err := ValidateFilePath("//foo\\br//#!90") 72 | require.Error(t, err) 73 | 74 | _, err = ValidateDirPath("//foo\\br//#!90") 75 | require.Error(t, err) 76 | } 77 | -------------------------------------------------------------------------------- /server/conf/load.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package conf 18 | 19 | import ( 20 | "fmt" 21 | "io/ioutil" 22 | 23 | "github.com/nats-io/nats-server/v2/conf" 24 | ) 25 | 26 | // LoadConfigFromFile - given a struct, load a config from a file and fill in the struct 27 | // If strict is true, all of the fields in the config struct must be in the file 28 | // otherwise, the fields in the config struct will act as defaults if the file doesn't contain them 29 | // Strict will also force an error if the struct contains any fields which are not settable with reflection 30 | func LoadConfigFromFile(configFile string, configStruct interface{}, strict bool) error { 31 | configString, err := ioutil.ReadFile(configFile) 32 | if err != nil { 33 | return fmt.Errorf("error reading configuration file: %s", err.Error()) 34 | } 35 | 36 | return LoadConfigFromString(string(configString), configStruct, strict) 37 | } 38 | 39 | // LoadConfigFromString - like LoadConfigFromFile but uses a string 40 | func LoadConfigFromString(configString string, configStruct interface{}, strict bool) error { 41 | m, err := conf.Parse(string(configString)) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | return parseStruct(m, configStruct, strict) 47 | } 48 | 49 | // LoadConfigFromMap load a config struct from a map, this is useful if the type of a config isn't known at 50 | // load time. 51 | func LoadConfigFromMap(m map[string]interface{}, configStruct interface{}, strict bool) error { 52 | return parseStruct(m, configStruct, strict) 53 | } 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![NATS](logos/large-logo.png) 2 | 3 | # NATS Replicator 4 | 5 | [![License][License-Image]][License-Url] 6 | [![ReportCard][ReportCard-Image]][ReportCard-Url] 7 | [![Build][Build-Status-Image]][Build-Status-Url] 8 | 9 | This project implements a multi-connector bridge, copier, replicator between NATS and NATS streaming endpoints. 10 | 11 | ## Features 12 | 13 | * Arbitrary subjects in NATS, wildcards for incoming messages 14 | * Arbitrary channels in NATS streaming 15 | * Optional durable subscriber names for streaming 16 | * Configurable std-out logging 17 | * A single configuration file, with support for reload 18 | * Optional SSL to/from NATS and NATS streaming 19 | * HTTP/HTTPS-based monitoring endpoints for health or statistics 20 | 21 | ## Overview 22 | 23 | The replicator runs as a single process with a configured set of connectors mapping a between a NATS subject or a NATS streaming channel. Each connector is a one-way replicator. 24 | 25 | Connectors share a NATS connection and an optional connection to the NATS streaming server. 26 | 27 | Request-reply is not supported. 28 | 29 | The replicator is [configured with a NATS server-like format](docs/config.md), in a single file and uses the NATS logger. 30 | 31 | An [optional HTTP/HTTPS endpoint](docs/monitoring.md) can be used for monitoring. 32 | 33 | ## Todo 34 | 35 | * Integrate with coveralls 36 | 37 | ## Documentation 38 | 39 | * [Build & Run the Replicator](docs/buildandrun.md) 40 | * [Configuration](docs/config.md) 41 | * [Monitoring](docs/monitoring.md) 42 | 43 | ## External Resources 44 | 45 | * [NATS](https://nats.io/documentation/) 46 | * [NATS server](https://github.com/nats-io/nats-server) 47 | * [NATS Streaming](https://github.com/nats-io/nats-streaming-server) 48 | 49 | [License-Url]: https://www.apache.org/licenses/LICENSE-2.0 50 | [License-Image]: https://img.shields.io/badge/License-Apache2-blue.svg 51 | [Build-Status-Url]: https://travis-ci.org/nats-io/nats-replicator 52 | [Build-Status-Image]: https://travis-ci.org/nats-io/nats-replicator.svg?branch=master 53 | [ReportCard-Url]: https://goreportcard.com/report/nats-io/nats-replicator 54 | [ReportCard-Image]: https://goreportcard.com/badge/github.com/nats-io/nats-replicator?v=2 55 | 56 | 57 | 58 | ## License 59 | 60 | Unless otherwise noted, the nats-replicator source files are distributed under the Apache Version 2.0 license found in the LICENSE file. 61 | -------------------------------------------------------------------------------- /docs/monitoring.md: -------------------------------------------------------------------------------- 1 | # Monitoring the NATS-Replicator 2 | 3 | The nats-replicator provides optional HTTP/s monitoring. When [configured with a monitoring port](config.md#monitoring) the server will provide two HTTP endpoints: 4 | 5 | * [/varz](#varz) 6 | * [/healthz](#healthz) 7 | 8 | You can also just navigate to the monitoring port, i.e. http://localhost:9090, and a page will point you at these two paths. 9 | 10 | 11 | 12 | ## /varz 13 | 14 | The `/varz` endpoint returns a JSON encoded set of statistics for the server. These statistics are wrapped in a root level object with the following properties: 15 | 16 | * `start_time` - the start time of the replicator, in the replicator's timezone. 17 | * `current_time` - the current time, in the replicator's timezone. 18 | * `uptime` - a string representation of the replicator's up time. 19 | * `http_requests` - a map of request paths to counts, the keys are `/`, `/varz` and `/healthz`. 20 | * `connectors` - an array of statistics for each connector. 21 | 22 | Each object in the connectors array, one per connector, will contain the following properties: 23 | 24 | * `name` - the name of the connector, a human readable description of the connector. 25 | * `id` - the connectors id, either set in the configuration or generated at runtime. 26 | * `connects` - a count of the number of times the connector has connected. 27 | * `disconnects` - a count of the number of times the connector has disconnected. 28 | * `bytes_in` - the number of bytes the connector has received, may differ from received due to headers and encoding. 29 | * `bytes_out` - the number of bytes the connector has sent, may differ from received due to headers and encoding. 30 | * `msg_in` - the number of messages received. 31 | * `msg_out` - the number of messages sent. 32 | * `count` - the total number of requests for this connector. 33 | * `rma` - a [running moving average](https://en.wikipedia.org/wiki/Moving_average) of the time required to handle each request. The time is in nanoseconds. 34 | * `q50` - the 50% quantile for response times, in nanoseconds. 35 | * `q75` - the 75% quantile for response times, in nanoseconds. 36 | * `q90` - the 90% quantile for response times, in nanoseconds. 37 | * `q95` - the 95% quantile for response times, in nanoseconds. 38 | 39 | Pass the URL property pretty=true to get formatted JSON. For example, http://localhost:8080/varz?pretty=true. 40 | 41 | 42 | 43 | ## /healthz 44 | 45 | The `/healthz` endpoint is provided for automated up/down style checks. The server returns an HTTP/200 when running and won't respond if it is down. 46 | -------------------------------------------------------------------------------- /server/core/nats_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "testing" 20 | "time" 21 | 22 | "github.com/nats-io/nats-replicator/server/conf" 23 | "github.com/nats-io/nuid" 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | func TestDisconnectReconnect(t *testing.T) { 28 | incoming := nuid.Next() 29 | outgoing := nuid.Next() 30 | 31 | connect := []conf.ConnectorConfig{ 32 | { 33 | Type: "StanToStan", 34 | IncomingChannel: incoming, 35 | OutgoingChannel: outgoing, 36 | IncomingConnection: "stan", 37 | OutgoingConnection: "stan", 38 | }, 39 | } 40 | 41 | // Default pings are 2 seconds, with a max of 3 for tests 42 | tbs, err := StartTestEnvironment(connect) 43 | require.NoError(t, err) 44 | defer tbs.Close() 45 | 46 | tbs.Bridge.checkConnections() 47 | 48 | require.NotNil(t, tbs.Bridge.nats["nats"]) 49 | require.NotNil(t, tbs.Bridge.stan["stan"]) 50 | require.True(t, tbs.Bridge.CheckNATS("nats")) 51 | require.True(t, tbs.Bridge.CheckStan("stan")) 52 | 53 | tbs.StopNATS() 54 | 55 | tbs.Bridge.nats["nats"].FlushTimeout(time.Second * 5) 56 | 57 | now := time.Now() 58 | 59 | // wait 5 seconds for connection lost 60 | // ping interval is 1sec for tests and max pings is 3 61 | for time.Since(now) < time.Second*5 { 62 | if tbs.Bridge.Stan("stan") == nil { 63 | break 64 | } 65 | 66 | time.Sleep(time.Millisecond * 500) 67 | } 68 | 69 | require.NotNil(t, tbs.Bridge.nats["nats"]) 70 | require.Nil(t, tbs.Bridge.stan["stan"]) 71 | require.False(t, tbs.Bridge.CheckNATS("nats")) 72 | require.False(t, tbs.Bridge.CheckStan("stan")) 73 | 74 | tbs.RestartNATS() 75 | 76 | now = time.Now() 77 | 78 | // Wait for a reconnect 79 | for time.Since(now) < time.Second*20 && !tbs.Bridge.CheckStan("stan") { 80 | time.Sleep(time.Millisecond * 100) 81 | } 82 | 83 | require.NotNil(t, tbs.Bridge.nats["nats"]) 84 | require.NotNil(t, tbs.Bridge.stan["stan"]) 85 | require.True(t, tbs.Bridge.CheckNATS("nats")) 86 | require.True(t, tbs.Bridge.CheckStan("stan")) 87 | } 88 | -------------------------------------------------------------------------------- /server/core/stats_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package core 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | func TestMessageCounts(t *testing.T) { 27 | statsH := NewConnectorStatsHolder("one", "two") 28 | loops := int64(101) 29 | sizeIn := int64(11) 30 | sizeOut := int64(13) 31 | 32 | for i := int64(0); i < loops; i++ { 33 | statsH.AddMessageIn(sizeIn) 34 | statsH.AddMessageOut(sizeOut) 35 | } 36 | 37 | stats := statsH.Stats() 38 | require.Equal(t, "one", statsH.Name()) 39 | require.Equal(t, "two", statsH.ID()) 40 | require.Equal(t, "one", stats.Name) 41 | require.Equal(t, "two", stats.ID) 42 | require.Equal(t, loops, stats.MessagesIn) 43 | require.Equal(t, loops, stats.MessagesOut) 44 | require.Equal(t, loops*sizeIn, stats.BytesIn) 45 | require.Equal(t, loops*sizeOut, stats.BytesOut) 46 | } 47 | 48 | func TestConnectDisconnectCounts(t *testing.T) { 49 | statsH := NewConnectorStatsHolder("one", "two") 50 | loops := int64(101) 51 | 52 | for i := int64(0); i < loops; i++ { 53 | statsH.AddDisconnect() 54 | statsH.AddConnect() 55 | statsH.AddConnect() 56 | } 57 | 58 | stats := statsH.Stats() 59 | require.Equal(t, "one", statsH.Name()) 60 | require.Equal(t, "two", statsH.ID()) 61 | require.Equal(t, "one", stats.Name) 62 | require.Equal(t, "two", stats.ID) 63 | require.Equal(t, loops, stats.Disconnects) 64 | require.Equal(t, 2*loops, stats.Connects) 65 | require.Equal(t, int64(0), stats.BytesIn) 66 | require.Equal(t, int64(0), stats.BytesOut) 67 | require.Equal(t, int64(0), stats.MessagesIn) 68 | require.Equal(t, int64(0), stats.MessagesOut) 69 | } 70 | 71 | func TestRequestTimes(t *testing.T) { 72 | statsH := NewConnectorStatsHolder("one", "two") 73 | 74 | dur := 4 * time.Second 75 | 76 | statsH.AddRequestTime(dur) 77 | 78 | stats := statsH.Stats() 79 | require.Equal(t, "one", statsH.Name()) 80 | require.Equal(t, "two", statsH.ID()) 81 | require.Equal(t, "one", stats.Name) 82 | require.Equal(t, "two", stats.ID) 83 | require.Equal(t, float64(dur.Nanoseconds()), stats.MovingAverage) 84 | require.Equal(t, int64(1), stats.RequestCount) 85 | } 86 | -------------------------------------------------------------------------------- /server/logging/nats.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package logging 18 | 19 | import ( 20 | "github.com/nats-io/nats-server/v2/logger" 21 | ) 22 | 23 | // NewNATSLogger creates a new logger that uses the nats-server library 24 | func NewNATSLogger(conf Config) Logger { 25 | l := logger.NewStdLogger(conf.Time, conf.Debug, conf.Trace, conf.Colors, conf.PID) 26 | return &NATSLogger{ 27 | logger: l, 28 | traceEnabled: conf.Trace, 29 | hide: conf.Hide, 30 | } 31 | } 32 | 33 | // NATSLogger - uses the nats-server logging code 34 | type NATSLogger struct { 35 | logger *logger.Logger 36 | traceEnabled bool 37 | hide bool 38 | } 39 | 40 | // TraceEnabled returns true if tracing is configured, useful for fast path logging 41 | func (logger *NATSLogger) TraceEnabled() bool { 42 | return logger.traceEnabled 43 | } 44 | 45 | // Close forwards to the nats logger 46 | func (logger *NATSLogger) Close() error { 47 | return logger.logger.Close() 48 | } 49 | 50 | // Debugf forwards to the nats logger 51 | func (logger *NATSLogger) Debugf(format string, v ...interface{}) { 52 | if logger.hide { 53 | return 54 | } 55 | logger.logger.Debugf(format, v...) 56 | } 57 | 58 | // Errorf forwards to the nats logger 59 | func (logger *NATSLogger) Errorf(format string, v ...interface{}) { 60 | if logger.hide { 61 | return 62 | } 63 | logger.logger.Errorf(format, v...) 64 | } 65 | 66 | // Fatalf forwards to the nats logger 67 | func (logger *NATSLogger) Fatalf(format string, v ...interface{}) { 68 | if logger.hide { 69 | return 70 | } 71 | logger.logger.Fatalf(format, v...) 72 | } 73 | 74 | // Noticef forwards to the nats logger 75 | func (logger *NATSLogger) Noticef(format string, v ...interface{}) { 76 | if logger.hide { 77 | return 78 | } 79 | logger.logger.Noticef(format, v...) 80 | } 81 | 82 | // Tracef forwards to the nats logger 83 | func (logger *NATSLogger) Tracef(format string, v ...interface{}) { 84 | if logger.hide || !logger.traceEnabled { 85 | return 86 | } 87 | logger.logger.Tracef(format, v...) 88 | } 89 | 90 | // Warnf forwards to the nats logger 91 | func (logger *NATSLogger) Warnf(format string, v ...interface{}) { 92 | if logger.hide { 93 | return 94 | } 95 | logger.logger.Warnf(format, v...) 96 | } 97 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "fmt" 22 | "log" 23 | "os" 24 | "os/signal" 25 | "runtime" 26 | "syscall" 27 | 28 | "github.com/nats-io/nats-replicator/server/core" 29 | ) 30 | 31 | func main() { 32 | var server *core.NATSReplicator 33 | var err error 34 | 35 | flags := core.Flags{} 36 | flag.StringVar(&flags.ConfigFile, "c", "", "configuration filepath, other flags take precedent over the config file, can be set with $NATS_REPLICATOR_CONFIG") 37 | flag.BoolVar(&flags.Debug, "D", false, "turn on debug logging") 38 | flag.BoolVar(&flags.Verbose, "V", false, "turn on verbose logging") 39 | flag.BoolVar(&flags.DebugAndVerbose, "DV", false, "turn on debug and verbose logging") 40 | flag.Parse() 41 | 42 | go func() { 43 | sigChan := make(chan os.Signal, 1) 44 | signal.Notify(sigChan, os.Interrupt, syscall.SIGHUP) 45 | 46 | for { 47 | signal := <-sigChan 48 | 49 | if signal == os.Interrupt { 50 | if server.Logger() != nil { 51 | fmt.Println() // clear the line for the control-C 52 | server.Logger().Noticef("received sig-interrupt, shutting down") 53 | } 54 | server.Stop() 55 | os.Exit(0) 56 | } 57 | 58 | if signal == syscall.SIGHUP { 59 | if server.Logger() != nil { 60 | server.Logger().Errorf("received sig-hup, restarting") 61 | } 62 | server.Stop() 63 | server := core.NewNATSReplicator() 64 | server.InitializeFromFlags(flags) 65 | err = server.Start() 66 | 67 | if err != nil { 68 | if server.Logger() != nil { 69 | server.Logger().Errorf("error starting replicator, %s", err.Error()) 70 | } else { 71 | log.Printf("error starting replicator, %s", err.Error()) 72 | } 73 | server.Stop() 74 | os.Exit(0) 75 | } 76 | } 77 | } 78 | }() 79 | 80 | server = core.NewNATSReplicator() 81 | server.InitializeFromFlags(flags) 82 | err = server.Start() 83 | 84 | if err != nil { 85 | if server.Logger() != nil { 86 | server.Logger().Errorf("error starting replicator, %s", err.Error()) 87 | } else { 88 | log.Printf("error starting replicator, %s", err.Error()) 89 | } 90 | server.Stop() 91 | os.Exit(0) 92 | } 93 | 94 | // exit main but keep running goroutines 95 | runtime.Goexit() 96 | } 97 | -------------------------------------------------------------------------------- /server/conf/load_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package conf 18 | 19 | import ( 20 | "io/ioutil" 21 | "os" 22 | "testing" 23 | 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | type SimpleConf struct { 28 | Name string 29 | Age int64 30 | OptOut bool 31 | Balance float64 32 | } 33 | 34 | func TestLoadFromString(t *testing.T) { 35 | configString := ` 36 | Name: "stephen" 37 | Age: 28 38 | OptOut: true 39 | Balance: 5.5 40 | ` 41 | 42 | config := SimpleConf{} 43 | 44 | err := LoadConfigFromString(configString, &config, false) 45 | require.NoError(t, err) 46 | require.Equal(t, "stephen", config.Name) 47 | require.Equal(t, int64(28), config.Age) 48 | require.Equal(t, true, config.OptOut) 49 | require.Equal(t, 5.5, config.Balance) 50 | } 51 | 52 | func TestLoadFromFile(t *testing.T) { 53 | file, err := ioutil.TempFile(os.TempDir(), "prefix") 54 | require.NoError(t, err) 55 | 56 | configString := ` 57 | Name: "stephen" 58 | Age: 28 59 | OptOut: true 60 | Balance: 5.5 61 | ` 62 | 63 | fullPath, err := ValidateFilePath(file.Name()) 64 | require.NoError(t, err) 65 | 66 | err = ioutil.WriteFile(fullPath, []byte(configString), 0644) 67 | require.NoError(t, err) 68 | 69 | config := SimpleConf{} 70 | 71 | err = LoadConfigFromFile(fullPath, &config, false) 72 | require.NoError(t, err) 73 | require.Equal(t, "stephen", config.Name) 74 | require.Equal(t, int64(28), config.Age) 75 | require.Equal(t, true, config.OptOut) 76 | require.Equal(t, 5.5, config.Balance) 77 | } 78 | 79 | func TestLoadFromMissingFile(t *testing.T) { 80 | config := SimpleConf{} 81 | err := LoadConfigFromFile("/foo/bar/baz", &config, false) 82 | require.Error(t, err) 83 | } 84 | 85 | type MapConf struct { 86 | One map[string]interface{} 87 | Two map[string]interface{} 88 | } 89 | 90 | func TestLoadFromMap(t *testing.T) { 91 | configString := ` 92 | One: { 93 | Name: "stephen" 94 | Age: 28 95 | OptOut: true 96 | Balance: 5.5 97 | }, Two: { 98 | Name: "zero" 99 | Age: 32 100 | OptOut: false 101 | Balance: 7.7 102 | } 103 | ` 104 | 105 | config := MapConf{} 106 | 107 | err := LoadConfigFromString(configString, &config, false) 108 | require.NoError(t, err) 109 | 110 | one := SimpleConf{} 111 | two := SimpleConf{} 112 | 113 | err = LoadConfigFromMap(config.One, &one, false) 114 | require.NoError(t, err) 115 | require.Equal(t, "stephen", one.Name) 116 | require.Equal(t, int64(28), one.Age) 117 | require.Equal(t, true, one.OptOut) 118 | require.Equal(t, 5.5, one.Balance) 119 | 120 | err = LoadConfigFromMap(config.Two, &two, false) 121 | require.NoError(t, err) 122 | require.Equal(t, "zero", two.Name) 123 | require.Equal(t, int64(32), two.Age) 124 | require.Equal(t, false, two.OptOut) 125 | require.Equal(t, 7.7, two.Balance) 126 | } 127 | -------------------------------------------------------------------------------- /server/core/monitoring_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "crypto/tls" 20 | "encoding/json" 21 | "io/ioutil" 22 | "net/http" 23 | "strings" 24 | "testing" 25 | "time" 26 | 27 | "github.com/nats-io/nats-replicator/server/conf" 28 | "github.com/stretchr/testify/require" 29 | ) 30 | 31 | func TestMonitoringPages(t *testing.T) { 32 | start := time.Now() 33 | 34 | connect := []conf.ConnectorConfig{ 35 | { 36 | Type: "NATSToNATS", 37 | OutgoingSubject: "test3", 38 | OutgoingConnection: "nats", 39 | IncomingSubject: "test.*", 40 | IncomingConnection: "nats", 41 | }, 42 | } 43 | 44 | tbs, err := StartTestEnvironment(connect) 45 | require.NoError(t, err) 46 | defer tbs.Close() 47 | 48 | client := http.Client{} 49 | response, err := client.Get(tbs.Bridge.GetMonitoringRootURL()) 50 | require.NoError(t, err) 51 | defer response.Body.Close() 52 | contents, err := ioutil.ReadAll(response.Body) 53 | require.NoError(t, err) 54 | html := string(contents) 55 | require.True(t, strings.Contains(html, "/varz")) 56 | require.True(t, strings.Contains(html, "/healthz")) 57 | 58 | response, err = client.Get(tbs.Bridge.GetMonitoringRootURL() + "healthz") 59 | require.NoError(t, err) 60 | require.Equal(t, http.StatusOK, response.StatusCode) 61 | 62 | response, err = client.Get(tbs.Bridge.GetMonitoringRootURL() + "varz") 63 | require.NoError(t, err) 64 | defer response.Body.Close() 65 | contents, err = ioutil.ReadAll(response.Body) 66 | require.NoError(t, err) 67 | 68 | bridgeStats := BridgeStats{} 69 | err = json.Unmarshal(contents, &bridgeStats) 70 | require.NoError(t, err) 71 | 72 | now := time.Now() 73 | require.True(t, bridgeStats.StartTime >= start.Unix()) 74 | require.True(t, bridgeStats.StartTime <= now.Unix()) 75 | require.True(t, bridgeStats.ServerTime >= start.Unix()) 76 | require.True(t, bridgeStats.ServerTime <= now.Unix()) 77 | 78 | require.Equal(t, bridgeStats.HTTPRequests["/"], int64(1)) 79 | require.Equal(t, bridgeStats.HTTPRequests["/varz"], int64(1)) 80 | require.Equal(t, bridgeStats.HTTPRequests["/healthz"], int64(1)) 81 | 82 | require.Equal(t, 1, len(bridgeStats.Connections)) 83 | require.True(t, bridgeStats.Connections[0].Connected) 84 | require.Equal(t, int64(1), bridgeStats.Connections[0].Connects) 85 | require.Equal(t, int64(0), bridgeStats.Connections[0].MessagesIn) 86 | require.Equal(t, int64(0), bridgeStats.Connections[0].MessagesOut) 87 | require.Equal(t, int64(0), bridgeStats.Connections[0].BytesIn) 88 | require.Equal(t, int64(0), bridgeStats.Connections[0].BytesOut) 89 | } 90 | 91 | func TestHealthzWithTLS(t *testing.T) { 92 | connect := []conf.ConnectorConfig{} 93 | 94 | tbs, err := StartTLSTestEnvironment(connect) 95 | require.NoError(t, err) 96 | defer tbs.Close() 97 | 98 | tr := &http.Transport{ 99 | TLSClientConfig: &tls.Config{ 100 | InsecureSkipVerify: true, 101 | }, 102 | } 103 | client := &http.Client{Transport: tr} 104 | response, err := client.Get(tbs.Bridge.GetMonitoringRootURL() + "healthz") 105 | require.NoError(t, err) 106 | require.Equal(t, http.StatusOK, response.StatusCode) 107 | } 108 | -------------------------------------------------------------------------------- /server/conf/conf_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package conf 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func TestDefaultConfig(t *testing.T) { 26 | config := DefaultConfig() 27 | require.Equal(t, false, config.Logging.Trace) 28 | require.Equal(t, 5000, config.Monitoring.ReadTimeout) 29 | } 30 | 31 | func TestMakeTLSConfig(t *testing.T) { 32 | tlsC := &TLSConf{ 33 | Cert: "../../resources/certs/client-cert.pem", 34 | Key: "../../resources/certs/client-key.pem", 35 | Root: "../../resources/certs/truststore.pem", 36 | } 37 | _, err := tlsC.MakeTLSConfig() 38 | require.NoError(t, err) 39 | } 40 | 41 | func TestNATSConfig(t *testing.T) { 42 | config := DefaultConfig() 43 | configString := ` 44 | { 45 | connect: [ 46 | { 47 | incomingConnection: "one" 48 | Outgoingconnection: "one" 49 | incomingSubject: "test" 50 | OutgoingSubject: "hello" 51 | } 52 | ], 53 | nats: [ 54 | { 55 | name: "one" 56 | servers: ["nats://localhost:4222"] 57 | } 58 | ] 59 | monitoring: { 60 | HTTPPort: -1, 61 | ReadTimeout: 2000, 62 | } 63 | } 64 | ` 65 | 66 | err := LoadConfigFromString(configString, &config, false) 67 | require.NoError(t, err) 68 | 69 | require.Equal(t, 1, len(config.NATS)) 70 | require.Equal(t, 1, len(config.NATS[0].Servers)) 71 | require.Equal(t, config.NATS[0].Servers[0], "nats://localhost:4222") 72 | require.Equal(t, config.Monitoring.ReadTimeout, 2000) 73 | require.Equal(t, config.Logging.Trace, false) 74 | require.Equal(t, config.Logging.Debug, false) 75 | require.Len(t, config.Connect, 1) 76 | require.Equal(t, config.Connect[0].IncomingConnection, "one") 77 | require.Equal(t, config.Connect[0].OutgoingConnection, "one") 78 | require.Equal(t, config.Connect[0].IncomingSubject, "test") 79 | require.Equal(t, config.Connect[0].OutgoingSubject, "hello") 80 | } 81 | 82 | func TestNATSConfigWithTags(t *testing.T) { 83 | config := DefaultConfig() 84 | configString := ` 85 | { 86 | reconnect_interval: 1000 87 | connect: [ 88 | { 89 | incoming_connection: "one" 90 | outgoing_connection: "one" 91 | incoming_subject: "test" 92 | outgoing_subject: "hello" 93 | } 94 | ], 95 | nats: [ 96 | { 97 | name: "one" 98 | servers: ["nats://localhost:4222"] 99 | connect_timeout: 5000 100 | } 101 | ] 102 | monitoring: { 103 | http_port: -1, 104 | read_timeout: 2000, 105 | } 106 | } 107 | ` 108 | 109 | err := LoadConfigFromString(configString, &config, false) 110 | require.NoError(t, err) 111 | 112 | require.Equal(t, config.ReconnectInterval, 1000) 113 | require.Equal(t, 1, len(config.NATS)) 114 | require.Equal(t, 1, len(config.NATS[0].Servers)) 115 | require.Equal(t, config.NATS[0].Servers[0], "nats://localhost:4222") 116 | require.Equal(t, config.NATS[0].ConnectTimeout, 5000) 117 | require.Equal(t, config.Monitoring.ReadTimeout, 2000) 118 | require.Equal(t, config.Logging.Trace, false) 119 | require.Equal(t, config.Logging.Debug, false) 120 | require.Len(t, config.Connect, 1) 121 | require.Equal(t, config.Connect[0].IncomingConnection, "one") 122 | require.Equal(t, config.Connect[0].OutgoingConnection, "one") 123 | require.Equal(t, config.Connect[0].IncomingSubject, "test") 124 | require.Equal(t, config.Connect[0].OutgoingSubject, "hello") 125 | } 126 | -------------------------------------------------------------------------------- /server/core/histogram.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | // Based on https://github.com/VividCortex/gohistogram MIT license 19 | // Updated to be json friendly 20 | // Histogram based on https://www.vividcortex.com/blog/2013/07/08/streaming-approximate-histograms/ 21 | 22 | // Bin holds a float64 value and count 23 | type Bin struct { 24 | Value float64 `json:"v"` 25 | Count float64 `json:"c"` 26 | } 27 | 28 | // Histogram stores N bins using the streaming approximate histogram approach 29 | // The histogram is not thread safe 30 | type Histogram struct { 31 | Bins []Bin `json:"bins"` 32 | MaxBins int `json:"max"` 33 | Total uint64 `json:"total"` 34 | } 35 | 36 | // NewHistogram returns a new Histogram with a maximum of n bins. 37 | // 38 | // There is no "optimal" bin count, but somewhere between 20 and 80 bins 39 | // should be sufficient. 40 | func NewHistogram(n int) *Histogram { 41 | return &Histogram{ 42 | Bins: make([]Bin, 0), 43 | MaxBins: n, 44 | Total: 0, 45 | } 46 | } 47 | 48 | // Scale the buckets by s, this is useful for requests or other 49 | // values that may be in large numbers ie nanoseconds 50 | func (h *Histogram) Scale(s float64) { 51 | for i := range h.Bins { 52 | h.Bins[i].Value *= s 53 | } 54 | } 55 | 56 | // Add a value to the histogram, creating a bucket if necessary 57 | func (h *Histogram) Add(n float64) { 58 | defer h.trim() 59 | h.Total++ 60 | for i := range h.Bins { 61 | if h.Bins[i].Value == n { 62 | h.Bins[i].Count++ 63 | return 64 | } 65 | 66 | if h.Bins[i].Value > n { 67 | 68 | newbin := Bin{Value: n, Count: 1} 69 | head := append(make([]Bin, 0), h.Bins[0:i]...) 70 | 71 | head = append(head, newbin) 72 | tail := h.Bins[i:] 73 | h.Bins = append(head, tail...) 74 | return 75 | } 76 | } 77 | 78 | h.Bins = append(h.Bins, Bin{Count: 1, Value: n}) 79 | } 80 | 81 | // Quantile returns the value for the bin at the provided quantile 82 | // This is "approximate" in the since that the bin may straddle the quantile value 83 | func (h *Histogram) Quantile(q float64) float64 { 84 | count := q * float64(h.Total) 85 | for i := range h.Bins { 86 | count -= float64(h.Bins[i].Count) 87 | 88 | if count <= 0 { 89 | return h.Bins[i].Value 90 | } 91 | } 92 | 93 | return -1 94 | } 95 | 96 | // Mean returns the sample mean of the distribution 97 | func (h *Histogram) Mean() float64 { 98 | if h.Total == 0 { 99 | return 0 100 | } 101 | 102 | sum := 0.0 103 | 104 | for i := range h.Bins { 105 | sum += h.Bins[i].Value * h.Bins[i].Count 106 | } 107 | 108 | return sum / float64(h.Total) 109 | } 110 | 111 | // Count returns the total number of entries in the histogram 112 | func (h *Histogram) Count() float64 { 113 | return float64(h.Total) 114 | } 115 | 116 | // trim merges adjacent bins to decrease the bin count to the maximum value 117 | func (h *Histogram) trim() { 118 | for len(h.Bins) > h.MaxBins { 119 | // Find closest bins in terms of value 120 | minDelta := 1e99 121 | minDeltaIndex := 0 122 | for i := range h.Bins { 123 | if i == 0 { 124 | continue 125 | } 126 | 127 | if delta := h.Bins[i].Value - h.Bins[i-1].Value; delta < minDelta { 128 | minDelta = delta 129 | minDeltaIndex = i 130 | } 131 | } 132 | 133 | // We need to merge bins minDeltaIndex-1 and minDeltaIndex 134 | totalCount := h.Bins[minDeltaIndex-1].Count + h.Bins[minDeltaIndex].Count 135 | mergedbin := Bin{ 136 | Value: (h.Bins[minDeltaIndex-1].Value* 137 | h.Bins[minDeltaIndex-1].Count + 138 | h.Bins[minDeltaIndex].Value* 139 | h.Bins[minDeltaIndex].Count) / 140 | totalCount, // weighted average 141 | Count: totalCount, // summed heights 142 | } 143 | head := append(make([]Bin, 0), h.Bins[0:minDeltaIndex-1]...) 144 | tail := append([]Bin{mergedbin}, h.Bins[minDeltaIndex+1:]...) 145 | h.Bins = append(head, tail...) 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /server/core/nats2stan_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/nats-io/nats-replicator/server/conf" 22 | "github.com/nats-io/nuid" 23 | stan "github.com/nats-io/stan.go" 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | func TestSimpleSendOnNATSReceiveOnStan(t *testing.T) { 28 | channel := nuid.Next() 29 | subject := nuid.Next() 30 | msg := "hello world" 31 | 32 | connect := []conf.ConnectorConfig{ 33 | { 34 | Type: "NATSToStan", 35 | OutgoingChannel: channel, 36 | IncomingSubject: subject, 37 | IncomingConnection: "nats", 38 | OutgoingConnection: "stan", 39 | }, 40 | } 41 | 42 | tbs, err := StartTestEnvironment(connect) 43 | require.NoError(t, err) 44 | defer tbs.Close() 45 | 46 | tbs.Bridge.checkConnections() 47 | 48 | done := make(chan string) 49 | sub, err := tbs.SC.Subscribe(channel, func(msg *stan.Msg) { 50 | done <- string(msg.Data) 51 | }) 52 | require.NoError(t, err) 53 | defer sub.Unsubscribe() 54 | 55 | err = tbs.NC.Publish(subject, []byte(msg)) 56 | require.NoError(t, err) 57 | 58 | received := tbs.WaitForIt(1, done) 59 | require.Equal(t, msg, received) 60 | 61 | stats := tbs.Bridge.SafeStats() 62 | connStats := stats.Connections[0] 63 | require.Equal(t, int64(1), connStats.MessagesIn) 64 | require.Equal(t, int64(1), connStats.MessagesOut) 65 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesIn) 66 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesOut) 67 | require.Equal(t, int64(1), connStats.Connects) 68 | require.Equal(t, int64(0), connStats.Disconnects) 69 | require.True(t, connStats.Connected) 70 | } 71 | 72 | func TestSimpleSendOnNATSReceiveOnStanWithGroup(t *testing.T) { 73 | channel := nuid.Next() 74 | subject := nuid.Next() 75 | group := nuid.Next() 76 | msg := "hello world" 77 | 78 | connect := []conf.ConnectorConfig{ 79 | { 80 | Type: "NATSToStan", 81 | IncomingSubject: subject, 82 | IncomingQueueName: group, 83 | IncomingConnection: "nats", 84 | OutgoingChannel: channel, 85 | OutgoingConnection: "stan", 86 | }, 87 | } 88 | 89 | tbs, err := StartTestEnvironment(connect) 90 | require.NoError(t, err) 91 | defer tbs.Close() 92 | 93 | done := make(chan string) 94 | sub, err := tbs.SC.Subscribe(channel, func(msg *stan.Msg) { 95 | done <- string(msg.Data) 96 | }) 97 | require.NoError(t, err) 98 | defer sub.Unsubscribe() 99 | 100 | err = tbs.NC.Publish(subject, []byte(msg)) 101 | require.NoError(t, err) 102 | 103 | received := tbs.WaitForIt(1, done) 104 | require.Equal(t, msg, received) 105 | 106 | stats := tbs.Bridge.SafeStats() 107 | connStats := stats.Connections[0] 108 | require.Equal(t, int64(1), connStats.MessagesIn) 109 | require.Equal(t, int64(1), connStats.MessagesOut) 110 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesIn) 111 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesOut) 112 | require.Equal(t, int64(1), connStats.Connects) 113 | require.Equal(t, int64(0), connStats.Disconnects) 114 | require.True(t, connStats.Connected) 115 | } 116 | 117 | func TestSimpleSendOnQueueReceiveOnStanWithTLS(t *testing.T) { 118 | channel := nuid.Next() 119 | subject := nuid.Next() 120 | msg := "hello world" 121 | 122 | connect := []conf.ConnectorConfig{ 123 | { 124 | Type: "natstostan", // test with different casing 125 | OutgoingChannel: channel, 126 | IncomingSubject: subject, 127 | IncomingConnection: "nats", 128 | OutgoingConnection: "stan", 129 | }, 130 | } 131 | 132 | tbs, err := StartTLSTestEnvironment(connect) 133 | require.NoError(t, err) 134 | defer tbs.Close() 135 | 136 | done := make(chan string) 137 | sub, err := tbs.SC.Subscribe(channel, func(msg *stan.Msg) { 138 | done <- string(msg.Data) 139 | }) 140 | require.NoError(t, err) 141 | defer sub.Unsubscribe() 142 | 143 | err = tbs.NC.Publish(subject, []byte(msg)) 144 | require.NoError(t, err) 145 | 146 | received := tbs.WaitForIt(1, done) 147 | require.Equal(t, msg, received) 148 | } 149 | -------------------------------------------------------------------------------- /server/core/server_config_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package core 18 | 19 | import ( 20 | "fmt" 21 | "io/ioutil" 22 | "net/http" 23 | "os" 24 | "testing" 25 | "time" 26 | 27 | "github.com/nats-io/nats-replicator/server/conf" 28 | "github.com/stretchr/testify/require" 29 | ) 30 | 31 | func TestStartWithConfigFileFlag(t *testing.T) { 32 | tbs, err := StartTestEnvironmentInfrastructure(false) 33 | require.NoError(t, err) 34 | defer tbs.Close() 35 | 36 | file, err := ioutil.TempFile(os.TempDir(), "config") 37 | require.NoError(t, err) 38 | 39 | configString := ` 40 | { 41 | connectors: [], 42 | nats: [ 43 | { 44 | name: "one" 45 | servers: ["%s"] 46 | norandom: true 47 | no_echo: true 48 | } 49 | ] 50 | monitoring: { 51 | HTTPPort: -1, 52 | ReadTimeout: 2000, 53 | } 54 | } 55 | ` 56 | configString = fmt.Sprintf(configString, tbs.natsURL) 57 | 58 | fullPath, err := conf.ValidateFilePath(file.Name()) 59 | require.NoError(t, err) 60 | 61 | err = ioutil.WriteFile(fullPath, []byte(configString), 0644) 62 | require.NoError(t, err) 63 | 64 | flags := Flags{ 65 | ConfigFile: fullPath, 66 | DebugAndVerbose: true, 67 | } 68 | 69 | server := NewNATSReplicator() 70 | server.InitializeFromFlags(flags) 71 | err = server.Start() 72 | require.NoError(t, err) 73 | defer server.Stop() 74 | 75 | require.Equal(t, 1, len(server.config.NATS)) 76 | require.Equal(t, 1, len(server.config.NATS[0].Servers)) 77 | require.Equal(t, server.config.NATS[0].Servers[0], tbs.natsURL) 78 | require.Equal(t, server.config.NATS[0].NoRandom, true) 79 | require.Equal(t, server.config.NATS[0].NoEcho, true) 80 | require.Equal(t, server.config.Monitoring.ReadTimeout, 2000) 81 | require.Equal(t, server.config.Logging.Trace, true) 82 | require.Equal(t, server.config.Logging.Debug, true) 83 | 84 | httpClient := http.Client{ 85 | Timeout: 5 * time.Second, 86 | } 87 | 88 | resp, err := httpClient.Get(fmt.Sprintf("%s/healthz", server.monitoringURL)) 89 | require.NoError(t, err) 90 | require.True(t, resp.StatusCode == http.StatusOK) 91 | } 92 | 93 | func TestStartWithConfigFileEnv(t *testing.T) { 94 | tbs, err := StartTestEnvironmentInfrastructure(false) 95 | require.NoError(t, err) 96 | defer tbs.Close() 97 | 98 | file, err := ioutil.TempFile(os.TempDir(), "config") 99 | require.NoError(t, err) 100 | 101 | configString := ` 102 | { 103 | connectors: [], 104 | nats: [ 105 | { 106 | name: "one" 107 | servers: ["%s"] 108 | } 109 | ] 110 | monitoring: { 111 | HTTPPort: -1, 112 | ReadTimeout: 2000, 113 | } 114 | } 115 | ` 116 | configString = fmt.Sprintf(configString, tbs.natsURL) 117 | 118 | fullPath, err := conf.ValidateFilePath(file.Name()) 119 | require.NoError(t, err) 120 | 121 | err = ioutil.WriteFile(fullPath, []byte(configString), 0644) 122 | require.NoError(t, err) 123 | 124 | flags := Flags{ 125 | ConfigFile: "", 126 | DebugAndVerbose: true, 127 | } 128 | 129 | os.Setenv("NATS_REPLICATOR_CONFIG", fullPath) 130 | server := NewNATSReplicator() 131 | server.InitializeFromFlags(flags) 132 | err = server.Start() 133 | require.NoError(t, err) 134 | defer server.Stop() 135 | os.Setenv("NATS_REPLICATOR_CONFIG", "") 136 | 137 | require.Equal(t, 1, len(server.config.NATS)) 138 | require.Equal(t, 1, len(server.config.NATS[0].Servers)) 139 | require.Equal(t, server.config.NATS[0].Servers[0], tbs.natsURL) 140 | require.Equal(t, server.config.Monitoring.ReadTimeout, 2000) 141 | require.Equal(t, server.config.Logging.Trace, true) 142 | require.Equal(t, server.config.Logging.Debug, true) 143 | 144 | httpClient := http.Client{ 145 | Timeout: 5 * time.Second, 146 | } 147 | 148 | resp, err := httpClient.Get(fmt.Sprintf("%s/healthz", server.monitoringURL)) 149 | require.NoError(t, err) 150 | require.True(t, resp.StatusCode == http.StatusOK) 151 | } 152 | 153 | func TestFailWithoutConfigFile(t *testing.T) { 154 | tbs, err := StartTestEnvironmentInfrastructure(false) 155 | require.NoError(t, err) 156 | defer tbs.Close() 157 | 158 | flags := Flags{ 159 | ConfigFile: "", 160 | DebugAndVerbose: true, 161 | } 162 | 163 | os.Setenv("NATS_REPLICATOR_CONFIG", "") 164 | server := NewNATSReplicator() 165 | err = server.InitializeFromFlags(flags) 166 | require.Error(t, err) 167 | } 168 | -------------------------------------------------------------------------------- /server/core/connector.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "fmt" 20 | "strings" 21 | "sync" 22 | "time" 23 | 24 | "github.com/nats-io/nats-replicator/server/conf" 25 | "github.com/nats-io/nuid" 26 | stan "github.com/nats-io/stan.go" 27 | ) 28 | 29 | // Connector is the abstraction for all of the bridge connector types 30 | type Connector interface { 31 | Start() error 32 | Shutdown() error 33 | 34 | CheckConnections() error 35 | 36 | String() string 37 | ID() string 38 | 39 | Stats() ConnectorStats 40 | } 41 | 42 | // CreateConnector builds a connector from the supplied configuration 43 | func CreateConnector(config conf.ConnectorConfig, bridge *NATSReplicator) (Connector, error) { 44 | switch strings.ToLower(config.Type) { 45 | case strings.ToLower(conf.NATSToNATS): 46 | return NewNATS2NATSConnector(bridge, config), nil 47 | case strings.ToLower(conf.StanToNATS): 48 | return NewStan2NATSConnector(bridge, config), nil 49 | case strings.ToLower(conf.NATSToStan): 50 | return NewNATS2StanConnector(bridge, config), nil 51 | case strings.ToLower(conf.StanToStan): 52 | return NewStan2StanConnector(bridge, config), nil 53 | default: 54 | return nil, fmt.Errorf("unknown connector type %q in configuration", config.Type) 55 | } 56 | } 57 | 58 | // ReplicatorConnector is the base type used for connectors so that they can share code 59 | // The config, bridge and stats are all fixed at creation, so no lock is required on the 60 | // connector at this level. The stats do keep a lock to protect their data. 61 | // The connector has a lock for use by composing types to protect themselves during start/shutdown. 62 | type ReplicatorConnector struct { 63 | sync.Mutex 64 | 65 | config conf.ConnectorConfig 66 | bridge *NATSReplicator 67 | stats *ConnectorStatsHolder 68 | } 69 | 70 | // Start is a no-op, designed for overriding 71 | func (conn *ReplicatorConnector) Start() error { 72 | return nil 73 | } 74 | 75 | // Shutdown is a no-op, designed for overriding 76 | func (conn *ReplicatorConnector) Shutdown() error { 77 | return nil 78 | } 79 | 80 | // CheckConnections is a no-op, designed for overriding 81 | // This is called when nats or stan goes down 82 | // the connector should return an error if it has to be shut down 83 | func (conn *ReplicatorConnector) CheckConnections() error { 84 | return nil 85 | } 86 | 87 | // String returns the name passed into init 88 | func (conn *ReplicatorConnector) String() string { 89 | return conn.stats.Name() 90 | } 91 | 92 | // ID returns the id from the stats 93 | func (conn *ReplicatorConnector) ID() string { 94 | return conn.stats.ID() 95 | } 96 | 97 | // Stats returns a copy of the current stats for this connector 98 | func (conn *ReplicatorConnector) Stats() ConnectorStats { 99 | return conn.stats.Stats() 100 | } 101 | 102 | // Init sets up common fields for all connectors 103 | func (conn *ReplicatorConnector) init(bridge *NATSReplicator, config conf.ConnectorConfig, name string) { 104 | conn.config = config 105 | conn.bridge = bridge 106 | 107 | id := conn.config.ID 108 | if id == "" { 109 | id = nuid.Next() 110 | } 111 | conn.stats = NewConnectorStatsHolder(name, id) 112 | } 113 | 114 | func createSubscriberOptions(config conf.ConnectorConfig) []stan.SubscriptionOption { 115 | 116 | options := []stan.SubscriptionOption{} 117 | 118 | if config.IncomingDurableName != "" { 119 | options = append(options, stan.DurableName(config.IncomingDurableName)) 120 | } 121 | 122 | if config.IncomingStartAtTime != 0 { 123 | t := time.Unix(config.IncomingStartAtTime, 0) 124 | options = append(options, stan.StartAtTime(t)) 125 | } else if config.IncomingStartAtSequence == -1 { 126 | options = append(options, stan.StartWithLastReceived()) 127 | } else if config.IncomingStartAtSequence > 0 { 128 | options = append(options, stan.StartAtSequence(uint64(config.IncomingStartAtSequence))) 129 | } else { 130 | options = append(options, stan.DeliverAllAvailable()) 131 | } 132 | 133 | if config.IncomingMaxInflight != 0 { 134 | options = append(options, stan.MaxInflight(int(config.IncomingMaxInflight))) 135 | } 136 | 137 | if config.IncomingAckWait != 0 { 138 | options = append(options, stan.AckWait(time.Duration(config.IncomingAckWait)*time.Millisecond)) 139 | } 140 | 141 | options = append(options, stan.SetManualAckMode()) 142 | 143 | return options 144 | } 145 | -------------------------------------------------------------------------------- /server/core/nats2nats.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "fmt" 20 | "time" 21 | 22 | "github.com/nats-io/nats-replicator/server/conf" 23 | nats "github.com/nats-io/nats.go" 24 | ) 25 | 26 | // NATS2NATSConnector connects a NATS subject to a different NATS subject 27 | type NATS2NATSConnector struct { 28 | ReplicatorConnector 29 | subscription *nats.Subscription 30 | } 31 | 32 | // NewNATS2NATSConnector create a new NATS to NATS connector 33 | func NewNATS2NATSConnector(bridge *NATSReplicator, config conf.ConnectorConfig) Connector { 34 | connector := &NATS2NATSConnector{} 35 | connector.init(bridge, config, fmt.Sprintf("NATS:%s to NATS:%s", config.IncomingSubject, config.OutgoingSubject)) 36 | return connector 37 | } 38 | 39 | // Start the connector 40 | func (conn *NATS2NATSConnector) Start() error { 41 | conn.Lock() 42 | defer conn.Unlock() 43 | 44 | config := conn.config 45 | incoming := config.IncomingConnection 46 | outgoing := config.OutgoingConnection 47 | 48 | if incoming == "" || outgoing == "" || config.IncomingSubject == "" || config.OutgoingSubject == "" { 49 | return fmt.Errorf("%s connector is improperly configured, incoming and outgoing settings are required", conn.String()) 50 | } 51 | 52 | if !conn.bridge.CheckNATS(incoming) { 53 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), incoming) 54 | } 55 | 56 | if !conn.bridge.CheckNATS(outgoing) { 57 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), outgoing) 58 | } 59 | 60 | conn.bridge.Logger().Tracef("starting connection %s", conn.String()) 61 | 62 | onc := conn.bridge.NATS(outgoing) 63 | if onc == nil { 64 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), outgoing) 65 | } 66 | 67 | traceEnabled := conn.bridge.Logger().TraceEnabled() 68 | callback := func(msg *nats.Msg) { 69 | start := time.Now() 70 | l := int64(len(msg.Data)) 71 | 72 | err := onc.PublishRequest(msg.Subject, msg.Reply, msg.Data) 73 | 74 | if err != nil { 75 | conn.stats.AddMessageIn(l) 76 | conn.bridge.Logger().Noticef("connector publish failure, %s, %s", conn.String(), err.Error()) 77 | } else { 78 | if traceEnabled { 79 | conn.bridge.Logger().Tracef(">>> [%s] s [%s], r [%s], size [%v]", conn.ID(), msg.Subject, msg.Reply, len(msg.Data)) 80 | } 81 | conn.stats.AddRequest(l, l, time.Since(start)) 82 | } 83 | } 84 | 85 | nc := conn.bridge.NATS(incoming) 86 | 87 | if nc == nil { 88 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), incoming) 89 | } 90 | 91 | var err error 92 | 93 | if config.IncomingQueueName == "" { 94 | conn.subscription, err = nc.Subscribe(config.IncomingSubject, callback) 95 | conn.bridge.Logger().Tracef("subscribe [%s], s [%s]", nc.Opts.Servers, config.IncomingSubject) 96 | } else { 97 | conn.subscription, err = nc.QueueSubscribe(config.IncomingSubject, config.IncomingQueueName, callback) 98 | } 99 | 100 | if err != nil { 101 | return err 102 | } 103 | 104 | conn.stats.AddConnect() 105 | conn.bridge.Logger().Tracef("opened and reading %s", conn.config.IncomingSubject) 106 | conn.bridge.Logger().Noticef("started connection %s", conn.String()) 107 | 108 | return nil 109 | } 110 | 111 | // Shutdown the connector 112 | func (conn *NATS2NATSConnector) Shutdown() error { 113 | conn.Lock() 114 | defer conn.Unlock() 115 | conn.stats.AddDisconnect() 116 | 117 | conn.bridge.Logger().Noticef("shutting down connection %s", conn.String()) 118 | 119 | sub := conn.subscription 120 | conn.subscription = nil 121 | 122 | if sub != nil { 123 | if err := sub.Unsubscribe(); err != nil { 124 | conn.bridge.Logger().Noticef("error unsubscribing for %s, %s", conn.String(), err.Error()) 125 | } 126 | } 127 | 128 | return nil // ignore the disconnect error 129 | } 130 | 131 | // CheckConnections ensures the nats/stan connection and report an error if it is down 132 | func (conn *NATS2NATSConnector) CheckConnections() error { 133 | config := conn.config 134 | incoming := config.IncomingConnection 135 | outgoing := config.OutgoingConnection 136 | if !conn.bridge.CheckNATS(incoming) { 137 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), incoming) 138 | } 139 | 140 | if !conn.bridge.CheckNATS(outgoing) { 141 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), outgoing) 142 | } 143 | return nil 144 | } 145 | -------------------------------------------------------------------------------- /server/core/nats2stan.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "fmt" 20 | "time" 21 | 22 | "github.com/nats-io/nats-replicator/server/conf" 23 | nats "github.com/nats-io/nats.go" 24 | ) 25 | 26 | // NATS2StanConnector connects NATS subject to a nats streaming channel 27 | type NATS2StanConnector struct { 28 | ReplicatorConnector 29 | 30 | subscription *nats.Subscription 31 | } 32 | 33 | // NewNATS2StanConnector create a new NATS to STAN connector 34 | func NewNATS2StanConnector(bridge *NATSReplicator, config conf.ConnectorConfig) Connector { 35 | connector := &NATS2StanConnector{} 36 | connector.init(bridge, config, fmt.Sprintf("NATS:%s to Stan:%s", config.IncomingSubject, config.OutgoingChannel)) 37 | return connector 38 | } 39 | 40 | // Start the connector 41 | func (conn *NATS2StanConnector) Start() error { 42 | conn.Lock() 43 | defer conn.Unlock() 44 | 45 | config := conn.config 46 | incoming := config.IncomingConnection 47 | outgoing := config.OutgoingConnection 48 | 49 | if incoming == "" || outgoing == "" || config.IncomingSubject == "" || config.OutgoingChannel == "" { 50 | return fmt.Errorf("%s connector is improperly configured, incoming and outgoing settings are required", conn.String()) 51 | } 52 | 53 | if !conn.bridge.CheckNATS(incoming) { 54 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), incoming) 55 | } 56 | 57 | if !conn.bridge.CheckStan(outgoing) { 58 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), outgoing) 59 | } 60 | 61 | conn.bridge.Logger().Tracef("starting connection %s", conn.String()) 62 | 63 | osc := conn.bridge.Stan(outgoing) 64 | if osc == nil { 65 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), outgoing) 66 | } 67 | 68 | traceEnabled := conn.bridge.Logger().TraceEnabled() 69 | callback := func(msg *nats.Msg) { 70 | start := time.Now() 71 | l := int64(len(msg.Data)) 72 | 73 | _, err := osc.PublishAsync(config.OutgoingChannel, msg.Data, func(ackguid string, err error) { 74 | // Handle the error on the ack handler after we cleaned up the outstanding acks map 75 | if err != nil { 76 | conn.stats.AddMessageIn(l) 77 | conn.bridge.ConnectorError(conn, err) 78 | return 79 | } 80 | 81 | if traceEnabled { 82 | conn.bridge.Logger().Tracef("%s wrote message to stan", conn.String()) 83 | } 84 | 85 | conn.stats.AddRequest(l, l, time.Since(start)) 86 | }) 87 | 88 | if err != nil { 89 | conn.stats.AddMessageIn(l) 90 | conn.bridge.Logger().Noticef("connector publish failure, %s, %s", conn.String(), err.Error()) 91 | } 92 | } 93 | 94 | nc := conn.bridge.NATS(incoming) 95 | 96 | if nc == nil { 97 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), incoming) 98 | } 99 | 100 | var err error 101 | 102 | if config.IncomingQueueName == "" { 103 | conn.subscription, err = nc.Subscribe(config.IncomingSubject, callback) 104 | } else { 105 | conn.subscription, err = nc.QueueSubscribe(config.IncomingSubject, config.IncomingQueueName, callback) 106 | } 107 | 108 | if err != nil { 109 | return err 110 | } 111 | 112 | conn.stats.AddConnect() 113 | conn.bridge.Logger().Tracef("opened and reading %s", conn.config.IncomingSubject) 114 | conn.bridge.Logger().Noticef("started connection %s", conn.String()) 115 | 116 | return nil 117 | } 118 | 119 | // Shutdown the connector 120 | func (conn *NATS2StanConnector) Shutdown() error { 121 | conn.Lock() 122 | defer conn.Unlock() 123 | conn.stats.AddDisconnect() 124 | 125 | conn.bridge.Logger().Noticef("shutting down connection %s", conn.String()) 126 | 127 | sub := conn.subscription 128 | conn.subscription = nil 129 | 130 | if sub != nil { 131 | if err := sub.Unsubscribe(); err != nil { 132 | conn.bridge.Logger().Noticef("error unsubscribing for %s, %s", conn.String(), err.Error()) 133 | } 134 | } 135 | 136 | return nil // ignore the disconnect error 137 | } 138 | 139 | // CheckConnections ensures the nats/stan connection and report an error if it is down 140 | func (conn *NATS2StanConnector) CheckConnections() error { 141 | config := conn.config 142 | incoming := config.IncomingConnection 143 | outgoing := config.OutgoingConnection 144 | if !conn.bridge.CheckNATS(incoming) { 145 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), incoming) 146 | } 147 | 148 | if !conn.bridge.CheckStan(outgoing) { 149 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), outgoing) 150 | } 151 | return nil 152 | } 153 | -------------------------------------------------------------------------------- /server/core/stan2nats.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package core 18 | 19 | import ( 20 | "fmt" 21 | "time" 22 | 23 | "github.com/nats-io/nats-replicator/server/conf" 24 | stan "github.com/nats-io/stan.go" 25 | ) 26 | 27 | // Stan2NATSConnector connects a STAN channel to NATS 28 | type Stan2NATSConnector struct { 29 | ReplicatorConnector 30 | sub stan.Subscription 31 | } 32 | 33 | // NewStan2NATSConnector create a new stan to a nats subject 34 | func NewStan2NATSConnector(bridge *NATSReplicator, config conf.ConnectorConfig) Connector { 35 | connector := &Stan2NATSConnector{} 36 | connector.init(bridge, config, fmt.Sprintf("Stan:%s to NATS:%s", config.IncomingChannel, config.OutgoingSubject)) 37 | return connector 38 | } 39 | 40 | // Start the connector 41 | func (conn *Stan2NATSConnector) Start() error { 42 | conn.Lock() 43 | defer conn.Unlock() 44 | 45 | config := conn.config 46 | incoming := config.IncomingConnection 47 | outgoing := config.OutgoingConnection 48 | 49 | if incoming == "" || outgoing == "" || config.IncomingChannel == "" || config.OutgoingSubject == "" { 50 | return fmt.Errorf("%s connector is improperly configured, incoming and outgoing settings are required", conn.String()) 51 | } 52 | 53 | if !conn.bridge.CheckStan(incoming) { 54 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), incoming) 55 | } 56 | 57 | if !conn.bridge.CheckNATS(outgoing) { 58 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), outgoing) 59 | } 60 | 61 | conn.bridge.Logger().Tracef("starting connection %s", conn.String()) 62 | 63 | options := createSubscriberOptions(config) 64 | traceEnabled := conn.bridge.Logger().TraceEnabled() 65 | 66 | onc := conn.bridge.NATS(outgoing) 67 | if onc == nil { 68 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), outgoing) 69 | } 70 | 71 | callback := func(msg *stan.Msg) { 72 | start := time.Now() 73 | l := int64(len(msg.Data)) 74 | 75 | if traceEnabled { 76 | conn.bridge.Logger().Tracef("%s received message", conn.String()) 77 | } 78 | 79 | err := onc.Publish(config.OutgoingSubject, msg.Data) 80 | 81 | if err != nil { 82 | conn.stats.AddMessageIn(l) 83 | conn.bridge.Logger().Noticef("connector publish failure, %s, %s", conn.String(), err.Error()) 84 | } else { 85 | if traceEnabled { 86 | conn.bridge.Logger().Tracef("%s wrote message to nats", conn.String()) 87 | } 88 | msg.Ack() 89 | if traceEnabled { 90 | conn.bridge.Logger().Tracef("%s acked message", conn.String()) 91 | } 92 | conn.stats.AddRequest(l, l, time.Since(start)) 93 | } 94 | } 95 | 96 | sc := conn.bridge.Stan(incoming) 97 | 98 | if sc == nil { 99 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), incoming) 100 | } 101 | 102 | sub, err := sc.Subscribe(conn.config.IncomingChannel, callback, options...) 103 | if err != nil { 104 | return err 105 | } 106 | 107 | conn.sub = sub 108 | 109 | conn.stats.AddConnect() 110 | if config.IncomingDurableName != "" { 111 | conn.bridge.Logger().Tracef("opened and reading %s with durable name %s", conn.config.IncomingChannel, config.IncomingDurableName) 112 | } else { 113 | conn.bridge.Logger().Tracef("opened and reading %s", conn.config.IncomingChannel) 114 | } 115 | conn.bridge.Logger().Noticef("started connection %s", conn.String()) 116 | 117 | return nil 118 | } 119 | 120 | // Shutdown the connector 121 | func (conn *Stan2NATSConnector) Shutdown() error { 122 | conn.Lock() 123 | defer conn.Unlock() 124 | conn.stats.AddDisconnect() 125 | 126 | conn.bridge.Logger().Noticef("shutting down connection %s", conn.String()) 127 | 128 | sub := conn.sub 129 | conn.sub = nil 130 | 131 | if sub != nil { 132 | if err := sub.Close(); err != nil { 133 | conn.bridge.Logger().Noticef("error closing for %s, %s", conn.String(), err.Error()) 134 | } 135 | } 136 | 137 | return nil // ignore the disconnect error 138 | } 139 | 140 | // CheckConnections ensures the nats/stan connection and report an error if it is down 141 | func (conn *Stan2NATSConnector) CheckConnections() error { 142 | config := conn.config 143 | incoming := config.IncomingConnection 144 | outgoing := config.OutgoingConnection 145 | if !conn.bridge.CheckStan(incoming) { 146 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), incoming) 147 | } 148 | 149 | if !conn.bridge.CheckNATS(outgoing) { 150 | return fmt.Errorf("%s connector requires nats connection named %s to be available", conn.String(), outgoing) 151 | } 152 | return nil 153 | } 154 | -------------------------------------------------------------------------------- /server/core/stan2stan.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "fmt" 20 | "time" 21 | 22 | "github.com/nats-io/nats-replicator/server/conf" 23 | stan "github.com/nats-io/stan.go" 24 | ) 25 | 26 | // Stan2StanConnector connects a streaming channel to another streaming channel 27 | type Stan2StanConnector struct { 28 | ReplicatorConnector 29 | sub stan.Subscription 30 | } 31 | 32 | // NewStan2StanConnector create a nats to MQ connector 33 | func NewStan2StanConnector(bridge *NATSReplicator, config conf.ConnectorConfig) Connector { 34 | connector := &Stan2StanConnector{} 35 | connector.init(bridge, config, fmt.Sprintf("Stan:%s to Stan:%s", config.IncomingChannel, config.OutgoingChannel)) 36 | return connector 37 | } 38 | 39 | // Start the connector 40 | func (conn *Stan2StanConnector) Start() error { 41 | conn.Lock() 42 | defer conn.Unlock() 43 | 44 | config := conn.config 45 | incoming := config.IncomingConnection 46 | outgoing := config.OutgoingConnection 47 | 48 | if incoming == "" || outgoing == "" || config.IncomingChannel == "" || config.OutgoingChannel == "" { 49 | return fmt.Errorf("%s connector is improperly configured, incoming and outgoing settings are required", conn.String()) 50 | } 51 | 52 | if !conn.bridge.CheckStan(incoming) { 53 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), incoming) 54 | } 55 | 56 | if !conn.bridge.CheckStan(outgoing) { 57 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), outgoing) 58 | } 59 | 60 | conn.bridge.Logger().Tracef("starting connection %s", conn.String()) 61 | 62 | options := createSubscriberOptions(config) 63 | traceEnabled := conn.bridge.Logger().TraceEnabled() 64 | 65 | osc := conn.bridge.Stan(outgoing) 66 | if osc == nil { 67 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), outgoing) 68 | } 69 | 70 | callback := func(msg *stan.Msg) { 71 | start := time.Now() 72 | 73 | if traceEnabled { 74 | conn.bridge.Logger().Tracef("%s received message", conn.String()) 75 | } 76 | 77 | _, err := osc.PublishAsync(config.OutgoingChannel, msg.Data, func(ackguid string, err error) { 78 | l := int64(len(msg.Data)) 79 | 80 | if err != nil { 81 | conn.stats.AddMessageIn(l) 82 | conn.bridge.ConnectorError(conn, err) 83 | return 84 | } 85 | 86 | if traceEnabled { 87 | conn.bridge.Logger().Tracef("%s wrote message to stan", conn.String()) 88 | } 89 | 90 | if err := msg.Ack(); err != nil { 91 | conn.stats.AddMessageIn(l) 92 | conn.bridge.ConnectorError(conn, err) 93 | return 94 | } 95 | 96 | if traceEnabled { 97 | conn.bridge.Logger().Tracef("%s acked message", conn.String()) 98 | } 99 | 100 | conn.stats.AddRequest(l, l, time.Since(start)) 101 | }) 102 | 103 | // TODO(dlc) - Should we attempt to make sure message is resent before ack timeout from incoming? 104 | if err != nil { 105 | conn.stats.AddMessageIn(int64(len(msg.Data))) 106 | conn.bridge.ConnectorError(conn, err) 107 | return 108 | } 109 | } 110 | 111 | sc := conn.bridge.Stan(incoming) 112 | 113 | if sc == nil { 114 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), incoming) 115 | } 116 | 117 | sub, err := sc.Subscribe(conn.config.IncomingChannel, callback, options...) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | conn.sub = sub 123 | 124 | conn.stats.AddConnect() 125 | 126 | if config.IncomingDurableName != "" { 127 | conn.bridge.Logger().Tracef("opened and reading %s with durable name %s", conn.config.IncomingChannel, config.IncomingDurableName) 128 | } else { 129 | conn.bridge.Logger().Tracef("opened and reading %s", conn.config.IncomingChannel) 130 | } 131 | conn.bridge.Logger().Noticef("started connection %s", conn.String()) 132 | 133 | return nil 134 | } 135 | 136 | // Shutdown the connector 137 | func (conn *Stan2StanConnector) Shutdown() error { 138 | conn.Lock() 139 | defer conn.Unlock() 140 | conn.stats.AddDisconnect() 141 | 142 | conn.bridge.Logger().Noticef("shutting down connection %s", conn.String()) 143 | 144 | sub := conn.sub 145 | conn.sub = nil 146 | 147 | if sub != nil { 148 | if err := sub.Close(); err != nil { 149 | conn.bridge.Logger().Noticef("error closing for %s, %s", conn.String(), err.Error()) 150 | } 151 | } 152 | 153 | return nil // ignore the disconnect error 154 | } 155 | 156 | // CheckConnections ensures the nats/stan connection and report an error if it is down 157 | func (conn *Stan2StanConnector) CheckConnections() error { 158 | config := conn.config 159 | incoming := config.IncomingConnection 160 | outgoing := config.OutgoingConnection 161 | if !conn.bridge.CheckStan(incoming) { 162 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), incoming) 163 | } 164 | 165 | if !conn.bridge.CheckStan(outgoing) { 166 | return fmt.Errorf("%s connector requires stan connection named %s to be available", conn.String(), outgoing) 167 | } 168 | return nil 169 | } 170 | -------------------------------------------------------------------------------- /server/core/stats.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package core 18 | 19 | import ( 20 | "sync" 21 | "time" 22 | ) 23 | 24 | // BridgeStats wraps the current status of the bridge and all of its connectors 25 | type BridgeStats struct { 26 | StartTime int64 `json:"start_time"` 27 | ServerTime int64 `json:"current_time"` 28 | UpTime string `json:"uptime"` 29 | RequestCount int64 `json:"request_count"` 30 | Connections []ConnectorStats `json:"connectors"` 31 | HTTPRequests map[string]int64 `json:"http_requests"` 32 | } 33 | 34 | // ConnectorStats captures the statistics for a single connector 35 | // times are in nanoseconds, use a holder to get the protection 36 | // of a lock and to fill in the quantiles 37 | type ConnectorStats struct { 38 | Name string `json:"name"` 39 | ID string `json:"id"` 40 | Connected bool `json:"connected"` 41 | Connects int64 `json:"connects"` 42 | Disconnects int64 `json:"disconnects"` 43 | BytesIn int64 `json:"bytes_in"` 44 | BytesOut int64 `json:"bytes_out"` 45 | MessagesIn int64 `json:"msg_in"` 46 | MessagesOut int64 `json:"msg_out"` 47 | RequestCount int64 `json:"count"` 48 | MovingAverage float64 `json:"rma"` 49 | Quintile50 float64 `json:"q50"` 50 | Quintile75 float64 `json:"q75"` 51 | Quintile90 float64 `json:"q90"` 52 | Quintile95 float64 `json:"q95"` 53 | } 54 | 55 | // ConnectorStatsHolder provides a lock and histogram 56 | // for a connector to updated it's stats. The holder's 57 | // Stats() method should be used to get the current values. 58 | type ConnectorStatsHolder struct { 59 | sync.Mutex 60 | stats ConnectorStats 61 | histogram *Histogram 62 | } 63 | 64 | // NewConnectorStatsHolder creates an empty stats holder, and initializes the request time histogram 65 | func NewConnectorStatsHolder(name string, id string) *ConnectorStatsHolder { 66 | return &ConnectorStatsHolder{ 67 | histogram: NewHistogram(60), 68 | stats: ConnectorStats{ 69 | Name: name, 70 | ID: id, 71 | }, 72 | } 73 | } 74 | 75 | // Name returns the name the holder was created with 76 | func (stats *ConnectorStatsHolder) Name() string { 77 | return stats.stats.Name 78 | } 79 | 80 | // ID returns the ID the holder was created with 81 | func (stats *ConnectorStatsHolder) ID() string { 82 | return stats.stats.ID 83 | } 84 | 85 | // AddMessageIn updates the messages in and bytes in fields 86 | // locks/unlocks the stats 87 | func (stats *ConnectorStatsHolder) AddMessageIn(bytes int64) { 88 | stats.Lock() 89 | stats.stats.MessagesIn++ 90 | stats.stats.BytesIn += bytes 91 | stats.Unlock() 92 | } 93 | 94 | // AddMessageOut updates the messages out and bytes out fields 95 | // locks/unlocks the stats 96 | func (stats *ConnectorStatsHolder) AddMessageOut(bytes int64) { 97 | stats.Lock() 98 | stats.stats.MessagesOut++ 99 | stats.stats.BytesOut += bytes 100 | stats.Unlock() 101 | } 102 | 103 | // AddDisconnect updates the disconnects field 104 | // locks/unlocks the stats 105 | func (stats *ConnectorStatsHolder) AddDisconnect() { 106 | stats.Lock() 107 | stats.stats.Disconnects++ 108 | stats.stats.Connected = false 109 | stats.Unlock() 110 | } 111 | 112 | // AddConnect updates the reconnects field 113 | // locks/unlocks the stats 114 | func (stats *ConnectorStatsHolder) AddConnect() { 115 | stats.Lock() 116 | stats.stats.Connects++ 117 | stats.stats.Connected = true 118 | stats.Unlock() 119 | } 120 | 121 | // AddRequestTime register a time, updating the request count, RMA and histogram 122 | // For information on the running moving average, see https://en.wikipedia.org/wiki/Moving_average 123 | // locks/unlocks the stats 124 | func (stats *ConnectorStatsHolder) AddRequestTime(reqTime time.Duration) { 125 | stats.Lock() 126 | reqns := float64(reqTime.Nanoseconds()) 127 | stats.stats.RequestCount++ 128 | stats.stats.MovingAverage = ((float64(stats.stats.RequestCount-1) * stats.stats.MovingAverage) + reqns) / float64(stats.stats.RequestCount) 129 | stats.histogram.Add(reqns) 130 | stats.Unlock() 131 | } 132 | 133 | // AddRequest groups addMessageIn, addMessageOut and addRequest time into a single call to reduce locking requirements. 134 | // locks/unlocks the stats 135 | func (stats *ConnectorStatsHolder) AddRequest(bytesIn int64, bytesOut int64, reqTime time.Duration) { 136 | stats.Lock() 137 | stats.stats.MessagesIn++ 138 | stats.stats.BytesIn += bytesIn 139 | stats.stats.MessagesOut++ 140 | stats.stats.BytesOut += bytesOut 141 | reqns := float64(reqTime.Nanoseconds()) 142 | stats.stats.RequestCount++ 143 | stats.stats.MovingAverage = ((float64(stats.stats.RequestCount-1) * stats.stats.MovingAverage) + reqns) / float64(stats.stats.RequestCount) 144 | stats.histogram.Add(reqns) 145 | stats.Unlock() 146 | } 147 | 148 | // Stats updates the quantiles and returns a copy of the stats 149 | // locks/unlocks the stats 150 | func (stats *ConnectorStatsHolder) Stats() ConnectorStats { 151 | stats.Lock() 152 | stats.stats.Quintile50 = stats.histogram.Quantile(0.5) 153 | stats.stats.Quintile75 = stats.histogram.Quantile(0.75) 154 | stats.stats.Quintile90 = stats.histogram.Quantile(0.9) 155 | stats.stats.Quintile95 = stats.histogram.Quantile(0.95) 156 | retVal := stats.stats 157 | stats.Unlock() 158 | return retVal 159 | } 160 | -------------------------------------------------------------------------------- /server/core/nats2nats_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "testing" 20 | "time" 21 | 22 | "github.com/nats-io/nats-replicator/server/conf" 23 | nats "github.com/nats-io/nats.go" 24 | "github.com/nats-io/nuid" 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestSimpleSendOnNATSReceiveOnNATS(t *testing.T) { 29 | incoming := nuid.Next() 30 | outgoing := nuid.Next() 31 | msg := "hello world" 32 | 33 | connect := []conf.ConnectorConfig{ 34 | { 35 | Type: "NATSToNATS", 36 | IncomingSubject: incoming, 37 | OutgoingSubject: outgoing, 38 | IncomingConnection: "nats", 39 | OutgoingConnection: "nats", 40 | }, 41 | } 42 | 43 | tbs, err := StartTestEnvironment(connect) 44 | require.NoError(t, err) 45 | defer tbs.Close() 46 | 47 | tbs.Bridge.checkConnections() 48 | 49 | done := make(chan string) 50 | sub, err := tbs.NC.Subscribe(outgoing, func(msg *nats.Msg) { 51 | done <- string(msg.Data) 52 | }) 53 | require.NoError(t, err) 54 | defer sub.Unsubscribe() 55 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 56 | 57 | err = tbs.NC.Publish(incoming, []byte(msg)) 58 | require.NoError(t, err) 59 | 60 | received := tbs.WaitForIt(1, done) 61 | require.Equal(t, msg, received) 62 | 63 | stats := tbs.Bridge.SafeStats() 64 | connStats := stats.Connections[0] 65 | require.Equal(t, int64(1), connStats.MessagesIn) 66 | require.Equal(t, int64(1), connStats.MessagesOut) 67 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesIn) 68 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesOut) 69 | require.Equal(t, int64(1), connStats.Connects) 70 | require.Equal(t, int64(0), connStats.Disconnects) 71 | require.True(t, connStats.Connected) 72 | } 73 | 74 | func TestWildcardSendOnNATSReceiveOnNATS(t *testing.T) { 75 | incoming := nuid.Next() + ".*" 76 | outgoing := nuid.Next() + "." + nuid.Next() 77 | msg := "hello world" 78 | 79 | connect := []conf.ConnectorConfig{ 80 | { 81 | Type: "NATSToNATS", 82 | IncomingSubject: incoming, 83 | OutgoingSubject: outgoing, 84 | IncomingConnection: "nats", 85 | OutgoingConnection: "nats", 86 | }, 87 | } 88 | 89 | tbs, err := StartTestEnvironment(connect) 90 | require.NoError(t, err) 91 | defer tbs.Close() 92 | 93 | tbs.Bridge.checkConnections() 94 | 95 | done := make(chan string) 96 | sub, err := tbs.NC.Subscribe(outgoing, func(msg *nats.Msg) { 97 | done <- string(msg.Data) 98 | }) 99 | require.NoError(t, err) 100 | defer sub.Unsubscribe() 101 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 102 | 103 | err = tbs.NC.Publish(incoming, []byte(msg)) 104 | require.NoError(t, err) 105 | 106 | received := tbs.WaitForIt(1, done) 107 | require.Equal(t, msg, received) 108 | 109 | stats := tbs.Bridge.SafeStats() 110 | connStats := stats.Connections[0] 111 | require.Equal(t, int64(1), connStats.MessagesIn) 112 | require.Equal(t, int64(1), connStats.MessagesOut) 113 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesIn) 114 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesOut) 115 | require.Equal(t, int64(1), connStats.Connects) 116 | require.Equal(t, int64(0), connStats.Disconnects) 117 | require.True(t, connStats.Connected) 118 | } 119 | 120 | func TestSimpleSendOnNATSReceiveOnNATSWithQueue(t *testing.T) { 121 | incoming := nuid.Next() 122 | outgoing := nuid.Next() 123 | group := nuid.Next() 124 | msg := "hello world" 125 | 126 | connect := []conf.ConnectorConfig{ 127 | { 128 | Type: "NATSToNATS", 129 | IncomingSubject: incoming, 130 | IncomingQueueName: group, 131 | IncomingConnection: "nats", 132 | OutgoingSubject: outgoing, 133 | OutgoingConnection: "nats", 134 | }, 135 | } 136 | 137 | tbs, err := StartTestEnvironment(connect) 138 | require.NoError(t, err) 139 | defer tbs.Close() 140 | 141 | done := make(chan string) 142 | sub, err := tbs.NC.Subscribe(outgoing, func(msg *nats.Msg) { 143 | done <- string(msg.Data) 144 | }) 145 | require.NoError(t, err) 146 | defer sub.Unsubscribe() 147 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 148 | 149 | err = tbs.NC.Publish(incoming, []byte(msg)) 150 | require.NoError(t, err) 151 | 152 | received := tbs.WaitForIt(1, done) 153 | require.Equal(t, msg, received) 154 | 155 | stats := tbs.Bridge.SafeStats() 156 | connStats := stats.Connections[0] 157 | require.Equal(t, int64(1), connStats.MessagesIn) 158 | require.Equal(t, int64(1), connStats.MessagesOut) 159 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesIn) 160 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesOut) 161 | require.Equal(t, int64(1), connStats.Connects) 162 | require.Equal(t, int64(0), connStats.Disconnects) 163 | require.True(t, connStats.Connected) 164 | } 165 | 166 | func TestSimpleSendOnQueueReceiveOnNatsWithTLS(t *testing.T) { 167 | incoming := nuid.Next() 168 | outgoing := nuid.Next() 169 | msg := "hello world" 170 | 171 | connect := []conf.ConnectorConfig{ 172 | { 173 | Type: "natstonats", // test with different casing 174 | IncomingSubject: incoming, 175 | IncomingConnection: "nats", 176 | OutgoingConnection: "nats", 177 | OutgoingSubject: outgoing, 178 | }, 179 | } 180 | 181 | tbs, err := StartTLSTestEnvironment(connect) 182 | require.NoError(t, err) 183 | defer tbs.Close() 184 | 185 | done := make(chan string) 186 | sub, err := tbs.NC.Subscribe(outgoing, func(msg *nats.Msg) { 187 | done <- string(msg.Data) 188 | }) 189 | require.NoError(t, err) 190 | defer sub.Unsubscribe() 191 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 192 | 193 | err = tbs.NC.Publish(incoming, []byte(msg)) 194 | require.NoError(t, err) 195 | 196 | received := tbs.WaitForIt(1, done) 197 | require.Equal(t, msg, received) 198 | } 199 | -------------------------------------------------------------------------------- /resources/certs/server.pem: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: localhost 3 | localKeyID: 54 69 6D 65 20 31 35 35 36 35 38 35 36 31 31 38 32 33 4 | Key Attributes: 5 | -----BEGIN PRIVATE KEY----- 6 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCguXJs7uuxr6MR 7 | aUKgeGbNDBR9+cXVt0LgLNLvd7ngyPD9+q+KNKaLvVw70k+R5NUUx5nSjM6yqyQm 8 | rJCbugcC3PnaOUa44Bc6szUeQrs1EEo/jDnx4IsRWM72XKei1tOl+qGGqaYcFt3s 9 | EUUdrdITjwHaqE8QGfOSodm0IxYWu4GFwFca4EH8nA0AO5khc/lHg4ZBXBdXqS6n 10 | aTa1dBZL9BPYAoUISZ2XBmztL6pHVgY20ZRrJ87dtfPiRSvaATIf4h952kikwW8R 11 | qmE6cvc6t81CZHvCHYDp5YjdNDK0Ik3KiP7b++s2CPmiHLPYEN78Tb5s0fytpxHg 12 | FuPb/OIHAgMBAAECggEANelen3mY57YxbaKwLaGVYgw+R29j0+cv8IA4lQjE+ciN 13 | RbQz85jrkI4JBSvrctWeP+UgDMccgkPz0WEq9IF23pf+2xtBRLwuLot0Mt1RbGA1 14 | d5Fy0lZub3gpm+vCc1W6Er+6NLBtPpjRAeFdHLJ0eAkNJyozswPxdusBWnI2c5MT 15 | g6iW3Cnl5T4VW/yYSM5ktJSfCBqtlPdrCjWzaB3g+VKXiI0hhtL5DUr/Jjx5GUKb 16 | h3UN3Us6QUHNT1oEQMNaeYQdHQxEtz7VPAeUmLqDlM4/Bo1Bd81Olb9uy3xIt1u+ 17 | 0Ns8IOtbtP81Qn+VRpLHNI/AKoVL/K28IM/I1IPe6QKBgQDVQ9F7T//1MoSDHbji 18 | FIeEBJc0yghDq3VmYSIir5jHXnh5Ay263nnVIvg8Y/f5/Zlgr6bJeR3EvWuvfI4A 19 | H6LrW5MAeB3InNygNKeJFM1lOdvo0sx4mETIZpEBGmYivFWZNhUrS8EhYfLrKBdp 20 | hE5d31yn0K+t8GuZpK9MIbRMowKBgQDA7mBZJWMBvrzrRi8NNbibYIKbLD3VGPhP 21 | M7fMUxLhATcS0tIKXV8am7gOKvbg1zVLhS4/aBxIDO9SaqNADT3CgxCc9g00iiQJ 22 | avVl9e82ROSmz55BCNudrnPogvbRnHiopeWIK2X/PdmvGsNwPh9AJrlwNMaqD/nQ 23 | uv4FHucnTQKBgE4tRiVF9jYUBq9pvdRiYirq4+LeDJmByM922+SLKh4ra04w9BqE 24 | Y3TWFNlObCCf7hGbUjCYzWjJZyg1KHizIC3Wq9SIM6LOnbG8m42Mqp5oz0xRudKb 25 | PtuXfaBB5R7mmxpG0QvGAU4TcdDyKWLstS2EK5r4zO2eLFNuIzRtRoKxAoGAd758 26 | BlhyDs83qj8xTN2e6rEH3w+igPSyiVXsKeEVwplieUhoHQ6+zGEB56k6+WoZJfpP 27 | LgOMKhv9HgYZtNODFYsLcKA2qfljTIUaMmJmSiSQVghejLbWuBNi1VkToB2hterh 28 | f5aQA897oHbX/n5QHxzp036uHzczMh4dM0hu57ECgYEAktJYUJFW+nxRbDguauoa 29 | dIh9NbKFqpcBhPtcKsEKBtcuNxuFmlMZHEBwCE66mqpnjDVByU3qi6nu2L1ZoVQ4 30 | QKWxjzkgnWJAhnSQO5QQLHeUYqBWJ2XMGLyoupgDFaahQxS2/+6Zu6/6XsorOAKj 31 | iNobI1GsQun0U1CqgHnzwCw= 32 | -----END PRIVATE KEY----- 33 | Bag Attributes 34 | friendlyName: localhost 35 | localKeyID: 54 69 6D 65 20 31 35 35 36 35 38 35 36 31 31 38 32 33 36 | subject=/C=US/ST=California/L=LosAngeles/O=None/OU=None/CN=localhost 37 | issuer=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 38 | -----BEGIN CERTIFICATE----- 39 | MIIDSjCCAjICCQCX1l7dC2mOKjANBgkqhkiG9w0BAQUFADBlMQswCQYDVQQGEwJV 40 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 41 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 42 | NDMwMDA1MzMwWhcNMjEwNDI5MDA1MzMwWjBpMQswCQYDVQQGEwJVUzETMBEGA1UE 43 | CBMKQ2FsaWZvcm5pYTETMBEGA1UEBxMKTG9zQW5nZWxlczENMAsGA1UEChMETm9u 44 | ZTENMAsGA1UECxMETm9uZTESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG 45 | 9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoLlybO7rsa+jEWlCoHhmzQwUffnF1bdC4CzS 46 | 73e54Mjw/fqvijSmi71cO9JPkeTVFMeZ0ozOsqskJqyQm7oHAtz52jlGuOAXOrM1 47 | HkK7NRBKP4w58eCLEVjO9lynotbTpfqhhqmmHBbd7BFFHa3SE48B2qhPEBnzkqHZ 48 | tCMWFruBhcBXGuBB/JwNADuZIXP5R4OGQVwXV6kup2k2tXQWS/QT2AKFCEmdlwZs 49 | 7S+qR1YGNtGUayfO3bXz4kUr2gEyH+IfedpIpMFvEaphOnL3OrfNQmR7wh2A6eWI 50 | 3TQytCJNyoj+2/vrNgj5ohyz2BDe/E2+bNH8racR4Bbj2/ziBwIDAQABMA0GCSqG 51 | SIb3DQEBBQUAA4IBAQBQN5Yt2iEIxjKBP+Xs7iA+tAV6YrCF4xlE1j9ufo+FaelN 52 | FzlNda7ZFDzPsiipGIU8wi3fAEOWVBgDKpVKihY7AXaEfjgxCg1NwSiDyzyljGCR 53 | /I9eE1wqWv4fMu5AmpaaFu763kL4L9SeMvu7J/tHYt2vxFF5f+ZSpu+ulp85Q+aD 54 | dyK4WpTT0dQKMNFGt000+C7t3GuinAt0yN9U/GpbBXSQAzd9+cA78PgmXTdalGvs 55 | fgHc3rHvVLInC+z/DGpnaeH2NDz+zBRTCB0PRHDghWbG8iXkzJcEXo6ZTMQ6wwVU 56 | PlgVgE7ZiCXXhD7kowoJQ5HZTuA4SR0pr5mSi3Au 57 | -----END CERTIFICATE----- 58 | Bag Attributes 59 | friendlyName: CN=nat.kafka.ssl,OU=None,O=None,L=LosAngeles,ST=CA,C=US 60 | subject=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 61 | issuer=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 62 | -----BEGIN CERTIFICATE----- 63 | MIIDRjCCAi4CCQCZ2pXVwpC9gzANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV 64 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 65 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 66 | NDMwMDA1MzI4WhcNMjEwNDI5MDA1MzI4WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE 67 | CAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5vbmUxDTALBgNV 68 | BAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwggEiMA0GCSqGSIb3DQEB 69 | AQUAA4IBDwAwggEKAoIBAQCvFAzhfXlw7eOmPTsCD1KISIOTBc9A7QvSspm3I2ka 70 | YKbsJgPJGuDVDosj0j74m77r56vXmNkCD5NNM3HuNxE5nAtaq848P+hbpxBgDp3n 71 | ZLO0BW+CSkfM1V6Qg4wp8jTAa6KMsK21ih1tKzS1VGnUJ7UcJhyU57joMmsiYl3E 72 | XHBnx0WKKgGQKdrYPv600Qdy77WoFWzawHqrroZnpH8Eb/YXfQEw7Hc+kedl0b5Y 73 | Rag6CEaB0B9nii321juhaAoMTpp8/hSca4zO3a9Jwz1KLlCFXjlHdyw4kv4uNGQk 74 | vIGwYFPJLbGi5SAaJwj9yS8v1e8TzK7YtJRsPUJYB+0nAgMBAAEwDQYJKoZIhvcN 75 | AQELBQADggEBADXRM+LVI+sXR64+8kKpDoFkR98RgIaXgMVN52M9O2fwU1tO5EDH 76 | MnqqRc5zkTjmLM6RDhGQnwTl+d+EcrFqteFU7qgj0EHBUE07rVgtinrJD/0INFzZ 77 | 6vMe+ZJVOJ6V6SjJBXQM0q2ASENH1zZCYfQgZPCBYdRhq3YzvgkCyt6OxIWE8APW 78 | Jlmmw4o/N5hbGwrDfTA3CyJU3Xy1zAsNSjRSQH1sHUHvVALZz9fHPamgpyFRiTmG 79 | 7qypC5FLAMo0mI8569I/ns1qIqk68aHJtynrxRIoPWc2eFK0J8v0oJ2LmaoK3NSa 80 | bByzkeofOaoaR2EzAKJoc3mkkX1+eDKmU34= 81 | -----END CERTIFICATE----- 82 | Bag Attributes 83 | friendlyName: caroot 84 | 2.16.840.1.113894.746875.1.1: 85 | subject=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 86 | issuer=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 87 | -----BEGIN CERTIFICATE----- 88 | MIIDRjCCAi4CCQCZ2pXVwpC9gzANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV 89 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 90 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 91 | NDMwMDA1MzI4WhcNMjEwNDI5MDA1MzI4WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE 92 | CAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5vbmUxDTALBgNV 93 | BAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwggEiMA0GCSqGSIb3DQEB 94 | AQUAA4IBDwAwggEKAoIBAQCvFAzhfXlw7eOmPTsCD1KISIOTBc9A7QvSspm3I2ka 95 | YKbsJgPJGuDVDosj0j74m77r56vXmNkCD5NNM3HuNxE5nAtaq848P+hbpxBgDp3n 96 | ZLO0BW+CSkfM1V6Qg4wp8jTAa6KMsK21ih1tKzS1VGnUJ7UcJhyU57joMmsiYl3E 97 | XHBnx0WKKgGQKdrYPv600Qdy77WoFWzawHqrroZnpH8Eb/YXfQEw7Hc+kedl0b5Y 98 | Rag6CEaB0B9nii321juhaAoMTpp8/hSca4zO3a9Jwz1KLlCFXjlHdyw4kv4uNGQk 99 | vIGwYFPJLbGi5SAaJwj9yS8v1e8TzK7YtJRsPUJYB+0nAgMBAAEwDQYJKoZIhvcN 100 | AQELBQADggEBADXRM+LVI+sXR64+8kKpDoFkR98RgIaXgMVN52M9O2fwU1tO5EDH 101 | MnqqRc5zkTjmLM6RDhGQnwTl+d+EcrFqteFU7qgj0EHBUE07rVgtinrJD/0INFzZ 102 | 6vMe+ZJVOJ6V6SjJBXQM0q2ASENH1zZCYfQgZPCBYdRhq3YzvgkCyt6OxIWE8APW 103 | Jlmmw4o/N5hbGwrDfTA3CyJU3Xy1zAsNSjRSQH1sHUHvVALZz9fHPamgpyFRiTmG 104 | 7qypC5FLAMo0mI8569I/ns1qIqk68aHJtynrxRIoPWc2eFK0J8v0oJ2LmaoK3NSa 105 | bByzkeofOaoaR2EzAKJoc3mkkX1+eDKmU34= 106 | -----END CERTIFICATE----- 107 | -------------------------------------------------------------------------------- /resources/certs/client.pem: -------------------------------------------------------------------------------- 1 | Bag Attributes 2 | friendlyName: localhost 3 | localKeyID: 54 69 6D 65 20 31 35 35 36 35 38 35 36 31 35 32 39 34 4 | Key Attributes: 5 | -----BEGIN PRIVATE KEY----- 6 | MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCBp8QZ+fliamqx 7 | iTi9QTewMFmwKZauVWMuAgu2/zLlYXuikSn6qWbCj+jeq1TGYV/UtJNt3h8TOgsg 8 | +ubw9mlksNCjwfNJZFMYMg8YkR9U3GE3o6FpaRZgLBrB4C9qmceJhAWSwZx/4RpS 9 | O0IZGumueBXwG4VAMX8qFa/Z+4rFEMEOo2FEXx0ODYDj+TF6VimaYGMc24jBYwYe 10 | bXl7N/gCUoYuasYeEJxaLybsJmjmvCFKTteR96U2ab7dTimQJGfnr0UcBjATva6V 11 | dMgQvycLfPD2quH46DEcthnQOg4KDnU+8vsno7qgkC+GzcY2DJ+PkKfRGg8SrVZv 12 | jfxOQ3zRAgMBAAECggEAGTrdRTTIm6cbpfqO6P0U6hsuiMI/CSOijiRlxTF55PsI 13 | RQU8hwOJ9Jhud04dzBDMxiZxffK4V1R3L+7DG+bHcYmawBMQ1ZpJNS8gkuZCk4/i 14 | 9yHHWizR5tsOReXrNDJCivb+4qT8YEIcjh3r6di72nCRzEx+rJQ1K6pWsvNo+JnV 15 | gZpQMSpIid0WpQsII2Gq+0ScPrw7RVWweoduqYgjfr1rM9taZZSzyOcQww34e1P0 16 | WDfjcIZkOdSGXBuxe0mRnuK4q6uE8dVMDlig5FaS6noNSePVerRCAn9o3K0NeKpa 17 | Z7A1l+n4kX9vKzR91Dwp25pufhHEOFFw72xOaOUEcQKBgQDu4PFrVTYfeY7OVEDA 18 | PbuO10Ag4jyal/QuemipZzLRplCk7FH8R55YEZtVMMcDHAP2VVeOoOBte77h1FdJ 19 | 3mdctZ6Ai7Q7piVP7HR7cuClxPgvrA9VgxPsPDTVlbklfU/ZgXxZdw49K0Dn3Jl3 20 | Bi9cjfvwtTIsios6YLIAO/2ShQKBgQCK8r4Hcu85b9kJ//WF4mBiLPudJ+wxo2bR 21 | DVjq9SaXk9qoNAOV42RUB3s8V0peo79hfmLJWFAm1706H8l+0aeWN4nr66Gx/0Pm 22 | 3g/ztf0IMJlGa+MqDFO5B0JEm7PAjG1J0YZX+gW2Zd954F65cU4DjtBP64sEoqMo 23 | i+8Jt6wA3QKBgQC1vKVvh1C5+X1QNIFewJilP9YmLnj1FI5Nngtqdn0PS30nPDPT 24 | v5kUX7DRy15dWz8Ydi068eJM6Ux7+1S9elshCXwhSChDCVBx0e98zBVliMlZW4n/ 25 | AM5zeAqqRFKr1v6c+Apm9lD68KFcxVRsXWRDAjKfGvulL3JlY5OI4hs1EQKBgQCH 26 | 9onufhAKkyC9AUK+NMr9pmi72nHsDKmyTK5Ck4qk7iAbUXJkvDLTatKzM/No8jB2 27 | dRazUQB1UcwvUSV5PCwR+Ny0B0mdBFzsT0UqxF0KI4wIdc++uHtAZhL5Uaat9nuG 28 | rUkZU2U9myf3eY8XRQECyD+cxK6u5XpkVbGdP6ZG3QKBgQCYFaGrWc7uPOj0iGac 29 | qU3dh03WfCScv7jo5uiQCAIP2Jwkc3dtIiMR7s+RFZzdMoAsCGpqP5EwEWbgC5UB 30 | wB6zSWIaYwp/v21jqvsZ5+Q+9ZV3iY8f56MpzShgTOGkZM1J7I9rsQACapC8t/aX 31 | XDttH8MprVIwasd6gqTnngXUXQ== 32 | -----END PRIVATE KEY----- 33 | Bag Attributes 34 | friendlyName: localhost 35 | localKeyID: 54 69 6D 65 20 31 35 35 36 35 38 35 36 31 35 32 39 34 36 | subject=/C=US/ST=California/L=LosAngeles/O=None/OU=None/CN=nat.kafka.client.ssl 37 | issuer=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 38 | -----BEGIN CERTIFICATE----- 39 | MIIDVTCCAj0CCQCX1l7dC2mOKzANBgkqhkiG9w0BAQUFADBlMQswCQYDVQQGEwJV 40 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 41 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 42 | NDMwMDA1MzMzWhcNMjEwNDI5MDA1MzMzWjB0MQswCQYDVQQGEwJVUzETMBEGA1UE 43 | CBMKQ2FsaWZvcm5pYTETMBEGA1UEBxMKTG9zQW5nZWxlczENMAsGA1UEChMETm9u 44 | ZTENMAsGA1UECxMETm9uZTEdMBsGA1UEAxMUbmF0LmthZmthLmNsaWVudC5zc2ww 45 | ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCBp8QZ+fliamqxiTi9QTew 46 | MFmwKZauVWMuAgu2/zLlYXuikSn6qWbCj+jeq1TGYV/UtJNt3h8TOgsg+ubw9mlk 47 | sNCjwfNJZFMYMg8YkR9U3GE3o6FpaRZgLBrB4C9qmceJhAWSwZx/4RpSO0IZGumu 48 | eBXwG4VAMX8qFa/Z+4rFEMEOo2FEXx0ODYDj+TF6VimaYGMc24jBYwYebXl7N/gC 49 | UoYuasYeEJxaLybsJmjmvCFKTteR96U2ab7dTimQJGfnr0UcBjATva6VdMgQvycL 50 | fPD2quH46DEcthnQOg4KDnU+8vsno7qgkC+GzcY2DJ+PkKfRGg8SrVZvjfxOQ3zR 51 | AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAD/v1U5zKnXh3e0MwYcsMGCNUUoGdAVa 52 | OLJuER9yq2I7CpYEVUlh1kGRkPs8u3Ww1NrYmASB9feaAPJ+U72zttkdKlXht6W+ 53 | b5rbQBCNPVd7yYfhl4kelnvSP1jhCT82KCHWo/6g4ku42d1oKy+hPyBErWfuC2MV 54 | tbls0br3LLlRa8316mlCL4ek4AGXvb60cB5g5B/PrgBsGBqtqTf1Yc7i+DhUOSZU 55 | Wlm5Yk3Z+AaztxfYhHAlubhzHFppA9H8kKjnbN0kTzPzaZhnrrF7Ro25ViLvc0cW 56 | rPnMY0+vNSAUd0zRMymuA0PGRSeEqVFbni6EGKuHXXM+wPh38JThACY= 57 | -----END CERTIFICATE----- 58 | Bag Attributes 59 | friendlyName: CN=nat.kafka.ssl,OU=None,O=None,L=LosAngeles,ST=CA,C=US 60 | subject=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 61 | issuer=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 62 | -----BEGIN CERTIFICATE----- 63 | MIIDRjCCAi4CCQCZ2pXVwpC9gzANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV 64 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 65 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 66 | NDMwMDA1MzI4WhcNMjEwNDI5MDA1MzI4WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE 67 | CAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5vbmUxDTALBgNV 68 | BAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwggEiMA0GCSqGSIb3DQEB 69 | AQUAA4IBDwAwggEKAoIBAQCvFAzhfXlw7eOmPTsCD1KISIOTBc9A7QvSspm3I2ka 70 | YKbsJgPJGuDVDosj0j74m77r56vXmNkCD5NNM3HuNxE5nAtaq848P+hbpxBgDp3n 71 | ZLO0BW+CSkfM1V6Qg4wp8jTAa6KMsK21ih1tKzS1VGnUJ7UcJhyU57joMmsiYl3E 72 | XHBnx0WKKgGQKdrYPv600Qdy77WoFWzawHqrroZnpH8Eb/YXfQEw7Hc+kedl0b5Y 73 | Rag6CEaB0B9nii321juhaAoMTpp8/hSca4zO3a9Jwz1KLlCFXjlHdyw4kv4uNGQk 74 | vIGwYFPJLbGi5SAaJwj9yS8v1e8TzK7YtJRsPUJYB+0nAgMBAAEwDQYJKoZIhvcN 75 | AQELBQADggEBADXRM+LVI+sXR64+8kKpDoFkR98RgIaXgMVN52M9O2fwU1tO5EDH 76 | MnqqRc5zkTjmLM6RDhGQnwTl+d+EcrFqteFU7qgj0EHBUE07rVgtinrJD/0INFzZ 77 | 6vMe+ZJVOJ6V6SjJBXQM0q2ASENH1zZCYfQgZPCBYdRhq3YzvgkCyt6OxIWE8APW 78 | Jlmmw4o/N5hbGwrDfTA3CyJU3Xy1zAsNSjRSQH1sHUHvVALZz9fHPamgpyFRiTmG 79 | 7qypC5FLAMo0mI8569I/ns1qIqk68aHJtynrxRIoPWc2eFK0J8v0oJ2LmaoK3NSa 80 | bByzkeofOaoaR2EzAKJoc3mkkX1+eDKmU34= 81 | -----END CERTIFICATE----- 82 | Bag Attributes 83 | friendlyName: caroot 84 | 2.16.840.1.113894.746875.1.1: 85 | subject=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 86 | issuer=/C=US/ST=CA/L=LosAngeles/O=None/OU=None/CN=nat.kafka.ssl 87 | -----BEGIN CERTIFICATE----- 88 | MIIDRjCCAi4CCQCZ2pXVwpC9gzANBgkqhkiG9w0BAQsFADBlMQswCQYDVQQGEwJV 89 | UzELMAkGA1UECAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5v 90 | bmUxDTALBgNVBAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwHhcNMTkw 91 | NDMwMDA1MzI4WhcNMjEwNDI5MDA1MzI4WjBlMQswCQYDVQQGEwJVUzELMAkGA1UE 92 | CAwCQ0ExEzARBgNVBAcMCkxvc0FuZ2VsZXMxDTALBgNVBAoMBE5vbmUxDTALBgNV 93 | BAsMBE5vbmUxFjAUBgNVBAMMDW5hdC5rYWZrYS5zc2wwggEiMA0GCSqGSIb3DQEB 94 | AQUAA4IBDwAwggEKAoIBAQCvFAzhfXlw7eOmPTsCD1KISIOTBc9A7QvSspm3I2ka 95 | YKbsJgPJGuDVDosj0j74m77r56vXmNkCD5NNM3HuNxE5nAtaq848P+hbpxBgDp3n 96 | ZLO0BW+CSkfM1V6Qg4wp8jTAa6KMsK21ih1tKzS1VGnUJ7UcJhyU57joMmsiYl3E 97 | XHBnx0WKKgGQKdrYPv600Qdy77WoFWzawHqrroZnpH8Eb/YXfQEw7Hc+kedl0b5Y 98 | Rag6CEaB0B9nii321juhaAoMTpp8/hSca4zO3a9Jwz1KLlCFXjlHdyw4kv4uNGQk 99 | vIGwYFPJLbGi5SAaJwj9yS8v1e8TzK7YtJRsPUJYB+0nAgMBAAEwDQYJKoZIhvcN 100 | AQELBQADggEBADXRM+LVI+sXR64+8kKpDoFkR98RgIaXgMVN52M9O2fwU1tO5EDH 101 | MnqqRc5zkTjmLM6RDhGQnwTl+d+EcrFqteFU7qgj0EHBUE07rVgtinrJD/0INFzZ 102 | 6vMe+ZJVOJ6V6SjJBXQM0q2ASENH1zZCYfQgZPCBYdRhq3YzvgkCyt6OxIWE8APW 103 | Jlmmw4o/N5hbGwrDfTA3CyJU3Xy1zAsNSjRSQH1sHUHvVALZz9fHPamgpyFRiTmG 104 | 7qypC5FLAMo0mI8569I/ns1qIqk68aHJtynrxRIoPWc2eFK0J8v0oJ2LmaoK3NSa 105 | bByzkeofOaoaR2EzAKJoc3mkkX1+eDKmU34= 106 | -----END CERTIFICATE----- 107 | -------------------------------------------------------------------------------- /server/conf/conf.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package conf 18 | 19 | import ( 20 | "crypto/tls" 21 | "crypto/x509" 22 | "fmt" 23 | "io/ioutil" 24 | 25 | "github.com/nats-io/nats-replicator/server/logging" 26 | ) 27 | 28 | const ( 29 | // NATSToNATS specifies a connector from NATS to NATS 30 | NATSToNATS = "NATSToNATS" 31 | // NATSToStan specifies a connector from NATS to NATS streaming 32 | NATSToStan = "NATSToStan" 33 | // StanToNATS specifies a connector from NATS streaming to NATS 34 | StanToNATS = "StanToNATS" 35 | // StanToStan specifies a connector from NATS streaming to NATS Streaming 36 | StanToStan = "StanToStan" 37 | ) 38 | 39 | // NATSReplicatorConfig is the root structure for a bridge configuration file. 40 | // NATS and STAN connections are specified in a map, where the key is a name used by 41 | // the connector to reference a connection. 42 | type NATSReplicatorConfig struct { 43 | ReconnectInterval int `conf:"reconnect_interval"` // milliseconds 44 | 45 | Logging logging.Config 46 | NATS []NATSConfig 47 | STAN []NATSStreamingConfig 48 | Monitoring HTTPConfig 49 | Connect []ConnectorConfig 50 | } 51 | 52 | // TLSConf holds the configuration for a TLS connection/server 53 | type TLSConf struct { 54 | Key string 55 | Cert string 56 | Root string 57 | } 58 | 59 | // MakeTLSConfig creates a tls.Config from a TLSConf, setting up the key pairs and certs 60 | func (tlsConf *TLSConf) MakeTLSConfig() (*tls.Config, error) { 61 | if tlsConf.Cert == "" || tlsConf.Key == "" { 62 | return nil, nil 63 | } 64 | 65 | cert, err := tls.LoadX509KeyPair(tlsConf.Cert, tlsConf.Key) 66 | if err != nil { 67 | return nil, fmt.Errorf("error loading X509 certificate/key pair: %v", err) 68 | } 69 | 70 | cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) 71 | if err != nil { 72 | return nil, fmt.Errorf("error parsing certificate: %v", err) 73 | } 74 | 75 | config := tls.Config{ 76 | Certificates: []tls.Certificate{cert}, 77 | MinVersion: tls.VersionTLS12, 78 | ClientAuth: tls.NoClientCert, 79 | PreferServerCipherSuites: true, 80 | } 81 | 82 | if tlsConf.Root != "" { 83 | // Load CA cert 84 | caCert, err := ioutil.ReadFile(tlsConf.Root) 85 | if err != nil { 86 | return nil, err 87 | } 88 | caCertPool := x509.NewCertPool() 89 | caCertPool.AppendCertsFromPEM(caCert) 90 | config.RootCAs = caCertPool 91 | } 92 | 93 | return &config, nil 94 | } 95 | 96 | // HTTPConfig is used to specify the host/port/tls for an HTTP server 97 | type HTTPConfig struct { 98 | HTTPHost string `conf:"http_host"` 99 | HTTPPort int `conf:"http_port"` 100 | HTTPSPort int `conf:"https_port"` 101 | TLS TLSConf 102 | 103 | ReadTimeout int `conf:"read_timeout"` //milliseconds 104 | WriteTimeout int `conf:"write_timeout"` //milliseconds 105 | } 106 | 107 | // NATSConfig configuration for a NATS connection 108 | type NATSConfig struct { 109 | Name string 110 | Servers []string 111 | 112 | ConnectTimeout int `conf:"connect_timeout"` //milliseconds 113 | ReconnectWait int `conf:"reconnect_wait"` //milliseconds 114 | MaxReconnects int `conf:"max_reconnects"` 115 | NoRandom bool `conf:"no_random"` 116 | NoEcho bool `conf:"no_echo"` 117 | 118 | TLS TLSConf 119 | UserCredentials string `conf:"user_credentials"` 120 | } 121 | 122 | // NATSStreamingConfig configuration for a STAN connection 123 | type NATSStreamingConfig struct { 124 | Name string 125 | ClusterID string `conf:"cluster_id"` 126 | ClientID string `conf:"client_id"` 127 | 128 | PubAckWait int `conf:"pub_ack_wait"` //milliseconds 129 | DiscoverPrefix string `conf:"discovery_prefix"` 130 | MaxPubAcksInflight int `conf:"max_pubacks_inflight"` 131 | ConnectWait int `conf:"connect_wait"` // milliseconds 132 | 133 | PingInterval int `conf:"ping_interval"` // seconds 134 | MaxPings int `conf:"max_pings"` 135 | 136 | NATSConnection string `conf:"nats_connection"` //name of the nats connection for this streaming connection 137 | } 138 | 139 | // DefaultConfig generates a default configuration with 140 | // logging set to colors, time, debug and trace 141 | func DefaultConfig() NATSReplicatorConfig { 142 | return NATSReplicatorConfig{ 143 | ReconnectInterval: 5000, 144 | Logging: logging.Config{ 145 | Colors: true, 146 | Time: true, 147 | Debug: false, 148 | Trace: false, 149 | }, 150 | Monitoring: HTTPConfig{ 151 | ReadTimeout: 5000, 152 | WriteTimeout: 5000, 153 | }, 154 | } 155 | } 156 | 157 | // ConnectorConfig configuration for a single connection (of any type) 158 | // Properties are available for any type, but only the ones necessary for the 159 | // connector type are used 160 | type ConnectorConfig struct { 161 | ID string // user specified id for a connector, will be defaulted if none is provided 162 | Type string // Can be any of the type constants (NATSToStan, ...) 163 | 164 | IncomingConnection string `conf:"incoming_connection"` // Name of the incoming connection (of either type), can be the same as outgoingConnection 165 | OutgoingConnection string `conf:"outgoing_connection"` // Name of the outgoing connection (of either type), can be the same as incomingConnection 166 | 167 | IncomingChannel string `conf:"incoming_channel"` // Used for stan connections 168 | IncomingDurableName string `conf:"incoming_durable_name"` // Optional, used for stan connections 169 | IncomingStartAtSequence int64 `conf:"incoming_startat_sequence"` // Start position for stan connection, -1 means StartWithLastReceived, 0 means DeliverAllAvailable (default) 170 | IncomingStartAtTime int64 `conf:"incoming_startat_time"` // Start time, as Unix, time takes precedence over sequence 171 | IncomingMaxInflight int64 `conf:"incoming_max_in_flight"` // maximum message in flight to this connector's subscription in Streaming 172 | IncomingAckWait int64 `conf:"incoming_ack_wait"` // max wait time in Milliseconds for the incoming subscription 173 | 174 | IncomingSubject string `conf:"incoming_subject"` // Used for nats connections 175 | IncomingQueueName string `conf:"incoming_queue_name"` // Optional, used for nats connections 176 | 177 | OutgoingChannel string `conf:"outgoing_channel"` // Used for stan connections 178 | OutgoingSubject string `conf:"outgoing_subject"` // Used for nats connections 179 | } 180 | -------------------------------------------------------------------------------- /server/core/nats.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package core 18 | 19 | import ( 20 | "fmt" 21 | "strings" 22 | "time" 23 | 24 | nats "github.com/nats-io/nats.go" 25 | stan "github.com/nats-io/stan.go" 26 | ) 27 | 28 | func (server *NATSReplicator) natsError(nc *nats.Conn, sub *nats.Subscription, err error) { 29 | server.logger.Warnf("nats error %s", err.Error()) 30 | } 31 | 32 | func (server *NATSReplicator) natsDisconnected(nc *nats.Conn) { 33 | if !server.checkRunning() { 34 | return 35 | } 36 | server.logger.Warnf("nats disconnected") 37 | server.checkConnections() 38 | } 39 | 40 | func (server *NATSReplicator) natsReconnected(nc *nats.Conn) { 41 | server.logger.Warnf("nats reconnected") 42 | } 43 | 44 | func (server *NATSReplicator) natsClosed(nc *nats.Conn) { 45 | if server.checkRunning() { 46 | server.logger.Errorf("nats connection closed, shutting down bridge") 47 | go server.Stop() 48 | } 49 | } 50 | 51 | func (server *NATSReplicator) natsDiscoveredServers(nc *nats.Conn) { 52 | server.logger.Debugf("discovered servers: %v\n", nc.DiscoveredServers()) 53 | server.logger.Debugf("known servers: %v\n", nc.Servers()) 54 | } 55 | 56 | func (server *NATSReplicator) connectToNATS() error { 57 | server.natsLock.Lock() 58 | defer server.natsLock.Unlock() 59 | 60 | for _, config := range server.config.NATS { 61 | name := config.Name 62 | server.logger.Noticef("connecting to NATS with configuration %s", name) 63 | 64 | maxReconnects := nats.DefaultMaxReconnect 65 | reconnectWait := nats.DefaultReconnectWait 66 | connectTimeout := nats.DefaultTimeout 67 | 68 | if config.MaxReconnects > 0 { 69 | maxReconnects = config.MaxReconnects 70 | } 71 | 72 | if config.ReconnectWait > 0 { 73 | reconnectWait = time.Duration(config.ReconnectWait) * time.Millisecond 74 | } 75 | 76 | if config.ConnectTimeout > 0 { 77 | connectTimeout = time.Duration(config.ConnectTimeout) * time.Millisecond 78 | } 79 | 80 | options := []nats.Option{nats.MaxReconnects(maxReconnects), 81 | nats.ReconnectWait(reconnectWait), 82 | nats.Timeout(connectTimeout), 83 | nats.ErrorHandler(server.natsError), 84 | nats.DiscoveredServersHandler(server.natsDiscoveredServers), 85 | nats.DisconnectHandler(server.natsDisconnected), 86 | nats.ReconnectHandler(server.natsReconnected), 87 | nats.ClosedHandler(server.natsClosed), 88 | } 89 | 90 | if config.NoRandom { 91 | options = append(options, nats.DontRandomize()) 92 | } 93 | 94 | if config.NoEcho { 95 | options = append(options, nats.NoEcho()) 96 | } 97 | 98 | if config.TLS.Root != "" { 99 | options = append(options, nats.RootCAs(config.TLS.Root)) 100 | } 101 | 102 | if config.TLS.Cert != "" { 103 | options = append(options, nats.ClientCert(config.TLS.Cert, config.TLS.Key)) 104 | } 105 | 106 | if config.UserCredentials != "" { 107 | options = append(options, nats.UserCredentials(config.UserCredentials)) 108 | } 109 | 110 | nc, err := nats.Connect(strings.Join(config.Servers, ","), 111 | options..., 112 | ) 113 | 114 | if err != nil { 115 | return err 116 | } 117 | 118 | server.nats[name] = nc 119 | } 120 | return nil 121 | } 122 | 123 | // assumes the server lock is held by the caller 124 | func (server *NATSReplicator) connectToSTAN() error { 125 | server.natsLock.Lock() 126 | defer server.natsLock.Unlock() 127 | 128 | for _, config := range server.config.STAN { 129 | name := config.Name 130 | sc, ok := server.stan[name] 131 | 132 | if ok && sc != nil { 133 | continue // that one is already connected 134 | } 135 | 136 | if config.ClusterID == "" { 137 | server.logger.Noticef("skipping NATS streaming connection %s, not configured", name) 138 | continue 139 | } 140 | 141 | server.logger.Noticef("connecting to NATS streaming with configuration %s, cluster id is %s", name, config.ClusterID) 142 | 143 | nc, ok := server.nats[config.NATSConnection] 144 | 145 | if !ok || nc == nil { 146 | return fmt.Errorf("stan connection %s requires NATS connection %s", name, config.NATSConnection) 147 | } 148 | 149 | pubAckWait := stan.DefaultAckWait 150 | 151 | if config.PubAckWait != 0 { 152 | pubAckWait = time.Duration(config.PubAckWait) * time.Millisecond 153 | } 154 | 155 | maxPubInFlight := stan.DefaultMaxPubAcksInflight 156 | 157 | if config.MaxPubAcksInflight > 0 { 158 | maxPubInFlight = config.MaxPubAcksInflight 159 | } 160 | 161 | connectWait := stan.DefaultConnectWait 162 | 163 | if config.ConnectWait > 0 { 164 | connectWait = time.Duration(config.ConnectWait) * time.Millisecond 165 | } 166 | 167 | maxPings := stan.DefaultPingMaxOut 168 | 169 | if config.MaxPings > 0 { 170 | maxPings = config.MaxPings 171 | } 172 | 173 | pingInterval := stan.DefaultPingInterval 174 | 175 | if config.PingInterval > 0 { 176 | pingInterval = config.PingInterval 177 | } 178 | 179 | sc, err := stan.Connect(config.ClusterID, config.ClientID, 180 | stan.NatsConn(nc), 181 | stan.PubAckWait(pubAckWait), 182 | stan.MaxPubAcksInflight(maxPubInFlight), 183 | stan.ConnectWait(connectWait), 184 | stan.Pings(pingInterval, maxPings), 185 | stan.SetConnectionLostHandler(func(sc stan.Conn, err error) { 186 | if !server.checkRunning() { 187 | return 188 | } 189 | server.logger.Warnf("nats streaming %s disconnected", name) 190 | 191 | server.natsLock.Lock() 192 | sc.Close() 193 | delete(server.stan, name) 194 | server.natsLock.Unlock() 195 | 196 | server.checkConnections() 197 | }), 198 | func(o *stan.Options) error { 199 | if config.DiscoverPrefix != "" { 200 | o.DiscoverPrefix = config.DiscoverPrefix 201 | } else { 202 | o.DiscoverPrefix = "_STAN.discover" 203 | } 204 | return nil 205 | }) 206 | 207 | if err != nil { 208 | return err 209 | } 210 | 211 | server.stan[name] = sc 212 | } 213 | 214 | return nil 215 | } 216 | 217 | // NATS hosts a shared nats connection for the connectors 218 | func (server *NATSReplicator) NATS(name string) *nats.Conn { 219 | server.natsLock.RLock() 220 | nc, ok := server.nats[name] 221 | server.natsLock.RUnlock() 222 | if !ok { 223 | nc = nil 224 | } 225 | return nc 226 | } 227 | 228 | // Stan hosts a shared streaming connection for the connectors 229 | func (server *NATSReplicator) Stan(name string) stan.Conn { 230 | server.natsLock.RLock() 231 | sc, ok := server.stan[name] 232 | server.natsLock.RUnlock() 233 | 234 | if !ok { 235 | sc = nil 236 | } 237 | return sc 238 | } 239 | 240 | // CheckNATS returns true if the bridge is connected to nats 241 | func (server *NATSReplicator) CheckNATS(name string) bool { 242 | server.natsLock.RLock() 243 | defer server.natsLock.RUnlock() 244 | 245 | nc, ok := server.nats[name] 246 | 247 | if ok && nc != nil { 248 | return nc.ConnectedUrl() != "" 249 | } 250 | 251 | return false 252 | } 253 | 254 | // CheckStan returns true if the bridge is connected to stan 255 | func (server *NATSReplicator) CheckStan(name string) bool { 256 | server.natsLock.RLock() 257 | defer server.natsLock.RUnlock() 258 | 259 | sc, ok := server.stan[name] 260 | 261 | return ok && sc != nil 262 | } 263 | -------------------------------------------------------------------------------- /server/core/monitoring.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "context" 20 | "crypto/tls" 21 | "encoding/json" 22 | "fmt" 23 | "net" 24 | "net/http" 25 | "strconv" 26 | "strings" 27 | "time" 28 | ) 29 | 30 | // HTTP endpoints 31 | const ( 32 | RootPath = "/" 33 | VarzPath = "/varz" 34 | HealthzPath = "/healthz" 35 | ) 36 | 37 | // startMonitoring starts the HTTP or HTTPs server if needed. 38 | // expects the lock to be held 39 | func (server *NATSReplicator) startMonitoring() error { 40 | config := server.config.Monitoring 41 | 42 | if config.HTTPPort != 0 && config.HTTPSPort != 0 { 43 | return fmt.Errorf("can't specify both HTTP (%v) and HTTPs (%v) ports", config.HTTPPort, config.HTTPSPort) 44 | } 45 | 46 | if config.HTTPPort == 0 && config.HTTPSPort == 0 { 47 | server.logger.Noticef("monitoring is disabled") 48 | return nil 49 | } 50 | 51 | secure := false 52 | 53 | if config.HTTPSPort != 0 { 54 | if config.TLS.Cert == "" || config.TLS.Key == "" { 55 | return fmt.Errorf("TLS cert and key required for HTTPS") 56 | } 57 | secure = true 58 | } 59 | 60 | // Used to track HTTP requests 61 | server.httpReqStats = map[string]int64{ 62 | RootPath: 0, 63 | VarzPath: 0, 64 | HealthzPath: 0, 65 | } 66 | 67 | var ( 68 | hp string 69 | err error 70 | listener net.Listener 71 | port int 72 | cer tls.Certificate 73 | ) 74 | 75 | monitorProtocol := "http" 76 | 77 | if secure { 78 | monitorProtocol += "s" 79 | port = config.HTTPSPort 80 | if port == -1 { 81 | port = 0 82 | } 83 | hp = net.JoinHostPort(config.HTTPHost, strconv.Itoa(port)) 84 | 85 | cer, err = tls.LoadX509KeyPair(config.TLS.Cert, config.TLS.Key) 86 | if err != nil { 87 | return err 88 | } 89 | 90 | config := &tls.Config{Certificates: []tls.Certificate{cer}} 91 | config.ClientAuth = tls.NoClientCert 92 | listener, err = tls.Listen("tcp", hp, config) 93 | } else { 94 | port = config.HTTPPort 95 | if port == -1 { 96 | port = 0 97 | } 98 | hp = net.JoinHostPort(config.HTTPHost, strconv.Itoa(port)) 99 | listener, err = net.Listen("tcp", hp) 100 | } 101 | 102 | if err != nil { 103 | return fmt.Errorf("can't listen to the monitor port: %v", err) 104 | } 105 | 106 | server.logger.Noticef("starting %s monitor on %s", monitorProtocol, 107 | net.JoinHostPort(config.HTTPHost, strconv.Itoa(listener.Addr().(*net.TCPAddr).Port))) 108 | 109 | mhp := net.JoinHostPort(config.HTTPHost, strconv.Itoa(listener.Addr().(*net.TCPAddr).Port)) 110 | if config.HTTPHost == "" { 111 | mhp = "localhost" + mhp 112 | } 113 | server.monitoringURL = fmt.Sprintf("%s://%s/", monitorProtocol, mhp) 114 | 115 | mux := http.NewServeMux() 116 | 117 | mux.HandleFunc(RootPath, server.HandleRoot) 118 | mux.HandleFunc(VarzPath, server.HandleVarz) 119 | mux.HandleFunc(HealthzPath, server.HandleHealthz) 120 | 121 | // Do not set a WriteTimeout because it could cause cURL/browser 122 | // to return empty response or unable to display page if the 123 | // server needs more time to build the response. 124 | srv := &http.Server{ 125 | Addr: hp, 126 | Handler: mux, 127 | MaxHeaderBytes: 1 << 20, 128 | } 129 | 130 | server.listener = listener 131 | server.httpHandler = mux 132 | server.http = srv 133 | 134 | go func() { 135 | srv.Serve(listener) 136 | srv.Handler = nil 137 | }() 138 | 139 | return nil 140 | } 141 | 142 | // StopMonitoring shuts down the http server used for monitoring 143 | // expects the server lock to be held 144 | func (server *NATSReplicator) StopMonitoring() error { 145 | server.logger.Tracef("stopping monitoring") 146 | if server.http != nil && server.httpHandler != nil { 147 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) 148 | defer cancel() 149 | 150 | if err := server.http.Shutdown(ctx); err != nil { 151 | return err 152 | } 153 | 154 | server.http = nil 155 | server.httpHandler = nil 156 | } 157 | 158 | if server.listener != nil { 159 | server.listener.Close() // ignore the error 160 | server.listener = nil 161 | } 162 | server.logger.Noticef("http monitoring stopped") 163 | 164 | return nil 165 | } 166 | 167 | // HandleRoot will show basic info and links to others handlers. 168 | func (server *NATSReplicator) HandleRoot(w http.ResponseWriter, r *http.Request) { 169 | if r.URL.Path != "/" { 170 | http.NotFound(w, r) 171 | return 172 | } 173 | server.statsLock.Lock() 174 | server.httpReqStats[RootPath]++ 175 | server.statsLock.Unlock() 176 | fmt.Fprintf(w, ` 177 | 178 | 179 | 183 | 184 | 185 | NATS 186 |
187 | varz
188 | healthz
189 |
190 | 191 | `) 192 | } 193 | 194 | // HandleVarz returns statistics about the server. 195 | func (server *NATSReplicator) HandleVarz(w http.ResponseWriter, r *http.Request) { 196 | server.statsLock.Lock() 197 | server.httpReqStats[VarzPath]++ 198 | server.statsLock.Unlock() 199 | 200 | compact := strings.ToLower(r.URL.Query().Get("compact")) == "true" 201 | 202 | stats := server.stats() 203 | 204 | var err error 205 | var varzJSON []byte 206 | 207 | if compact { 208 | varzJSON, err = json.Marshal(stats) 209 | } else { 210 | varzJSON, err = json.MarshalIndent(stats, "", " ") 211 | } 212 | 213 | if err != nil { 214 | w.WriteHeader(http.StatusInternalServerError) 215 | return 216 | } 217 | 218 | w.Header().Set("Content-Type", "application/json") 219 | w.WriteHeader(http.StatusOK) 220 | w.Write(varzJSON) 221 | } 222 | 223 | // HandleHealthz returns status 200. 224 | func (server *NATSReplicator) HandleHealthz(w http.ResponseWriter, r *http.Request) { 225 | server.statsLock.Lock() 226 | server.httpReqStats[HealthzPath]++ 227 | server.statsLock.Unlock() 228 | w.WriteHeader(http.StatusOK) 229 | } 230 | 231 | // stats calculates the stats for the server and connectors 232 | // assumes that the running lock is held by the caller 233 | func (server *NATSReplicator) stats() BridgeStats { 234 | now := time.Now() 235 | 236 | stats := BridgeStats{} 237 | stats.StartTime = server.startTime.Unix() 238 | stats.UpTime = now.Sub(server.startTime).String() 239 | stats.ServerTime = now.Unix() 240 | 241 | for _, connector := range server.connectors { 242 | cstats := connector.Stats() 243 | stats.Connections = append(stats.Connections, cstats) 244 | stats.RequestCount += cstats.RequestCount 245 | } 246 | 247 | stats.HTTPRequests = map[string]int64{} 248 | 249 | server.statsLock.Lock() 250 | for k, v := range server.httpReqStats { 251 | stats.HTTPRequests[k] = int64(v) 252 | } 253 | server.statsLock.Unlock() 254 | 255 | return stats 256 | } 257 | 258 | // SafeStats grabs the lock then calls stats(), useful for tests 259 | func (server *NATSReplicator) SafeStats() BridgeStats { 260 | server.Lock() 261 | defer server.Unlock() 262 | return server.stats() 263 | } 264 | 265 | // GetMonitoringRootURL returns the protocol://host:port for the monitoring server, useful for testing 266 | func (server *NATSReplicator) GetMonitoringRootURL() string { 267 | server.Lock() 268 | defer server.Unlock() 269 | return server.monitoringURL 270 | } 271 | -------------------------------------------------------------------------------- /server/conf/parse.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package conf 18 | 19 | import ( 20 | "fmt" 21 | "reflect" 22 | "strconv" 23 | "strings" 24 | ) 25 | 26 | func parseBoolean(keyname string, v interface{}) (bool, error) { 27 | switch t := v.(type) { 28 | case bool: 29 | return bool(t), nil 30 | case string: 31 | sv := v.(string) 32 | return strings.ToLower(sv) == "true", nil 33 | default: 34 | return false, fmt.Errorf("error parsing %s option %v", keyname, v) 35 | } 36 | } 37 | 38 | func parseInt(keyname string, v interface{}) (int64, error) { 39 | i := 0 40 | var err error 41 | switch v := v.(type) { 42 | case int: 43 | i = int(v) 44 | case int8: 45 | i = int(v) 46 | case int16: 47 | i = int(v) 48 | case int32: 49 | i = int(v) 50 | case int64: 51 | i = int(v) 52 | case string: 53 | i, err = strconv.Atoi(string(v)) 54 | if err != nil { 55 | err = fmt.Errorf("unable to parse integer %v for key %s", v, keyname) 56 | } 57 | default: 58 | err = fmt.Errorf("unable to parse integer %v for key %s", v, keyname) 59 | } 60 | return int64(i), err 61 | } 62 | 63 | func parseFloat(keyname string, v interface{}) (float64, error) { 64 | var err error 65 | i := 0.0 66 | switch v := v.(type) { 67 | case float32: 68 | i = float64(v) 69 | case float64: 70 | i = float64(v) 71 | case string: 72 | i, err = strconv.ParseFloat(string(v), 64) 73 | if err != nil { 74 | err = fmt.Errorf("unable to parse float %v for key %s", v, keyname) 75 | } 76 | default: 77 | err = fmt.Errorf("unable to parse float %v for key %s", v, keyname) 78 | } 79 | return i, err 80 | } 81 | 82 | func parseString(keyName string, v interface{}) (string, error) { 83 | sv, ok := v.(string) 84 | if !ok { 85 | return "", fmt.Errorf("error parsing %s option %v", keyName, v) 86 | } 87 | return sv, nil 88 | } 89 | 90 | func parsePrimitiveArray(keyName string, t reflect.Type, v interface{}) (reflect.Value, error) { 91 | buf := reflect.MakeSlice(reflect.SliceOf(t), 0, 0) 92 | ia, iaok := v.([]interface{}) 93 | 94 | if iaok { 95 | for _, e := range ia { 96 | sv := reflect.ValueOf(e) 97 | 98 | if sv.Type().ConvertibleTo(t) { 99 | buf = reflect.Append(buf, sv.Convert(t)) 100 | } else { 101 | return buf, fmt.Errorf("error parsing %s option %v, contents are not convertable", keyName, v) 102 | } 103 | } 104 | return buf, nil 105 | } 106 | 107 | if reflect.TypeOf(v).ConvertibleTo(t) { 108 | sv := reflect.ValueOf(v) 109 | buf = reflect.Append(buf, sv.Convert(t)) 110 | return buf, nil 111 | } 112 | 113 | return buf, fmt.Errorf("error parsing %s option %v, single element is not convertable", keyName, v) 114 | } 115 | 116 | func parseStructs(keyName string, t reflect.Type, v interface{}, strict bool) (reflect.Value, error) { 117 | buf := reflect.MakeSlice(reflect.SliceOf(t), 0, 0) 118 | ia, iaok := v.([]interface{}) 119 | if iaok { 120 | for _, e := range ia { 121 | sv, sok := e.(map[string]interface{}) 122 | if sok { 123 | val := reflect.New(t) 124 | err := parseStruct(sv, val, strict) 125 | if err != nil { 126 | return buf, err 127 | } 128 | buf = reflect.Append(buf, val.Elem()) 129 | } else { 130 | return buf, fmt.Errorf("struct array contained invalid value") 131 | } 132 | } 133 | return buf, nil 134 | } 135 | 136 | sv, sok := v.(map[string]interface{}) 137 | if sok { 138 | val := reflect.New(t) 139 | err := parseStruct(sv, val, strict) 140 | if err != nil { 141 | return buf, err 142 | } 143 | buf = reflect.Append(buf, val.Elem()) 144 | return buf, nil 145 | } 146 | 147 | return buf, fmt.Errorf("error parsing %s option %v", keyName, v) 148 | } 149 | 150 | func get(data map[string]interface{}, key string, confTag string) interface{} { 151 | if len(key) == 0 || data == nil { 152 | return nil 153 | } 154 | 155 | if confTag != "" { 156 | val, ok := data[confTag] 157 | if ok { 158 | return val 159 | } 160 | } 161 | 162 | val, ok := data[key] 163 | 164 | if ok { 165 | return val 166 | } 167 | 168 | // try lower case 169 | key = strings.ToLower(key) 170 | 171 | val, ok = data[key] 172 | 173 | if ok { 174 | return val 175 | } 176 | 177 | // Worst case, lower case all the possibles 178 | for k, v := range data { 179 | if strings.ToLower(k) == key { 180 | return v 181 | } 182 | } 183 | 184 | return nil 185 | } 186 | 187 | func parseStruct(data map[string]interface{}, config interface{}, strict bool) error { 188 | var err error 189 | var fields reflect.Value 190 | var maybeFields reflect.Value 191 | 192 | mapStringInterfaceType := reflect.TypeOf(map[string]interface{}{}) 193 | 194 | // Get all the fields in the config struct 195 | if reflect.TypeOf(config).ConvertibleTo(reflect.TypeOf(reflect.Value{})) { 196 | maybeFields = config.(reflect.Value) 197 | 198 | } else { 199 | maybeFields = reflect.ValueOf(config) 200 | } 201 | 202 | fields = reflect.Indirect(maybeFields) 203 | dataType := fields.Type() 204 | 205 | // Loop over the fields in the struct 206 | for i := 0; i < fields.NumField(); i++ { 207 | field := fields.Field(i) 208 | // Skip what we can't write 209 | if !field.CanSet() { 210 | if strict { 211 | return fmt.Errorf("unsettable field in configuration struct %s", dataType.Field(i).Name) 212 | } 213 | continue 214 | } 215 | 216 | fieldType := dataType.Field(i) 217 | fieldName := fieldType.Name 218 | fieldTag := fieldType.Tag 219 | confTag := fieldTag.Get("conf") 220 | configVal := get(data, fieldName, confTag) 221 | 222 | if configVal == nil { 223 | if strict { 224 | return fmt.Errorf("missing field in configuration file %s", fieldName) 225 | } 226 | continue 227 | } 228 | 229 | switch field.Type().Kind() { 230 | case reflect.Bool: 231 | var v bool 232 | if configVal != nil { 233 | v, err = parseBoolean(fieldName, configVal) 234 | if err != nil { 235 | return err 236 | } 237 | field.SetBool(v) 238 | } 239 | case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8: 240 | var v int64 241 | if configVal != nil { 242 | v, err = parseInt(fieldName, configVal) 243 | if err != nil { 244 | return err 245 | } 246 | field.SetInt(v) 247 | } 248 | case reflect.Float64, reflect.Float32: 249 | var v float64 250 | if configVal != nil { 251 | v, err = parseFloat(fieldName, configVal) 252 | if err != nil { 253 | return err 254 | } 255 | field.SetFloat(v) 256 | } 257 | case reflect.String: 258 | var v string 259 | if configVal != nil { 260 | v, err = parseString(fieldName, configVal) 261 | if err != nil { 262 | return err 263 | } 264 | field.SetString(v) 265 | } 266 | case reflect.Map: 267 | configData, ok := configVal.(map[string]interface{}) 268 | if !ok { 269 | return fmt.Errorf("map field %s doesn't have a matching map in the config file", fieldName) 270 | } 271 | if !field.Type().AssignableTo(mapStringInterfaceType) { 272 | return fmt.Errorf("only map[string]interface{} fields are supported") 273 | } 274 | field.Set(reflect.ValueOf(configData)) 275 | case reflect.Array, reflect.Slice: 276 | switch fieldType.Type.Elem().Kind() { 277 | case reflect.String, reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8, reflect.Float64, reflect.Float32: 278 | theArray, err := parsePrimitiveArray(fieldName, fieldType.Type.Elem(), configVal) 279 | if err != nil { 280 | return err 281 | } 282 | field.Set(theArray) 283 | case reflect.Struct: 284 | var structs reflect.Value 285 | structs, err = parseStructs(fieldName, fieldType.Type.Elem(), configVal, strict) 286 | if err != nil { 287 | return err 288 | } 289 | 290 | field.Set(structs) 291 | default: 292 | if strict { 293 | return fmt.Errorf("unknown field type in configuration %s, bool, int, float, string and arrays/structs of those are supported", fieldName) 294 | } 295 | } 296 | case reflect.Struct: 297 | if configVal != nil { 298 | configData, ok := configVal.(map[string]interface{}) 299 | if !ok { 300 | return fmt.Errorf("struct field %s doesn't have a matching map in the config file", fieldName) 301 | } 302 | 303 | err = parseStruct(configData, field, strict) 304 | if err != nil { 305 | return err 306 | } 307 | } 308 | default: 309 | if strict { 310 | return fmt.Errorf("unknown field type in configuration %s, bool, int, float, string, hostport and arrays/structs of those are supported", fieldName) 311 | } 312 | } 313 | 314 | } 315 | 316 | return nil 317 | } 318 | -------------------------------------------------------------------------------- /server/core/stan2stan_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | package core 17 | 18 | import ( 19 | "testing" 20 | "time" 21 | 22 | "github.com/nats-io/nats-replicator/server/conf" 23 | "github.com/nats-io/nuid" 24 | stan "github.com/nats-io/stan.go" 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | func TestSimpleSendOnStanReceiveOnStan(t *testing.T) { 29 | incoming := nuid.Next() 30 | outgoing := nuid.Next() 31 | msg := "hello world" 32 | 33 | connect := []conf.ConnectorConfig{ 34 | { 35 | Type: "StanToStan", 36 | IncomingChannel: incoming, 37 | OutgoingChannel: outgoing, 38 | IncomingConnection: "stan", 39 | OutgoingConnection: "stan", 40 | }, 41 | } 42 | 43 | tbs, err := StartTestEnvironment(connect) 44 | require.NoError(t, err) 45 | defer tbs.Close() 46 | 47 | tbs.Bridge.checkConnections() 48 | 49 | done := make(chan string) 50 | sub, err := tbs.SC.Subscribe(outgoing, func(msg *stan.Msg) { 51 | done <- string(msg.Data) 52 | }) 53 | require.NoError(t, err) 54 | defer sub.Unsubscribe() 55 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 56 | 57 | err = tbs.SC.Publish(incoming, []byte(msg)) 58 | require.NoError(t, err) 59 | 60 | received := tbs.WaitForIt(1, done) 61 | require.Equal(t, msg, received) 62 | 63 | stats := tbs.Bridge.SafeStats() 64 | connStats := stats.Connections[0] 65 | require.Equal(t, int64(1), connStats.MessagesIn) 66 | require.Equal(t, int64(1), connStats.MessagesOut) 67 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesIn) 68 | require.Equal(t, int64(1), connStats.Connects) 69 | require.Equal(t, int64(0), connStats.Disconnects) 70 | require.True(t, connStats.Connected) 71 | } 72 | 73 | func TestQueueStartAtPositionToStan(t *testing.T) { 74 | incoming := nuid.Next() 75 | outgoing := nuid.Next() 76 | msg := "hello world" 77 | msg2 := "goodbye world" 78 | 79 | connect := []conf.ConnectorConfig{ 80 | { 81 | Type: "StanToStan", 82 | IncomingChannel: incoming, 83 | OutgoingChannel: outgoing, 84 | IncomingConnection: "stan", 85 | OutgoingConnection: "stan", 86 | IncomingStartAtSequence: 2, 87 | }, 88 | } 89 | 90 | tbs, err := StartTestEnvironmentInfrastructure(false) 91 | require.NoError(t, err) 92 | defer tbs.Close() 93 | 94 | done := make(chan string) 95 | sub, err := tbs.SC.Subscribe(outgoing, func(msg *stan.Msg) { 96 | done <- string(msg.Data) 97 | }) 98 | require.NoError(t, err) 99 | defer sub.Unsubscribe() 100 | 101 | // Send 2 messages, should only get 2nd 102 | err = tbs.SC.Publish(incoming, []byte(msg)) 103 | require.NoError(t, err) 104 | err = tbs.SC.Publish(incoming, []byte(msg2)) 105 | require.NoError(t, err) 106 | 107 | err = tbs.StartReplicator(connect) 108 | require.NoError(t, err) 109 | 110 | received := tbs.WaitForIt(1, done) 111 | require.Equal(t, msg2, received) 112 | 113 | received = tbs.WaitForIt(2, done) 114 | require.Empty(t, received) 115 | 116 | stats := tbs.Bridge.SafeStats() 117 | connStats := stats.Connections[0] 118 | require.Equal(t, int64(1), connStats.MessagesIn) 119 | require.Equal(t, int64(1), connStats.MessagesOut) 120 | } 121 | 122 | func TestQueueDeliverLatestToStan(t *testing.T) { 123 | incoming := nuid.Next() 124 | outgoing := nuid.Next() 125 | 126 | connect := []conf.ConnectorConfig{ 127 | { 128 | Type: "StanToStan", 129 | IncomingChannel: incoming, 130 | OutgoingChannel: outgoing, 131 | IncomingConnection: "stan", 132 | OutgoingConnection: "stan", 133 | IncomingStartAtSequence: -1, 134 | }, 135 | } 136 | 137 | tbs, err := StartTestEnvironmentInfrastructure(false) 138 | require.NoError(t, err) 139 | defer tbs.Close() 140 | 141 | done := make(chan string) 142 | sub, err := tbs.SC.Subscribe(outgoing, func(msg *stan.Msg) { 143 | done <- string(msg.Data) 144 | }) 145 | require.NoError(t, err) 146 | defer sub.Unsubscribe() 147 | 148 | // Send 2 messages, should only get 2nd 149 | err = tbs.SC.Publish(incoming, []byte("one")) 150 | require.NoError(t, err) 151 | err = tbs.SC.Publish(incoming, []byte("two")) 152 | require.NoError(t, err) 153 | 154 | err = tbs.StartReplicator(connect) 155 | require.NoError(t, err) 156 | 157 | err = tbs.SC.Publish(incoming, []byte("three")) 158 | require.NoError(t, err) 159 | 160 | received := tbs.WaitForIt(1, done) 161 | require.Equal(t, "two", received) 162 | 163 | received = tbs.WaitForIt(2, done) 164 | require.Equal(t, "three", received) 165 | 166 | received = tbs.WaitForIt(3, done) 167 | require.Empty(t, received) 168 | 169 | stats := tbs.Bridge.SafeStats() 170 | connStats := stats.Connections[0] 171 | require.Equal(t, int64(2), connStats.MessagesIn) 172 | require.Equal(t, int64(2), connStats.MessagesOut) 173 | } 174 | 175 | func TestQueueStartAtTimeToStan(t *testing.T) { 176 | incoming := nuid.Next() 177 | outgoing := nuid.Next() 178 | msg := "hello world" 179 | 180 | tbs, err := StartTestEnvironmentInfrastructure(false) 181 | require.NoError(t, err) 182 | defer tbs.Close() 183 | 184 | done := make(chan string) 185 | sub, err := tbs.SC.Subscribe(outgoing, func(msg *stan.Msg) { 186 | done <- string(msg.Data) 187 | }) 188 | require.NoError(t, err) 189 | defer sub.Unsubscribe() 190 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 191 | 192 | // Send 2 messages, should only get 2nd 193 | err = tbs.SC.Publish(incoming, []byte(msg)) 194 | require.NoError(t, err) 195 | err = tbs.SC.Publish(incoming, []byte(msg)) 196 | require.NoError(t, err) 197 | 198 | time.Sleep(2 * time.Second) // move the time along 199 | 200 | connect := []conf.ConnectorConfig{ 201 | { 202 | Type: "StanToStan", 203 | IncomingChannel: incoming, 204 | IncomingConnection: "stan", 205 | IncomingStartAtTime: time.Now().Unix(), 206 | OutgoingChannel: outgoing, 207 | OutgoingConnection: "stan", 208 | }, 209 | } 210 | 211 | err = tbs.StartReplicator(connect) 212 | require.NoError(t, err) 213 | 214 | time.Sleep(1 * time.Second) // move the time along 215 | 216 | err = tbs.SC.Publish(incoming, []byte(msg)) 217 | require.NoError(t, err) 218 | 219 | received := tbs.WaitForIt(1, done) 220 | require.Equal(t, msg, received) 221 | 222 | received = tbs.WaitForIt(2, done) 223 | require.Empty(t, received) 224 | 225 | stats := tbs.Bridge.SafeStats() 226 | connStats := stats.Connections[0] 227 | require.Equal(t, int64(1), connStats.MessagesIn) 228 | require.Equal(t, int64(1), connStats.MessagesOut) 229 | } 230 | 231 | func TestDurableSubscriberToStan(t *testing.T) { 232 | incoming := nuid.Next() 233 | outgoing := nuid.Next() 234 | 235 | tbs, err := StartTestEnvironmentInfrastructure(false) 236 | require.NoError(t, err) 237 | defer tbs.Close() 238 | 239 | connect := []conf.ConnectorConfig{ 240 | { 241 | Type: "StanToStan", 242 | IncomingChannel: incoming, 243 | IncomingConnection: "stan", 244 | IncomingDurableName: nuid.Next(), 245 | 246 | OutgoingChannel: outgoing, 247 | OutgoingConnection: "stan", 248 | }, 249 | } 250 | 251 | done := make(chan string) 252 | sub, err := tbs.SC.Subscribe(outgoing, func(msg *stan.Msg) { 253 | done <- string(msg.Data) 254 | }) 255 | require.NoError(t, err) 256 | defer sub.Unsubscribe() 257 | 258 | err = tbs.StartReplicator(connect) 259 | require.NoError(t, err) 260 | 261 | err = tbs.SC.Publish(incoming, []byte("one")) 262 | require.NoError(t, err) 263 | 264 | received := tbs.WaitForIt(1, done) 265 | require.Equal(t, "one", received) 266 | 267 | tbs.StopReplicator() 268 | 269 | err = tbs.SC.Publish(incoming, []byte("two")) 270 | require.NoError(t, err) 271 | 272 | err = tbs.SC.Publish(incoming, []byte("three")) 273 | require.NoError(t, err) 274 | 275 | err = tbs.StartReplicator(connect) 276 | require.NoError(t, err) 277 | 278 | received = tbs.WaitForIt(1, done) // reset counter, we restarted replicator 279 | require.Equal(t, "two", received) 280 | 281 | received = tbs.WaitForIt(2, done) 282 | require.Equal(t, "three", received) 283 | 284 | received = tbs.WaitForIt(4, done) 285 | require.Empty(t, received) 286 | 287 | // Should have 2 messages since the relaunch 288 | stats := tbs.Bridge.SafeStats() 289 | connStats := stats.Connections[0] 290 | require.Equal(t, int64(2), connStats.MessagesIn) 291 | require.Equal(t, int64(2), connStats.MessagesOut) 292 | } 293 | 294 | func TestSimpleSendOnStanReceiveOnStanWithTLS(t *testing.T) { 295 | incoming := nuid.Next() 296 | outgoing := nuid.Next() 297 | msg := "hello world" 298 | 299 | connect := []conf.ConnectorConfig{ 300 | { 301 | Type: "stantostan", // test with different casing 302 | IncomingChannel: incoming, 303 | IncomingConnection: "stan", 304 | OutgoingChannel: outgoing, 305 | OutgoingConnection: "stan", 306 | }, 307 | } 308 | 309 | tbs, err := StartTLSTestEnvironment(connect) 310 | require.NoError(t, err) 311 | defer tbs.Close() 312 | 313 | done := make(chan string) 314 | sub, err := tbs.SC.Subscribe(outgoing, func(msg *stan.Msg) { 315 | done <- string(msg.Data) 316 | }) 317 | require.NoError(t, err) 318 | defer sub.Unsubscribe() 319 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 320 | 321 | err = tbs.SC.Publish(incoming, []byte(msg)) 322 | require.NoError(t, err) 323 | 324 | received := tbs.WaitForIt(1, done) 325 | require.Equal(t, msg, received) 326 | } 327 | -------------------------------------------------------------------------------- /server/core/stan2nats_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package core 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | 23 | "github.com/nats-io/nats-replicator/server/conf" 24 | nats "github.com/nats-io/nats.go" 25 | "github.com/nats-io/nuid" 26 | "github.com/stretchr/testify/require" 27 | ) 28 | 29 | func TestSimpleSendOnStanReceiveOnNats(t *testing.T) { 30 | incoming := nuid.Next() 31 | outgoing := nuid.Next() 32 | msg := "hello world" 33 | 34 | connect := []conf.ConnectorConfig{ 35 | { 36 | Type: "StanToNATS", 37 | IncomingChannel: incoming, 38 | OutgoingSubject: outgoing, 39 | IncomingConnection: "stan", 40 | OutgoingConnection: "nats", 41 | }, 42 | } 43 | 44 | tbs, err := StartTestEnvironment(connect) 45 | require.NoError(t, err) 46 | defer tbs.Close() 47 | 48 | tbs.Bridge.checkConnections() 49 | 50 | done := make(chan string) 51 | sub, err := tbs.NC.Subscribe(outgoing, func(msg *nats.Msg) { 52 | done <- string(msg.Data) 53 | }) 54 | require.NoError(t, err) 55 | defer sub.Unsubscribe() 56 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 57 | 58 | err = tbs.SC.Publish(incoming, []byte(msg)) 59 | require.NoError(t, err) 60 | 61 | received := tbs.WaitForIt(1, done) 62 | require.Equal(t, msg, received) 63 | 64 | stats := tbs.Bridge.SafeStats() 65 | connStats := stats.Connections[0] 66 | require.Equal(t, int64(1), connStats.MessagesIn) 67 | require.Equal(t, int64(1), connStats.MessagesOut) 68 | require.Equal(t, int64(len([]byte(msg))), connStats.BytesIn) 69 | require.Equal(t, int64(1), connStats.Connects) 70 | require.Equal(t, int64(0), connStats.Disconnects) 71 | require.True(t, connStats.Connected) 72 | } 73 | 74 | func TestStanToNatsStartAtPosition(t *testing.T) { 75 | incoming := nuid.Next() 76 | outgoing := nuid.Next() 77 | msg := "hello world" 78 | msg2 := "goodbye world" 79 | 80 | connect := []conf.ConnectorConfig{ 81 | { 82 | Type: "StanToNATS", 83 | IncomingChannel: incoming, 84 | IncomingConnection: "stan", 85 | IncomingStartAtSequence: 2, 86 | 87 | OutgoingSubject: outgoing, 88 | OutgoingConnection: "nats", 89 | }, 90 | } 91 | 92 | tbs, err := StartTestEnvironmentInfrastructure(false) 93 | require.NoError(t, err) 94 | defer tbs.Close() 95 | 96 | done := make(chan string) 97 | sub, err := tbs.NC.Subscribe(outgoing, func(msg *nats.Msg) { 98 | done <- string(msg.Data) 99 | }) 100 | require.NoError(t, err) 101 | defer sub.Unsubscribe() 102 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 103 | 104 | // Send 2 messages, should only get 2nd 105 | err = tbs.SC.Publish(incoming, []byte(msg)) 106 | require.NoError(t, err) 107 | err = tbs.SC.Publish(incoming, []byte(msg2)) 108 | require.NoError(t, err) 109 | 110 | err = tbs.StartReplicator(connect) 111 | require.NoError(t, err) 112 | 113 | received := tbs.WaitForIt(1, done) 114 | require.Equal(t, msg2, received) 115 | 116 | received = tbs.WaitForIt(2, done) 117 | require.Empty(t, received) 118 | 119 | stats := tbs.Bridge.SafeStats() 120 | connStats := stats.Connections[0] 121 | require.Equal(t, int64(1), connStats.MessagesIn) 122 | require.Equal(t, int64(1), connStats.MessagesOut) 123 | } 124 | 125 | func TestStanToNatsDeliverLatest(t *testing.T) { 126 | incoming := nuid.Next() 127 | outgoing := nuid.Next() 128 | 129 | connect := []conf.ConnectorConfig{ 130 | { 131 | Type: "StanToNATS", 132 | IncomingChannel: incoming, 133 | IncomingConnection: "stan", 134 | IncomingStartAtSequence: -1, 135 | 136 | OutgoingSubject: outgoing, 137 | OutgoingConnection: "nats", 138 | }, 139 | } 140 | 141 | tbs, err := StartTestEnvironmentInfrastructure(false) 142 | require.NoError(t, err) 143 | defer tbs.Close() 144 | 145 | done := make(chan string) 146 | sub, err := tbs.NC.Subscribe(outgoing, func(msg *nats.Msg) { 147 | done <- string(msg.Data) 148 | }) 149 | require.NoError(t, err) 150 | defer sub.Unsubscribe() 151 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 152 | 153 | // Send 2 messages, should only get 2nd 154 | err = tbs.SC.Publish(incoming, []byte("one")) 155 | require.NoError(t, err) 156 | err = tbs.SC.Publish(incoming, []byte("two")) 157 | require.NoError(t, err) 158 | 159 | err = tbs.StartReplicator(connect) 160 | require.NoError(t, err) 161 | 162 | err = tbs.SC.Publish(incoming, []byte("three")) 163 | require.NoError(t, err) 164 | 165 | received := tbs.WaitForIt(1, done) 166 | require.Equal(t, "two", received) 167 | 168 | received = tbs.WaitForIt(2, done) 169 | require.Equal(t, "three", received) 170 | 171 | received = tbs.WaitForIt(3, done) 172 | require.Empty(t, received) 173 | 174 | stats := tbs.Bridge.SafeStats() 175 | connStats := stats.Connections[0] 176 | require.Equal(t, int64(2), connStats.MessagesIn) 177 | require.Equal(t, int64(2), connStats.MessagesOut) 178 | } 179 | 180 | func TestStanToNatsStartAtTime(t *testing.T) { 181 | incoming := nuid.Next() 182 | outgoing := nuid.Next() 183 | msg := "hello world" 184 | 185 | tbs, err := StartTestEnvironmentInfrastructure(false) 186 | require.NoError(t, err) 187 | defer tbs.Close() 188 | 189 | done := make(chan string) 190 | sub, err := tbs.NC.Subscribe(outgoing, func(msg *nats.Msg) { 191 | done <- string(msg.Data) 192 | }) 193 | require.NoError(t, err) 194 | defer sub.Unsubscribe() 195 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 196 | 197 | // Send 2 messages, should only get 2nd 198 | err = tbs.SC.Publish(incoming, []byte(msg)) 199 | require.NoError(t, err) 200 | err = tbs.SC.Publish(incoming, []byte(msg)) 201 | require.NoError(t, err) 202 | 203 | time.Sleep(2 * time.Second) // move the time along 204 | 205 | connect := []conf.ConnectorConfig{ 206 | { 207 | Type: "StanToNATS", 208 | IncomingChannel: incoming, 209 | OutgoingSubject: outgoing, 210 | IncomingConnection: "stan", 211 | OutgoingConnection: "nats", 212 | IncomingStartAtTime: time.Now().Unix(), 213 | }, 214 | } 215 | err = tbs.StartReplicator(connect) 216 | require.NoError(t, err) 217 | 218 | time.Sleep(1 * time.Second) // move the time along 219 | 220 | err = tbs.SC.Publish(incoming, []byte(msg)) 221 | require.NoError(t, err) 222 | 223 | received := tbs.WaitForIt(1, done) 224 | require.Equal(t, msg, received) 225 | 226 | received = tbs.WaitForIt(2, done) 227 | require.Empty(t, received) 228 | 229 | stats := tbs.Bridge.SafeStats() 230 | connStats := stats.Connections[0] 231 | require.Equal(t, int64(1), connStats.MessagesIn) 232 | require.Equal(t, int64(1), connStats.MessagesOut) 233 | } 234 | 235 | func TestStanToNatsDurableSubscriber(t *testing.T) { 236 | incoming := nuid.Next() 237 | outgoing := nuid.Next() 238 | 239 | tbs, err := StartTestEnvironmentInfrastructure(false) 240 | require.NoError(t, err) 241 | defer tbs.Close() 242 | 243 | done := make(chan string) 244 | sub, err := tbs.NC.Subscribe(outgoing, func(msg *nats.Msg) { 245 | done <- string(msg.Data) 246 | }) 247 | require.NoError(t, err) 248 | defer sub.Unsubscribe() 249 | require.NoError(t, tbs.NC.FlushTimeout(5*time.Second)) 250 | 251 | connect := []conf.ConnectorConfig{ 252 | { 253 | Type: "StanToNATS", 254 | IncomingConnection: "stan", 255 | IncomingChannel: incoming, 256 | IncomingDurableName: nuid.Next(), 257 | 258 | OutgoingConnection: "nats", 259 | OutgoingSubject: outgoing, 260 | }, 261 | } 262 | 263 | err = tbs.StartReplicator(connect) 264 | require.NoError(t, err) 265 | 266 | err = tbs.SC.Publish(incoming, []byte("one")) 267 | require.NoError(t, err) 268 | 269 | received := tbs.WaitForIt(1, done) 270 | require.Equal(t, "one", received) 271 | 272 | tbs.StopReplicator() 273 | 274 | // Publish two while the replicator is down 275 | // these should be waiting for the durable subscriber 276 | err = tbs.SC.Publish(incoming, []byte("two")) 277 | require.NoError(t, err) 278 | 279 | err = tbs.SC.Publish(incoming, []byte("three")) 280 | require.NoError(t, err) 281 | 282 | err = tbs.StartReplicator(connect) 283 | require.NoError(t, err) 284 | 285 | received = tbs.WaitForIt(1, done) // Reset counter on restart 286 | require.Equal(t, "two", received) 287 | 288 | received = tbs.WaitForIt(2, done) 289 | require.Equal(t, "three", received) 290 | 291 | received = tbs.WaitForIt(3, done) 292 | require.Empty(t, received) 293 | 294 | // Should have 2 messages since the relaunch 295 | stats := tbs.Bridge.SafeStats() 296 | connStats := stats.Connections[0] 297 | require.Equal(t, int64(2), connStats.MessagesIn) 298 | require.Equal(t, int64(2), connStats.MessagesOut) 299 | } 300 | 301 | func TestSimpleSendOnStanReceiveOnNatsWithTLS(t *testing.T) { 302 | incoming := nuid.Next() 303 | outgoing := nuid.Next() 304 | msg := "hello world" 305 | 306 | connect := []conf.ConnectorConfig{ 307 | { 308 | Type: "stantonats", // test with different casing 309 | IncomingChannel: incoming, 310 | OutgoingSubject: outgoing, 311 | IncomingConnection: "stan", 312 | OutgoingConnection: "nats", 313 | }, 314 | } 315 | 316 | tbs, err := StartTLSTestEnvironment(connect) 317 | require.NoError(t, err) 318 | defer tbs.Close() 319 | 320 | done := make(chan string) 321 | sub, err := tbs.NC.Subscribe(outgoing, func(msg *nats.Msg) { 322 | done <- string(msg.Data) 323 | }) 324 | require.NoError(t, err) 325 | defer sub.Unsubscribe() 326 | require.NoError(t, tbs.NC.FlushTimeout(time.Second*5)) 327 | 328 | err = tbs.SC.Publish(incoming, []byte(msg)) 329 | require.NoError(t, err) 330 | 331 | received := tbs.WaitForIt(1, done) 332 | require.Equal(t, msg, received) 333 | } 334 | -------------------------------------------------------------------------------- /server/core/server_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package core 18 | 19 | import ( 20 | "fmt" 21 | "time" 22 | 23 | "github.com/nats-io/nats-replicator/server/conf" 24 | gnatsserver "github.com/nats-io/nats-server/v2/server" 25 | gnatsd "github.com/nats-io/nats-server/v2/test" 26 | nss "github.com/nats-io/nats-streaming-server/server" 27 | nats "github.com/nats-io/nats.go" 28 | "github.com/nats-io/nuid" 29 | stan "github.com/nats-io/stan.go" 30 | ) 31 | 32 | const ( 33 | serverCert = "../../resources/certs/server-cert.pem" 34 | serverKey = "../../resources/certs/server-key.pem" 35 | // clientCert = "../../resources/certs/client-cert.pem" 36 | // clientKey = "../../resources/certs/client-key.pem" 37 | caFile = "../../resources/certs/truststore.pem" 38 | ) 39 | 40 | // TestEnv encapsulate a bridge test environment 41 | type TestEnv struct { 42 | Config *conf.NATSReplicatorConfig 43 | Gnatsd *gnatsserver.Server 44 | Stan *nss.StanServer 45 | 46 | NC *nats.Conn // for bypassing the bridge 47 | SC stan.Conn // for bypassing the bridge 48 | 49 | natsPort int 50 | natsURL string 51 | clusterName string 52 | clientID string // we keep this so we stay the same on reconnect 53 | bridgeClientID string 54 | 55 | Bridge *NATSReplicator 56 | 57 | useTLS bool 58 | } 59 | 60 | // StartTestEnvironment calls StartTestEnvironmentInfrastructure 61 | // followed by StartReplicator 62 | func StartTestEnvironment(connections []conf.ConnectorConfig) (*TestEnv, error) { 63 | tbs, err := StartTestEnvironmentInfrastructure(false) 64 | if err != nil { 65 | return nil, err 66 | } 67 | err = tbs.StartReplicator(connections) 68 | if err != nil { 69 | tbs.Close() 70 | return nil, err 71 | } 72 | return tbs, err 73 | } 74 | 75 | // StartTLSTestEnvironment calls StartTestEnvironmentInfrastructure 76 | // followed by StartReplicator, with TLS enabled 77 | func StartTLSTestEnvironment(connections []conf.ConnectorConfig) (*TestEnv, error) { 78 | tbs, err := StartTestEnvironmentInfrastructure(true) 79 | if err != nil { 80 | return nil, err 81 | } 82 | err = tbs.StartReplicator(connections) 83 | if err != nil { 84 | tbs.Close() 85 | return nil, err 86 | } 87 | return tbs, err 88 | } 89 | 90 | // StartTestEnvironmentInfrastructure creates the kafka server, Nats and streaming 91 | // but does not start a bridge, you can use StartReplicator to start a bridge afterward 92 | func StartTestEnvironmentInfrastructure(useTLS bool) (*TestEnv, error) { 93 | tbs := &TestEnv{} 94 | tbs.useTLS = useTLS 95 | 96 | err := tbs.StartNATSandStan(-1, nuid.Next(), nuid.Next(), nuid.Next()) 97 | if err != nil { 98 | tbs.Close() 99 | return nil, err 100 | } 101 | 102 | return tbs, nil 103 | } 104 | 105 | // StartReplicator is the second half of StartTestEnvironment 106 | // it is provided separately so that environment can be created before the bridge runs 107 | func (tbs *TestEnv) StartReplicator(connections []conf.ConnectorConfig) error { 108 | config := conf.DefaultConfig() 109 | config.ReconnectInterval = 200 110 | config.Logging.Debug = true 111 | config.Logging.Trace = true 112 | config.Logging.Colors = false 113 | config.Monitoring = conf.HTTPConfig{ 114 | HTTPPort: -1, 115 | } 116 | config.NATS = []conf.NATSConfig{} 117 | config.STAN = []conf.NATSStreamingConfig{} 118 | 119 | config.NATS = append(config.NATS, conf.NATSConfig{ 120 | Name: "nats", 121 | Servers: []string{tbs.natsURL}, 122 | ConnectTimeout: 2000, 123 | ReconnectWait: 2000, 124 | MaxReconnects: 5, 125 | }) 126 | config.STAN = append(config.STAN, conf.NATSStreamingConfig{ 127 | Name: "stan", 128 | ClusterID: tbs.clusterName, 129 | ClientID: tbs.bridgeClientID, 130 | PubAckWait: 5000, 131 | DiscoverPrefix: stan.DefaultDiscoverPrefix, 132 | MaxPubAcksInflight: stan.DefaultMaxPubAcksInflight, 133 | ConnectWait: 2000, 134 | NATSConnection: "nats", 135 | PingInterval: 1, 136 | MaxPings: 3, 137 | }) 138 | 139 | if tbs.useTLS { 140 | config.Monitoring.HTTPPort = 0 141 | config.Monitoring.HTTPSPort = -1 142 | 143 | config.Monitoring.TLS = conf.TLSConf{ 144 | Cert: serverCert, 145 | Key: serverKey, 146 | } 147 | 148 | c := config.NATS[0] 149 | c.TLS = conf.TLSConf{ 150 | Root: caFile, 151 | } 152 | config.NATS[0] = c 153 | } 154 | 155 | config.Connect = connections 156 | 157 | tbs.Config = &config 158 | tbs.Bridge = NewNATSReplicator() 159 | err := tbs.Bridge.InitializeFromConfig(config) 160 | if err != nil { 161 | tbs.Close() 162 | return err 163 | } 164 | err = tbs.Bridge.Start() 165 | if err != nil { 166 | tbs.Close() 167 | return err 168 | } 169 | 170 | return nil 171 | } 172 | 173 | // StartNATSandStan starts up the nats and stan servers 174 | func (tbs *TestEnv) StartNATSandStan(port int, clusterID string, clientID string, bridgeClientID string) error { 175 | var err error 176 | opts := gnatsd.DefaultTestOptions 177 | opts.Port = port 178 | 179 | if tbs.useTLS { 180 | opts.TLSCert = serverCert 181 | opts.TLSKey = serverKey 182 | opts.TLSTimeout = 5 183 | 184 | tc := gnatsserver.TLSConfigOpts{} 185 | tc.CertFile = opts.TLSCert 186 | tc.KeyFile = opts.TLSKey 187 | 188 | opts.TLSConfig, err = gnatsserver.GenTLSConfig(&tc) 189 | 190 | if err != nil { 191 | return err 192 | } 193 | } 194 | tbs.Gnatsd = gnatsd.RunServer(&opts) 195 | 196 | if tbs.useTLS { 197 | tbs.natsURL = fmt.Sprintf("tls://localhost:%d", opts.Port) 198 | } else { 199 | tbs.natsURL = fmt.Sprintf("nats://localhost:%d", opts.Port) 200 | } 201 | 202 | tbs.natsPort = opts.Port 203 | tbs.clusterName = clusterID 204 | sOpts := nss.GetDefaultOptions() 205 | sOpts.ID = tbs.clusterName 206 | sOpts.NATSServerURL = tbs.natsURL 207 | 208 | if tbs.useTLS { 209 | sOpts.ClientCA = caFile 210 | } 211 | 212 | nOpts := nss.DefaultNatsServerOptions 213 | nOpts.Port = -1 214 | 215 | s, err := nss.RunServerWithOpts(sOpts, &nOpts) 216 | if err != nil { 217 | return err 218 | } 219 | 220 | tbs.Stan = s 221 | tbs.clientID = clientID 222 | tbs.bridgeClientID = bridgeClientID 223 | 224 | var nc *nats.Conn 225 | 226 | if tbs.useTLS { 227 | nc, err = nats.Connect(tbs.natsURL, nats.RootCAs(caFile)) 228 | } else { 229 | nc, err = nats.Connect(tbs.natsURL) 230 | } 231 | 232 | if err != nil { 233 | return err 234 | } 235 | 236 | tbs.NC = nc 237 | 238 | sc, err := stan.Connect(tbs.clusterName, tbs.clientID, stan.NatsConn(tbs.NC)) 239 | if err != nil { 240 | return err 241 | } 242 | tbs.SC = sc 243 | 244 | return nil 245 | } 246 | 247 | // StopReplicator stops the bridge 248 | func (tbs *TestEnv) StopReplicator() { 249 | if tbs.Bridge != nil { 250 | tbs.Bridge.Stop() 251 | tbs.Bridge = nil 252 | } 253 | } 254 | 255 | // StopNATS shuts down the NATS and Stan servers 256 | func (tbs *TestEnv) StopNATS() error { 257 | if tbs.SC != nil { 258 | tbs.SC.Close() 259 | tbs.SC = nil 260 | } 261 | 262 | if tbs.NC != nil { 263 | tbs.NC.Close() 264 | tbs.NC = nil 265 | } 266 | 267 | if tbs.Stan != nil { 268 | tbs.Stan.Shutdown() 269 | tbs.Stan = nil 270 | } 271 | 272 | if tbs.Gnatsd != nil { 273 | tbs.Gnatsd.Shutdown() 274 | tbs.Gnatsd = nil 275 | } 276 | 277 | return nil 278 | } 279 | 280 | // RestartNATS shuts down the NATS and stan server and then starts it again 281 | func (tbs *TestEnv) RestartNATS() error { 282 | if tbs.SC != nil { 283 | tbs.SC.Close() 284 | } 285 | 286 | if tbs.NC != nil { 287 | tbs.NC.Close() 288 | } 289 | 290 | if tbs.Stan != nil { 291 | tbs.Stan.Shutdown() 292 | } 293 | 294 | if tbs.Gnatsd != nil { 295 | tbs.Gnatsd.Shutdown() 296 | } 297 | 298 | err := tbs.StartNATSandStan(tbs.natsPort, tbs.clusterName, tbs.clientID, tbs.bridgeClientID) 299 | if err != nil { 300 | return err 301 | } 302 | 303 | return nil 304 | } 305 | 306 | // Close the bridge server and clean up the test environment 307 | func (tbs *TestEnv) Close() { 308 | // Stop the bridge first! 309 | if tbs.Bridge != nil { 310 | tbs.Bridge.Stop() 311 | } 312 | 313 | if tbs.SC != nil { 314 | tbs.SC.Close() 315 | } 316 | 317 | if tbs.NC != nil { 318 | tbs.NC.Close() 319 | } 320 | 321 | if tbs.Stan != nil { 322 | tbs.Stan.Shutdown() 323 | } 324 | 325 | if tbs.Gnatsd != nil { 326 | tbs.Gnatsd.Shutdown() 327 | } 328 | } 329 | 330 | func (tbs *TestEnv) WaitForIt(requestCount int64, done chan string) string { 331 | timeout := time.Duration(5000) * time.Millisecond // 5 second timeout for tests 332 | stop := time.Now().Add(timeout) 333 | timer := time.NewTimer(timeout) 334 | requestsOk := make(chan bool) 335 | 336 | // Timeout the done channel 337 | go func() { 338 | <-timer.C 339 | done <- "" 340 | }() 341 | 342 | ticker := time.NewTicker(50 * time.Millisecond) 343 | go func() { 344 | for t := range ticker.C { 345 | if t.After(stop) { 346 | requestsOk <- false 347 | break 348 | } 349 | 350 | if tbs.Bridge.SafeStats().RequestCount >= requestCount { 351 | requestsOk <- true 352 | break 353 | } 354 | } 355 | ticker.Stop() 356 | }() 357 | 358 | received := <-done 359 | ok := <-requestsOk 360 | 361 | if !ok { 362 | received = "" 363 | } 364 | 365 | return received 366 | } 367 | 368 | func (tbs *TestEnv) WaitForRequests(requestCount int64) { 369 | timeout := time.Duration(5000) * time.Millisecond // 5 second timeout for tests 370 | stop := time.Now().Add(timeout) 371 | requestsOk := make(chan bool) 372 | 373 | ticker := time.NewTicker(50 * time.Millisecond) 374 | go func() { 375 | for t := range ticker.C { 376 | if t.After(stop) { 377 | requestsOk <- false 378 | break 379 | } 380 | 381 | if tbs.Bridge.SafeStats().RequestCount >= requestCount { 382 | requestsOk <- true 383 | break 384 | } 385 | } 386 | ticker.Stop() 387 | }() 388 | 389 | <-requestsOk 390 | } 391 | -------------------------------------------------------------------------------- /docs/config.md: -------------------------------------------------------------------------------- 1 | # NATS-Replicator Configuration 2 | 3 | The replicator uses a single configuration file passed on the command line or environment variable. Configuration is organized into a root section and several blocks. 4 | 5 | * [Specifying the Configuration File](#specify) 6 | * [Shared](#root) 7 | * [TLS](#tls) 8 | * [Logging](#logging) 9 | * [Monitoring](#monitoring) 10 | * [NATS](#nats) 11 | * [NATS Streaming](#stan) 12 | * [Connectors](#connectors) 13 | 14 | The configuration file format matches the NATS server and supports file includes of the form: 15 | 16 | ```yaml 17 | include "./includes/connectors.conf" 18 | ``` 19 | 20 | All properties are semi-case-insensitive. If you use the case in the doc it is ok, or use all lower case. Also, multi-word properties have two forms, to match your style better. 21 | 22 | 23 | 24 | ## Specifying the Configuration File 25 | 26 | To set the configuration on the command line, use: 27 | 28 | ```bash 29 | % nats-replicator -c 30 | ``` 31 | 32 | To set the configuration file using an environment variable, export `NATS_REPLICATOR_CONFIG` with the path to the configuration. 33 | 34 | 35 | 36 | ## Root Section 37 | 38 | The root section: 39 | 40 | ```yaml 41 | reconnectinterval: 5000, 42 | ``` 43 | 44 | can currently contain settings for: 45 | 46 | * `reconnectinterval` or `reconnect_interval` - this value, in milliseconds, is the time used in between reconnection attempts for a connector when it fails. For example, if a connector loses access to NATS, the replicator will try to restart it every `reconnectinterval` milliseconds. 47 | 48 | ## TLS 49 | 50 | NATS, streaming and HTTP configurations all take an optional TLS setting. The TLS configuration takes three possible settings: 51 | 52 | * `root` - file path to a CA root certificate store, used for NATS connections 53 | * `cert` - file path to a server certificate, used for HTTPS monitoring and optionally for client side certificates with NATS 54 | * `key` - key for the certificate store specified in cert 55 | 56 | 57 | 58 | ### Logging 59 | 60 | Logging is configured in a manner similar to the nats-server: 61 | 62 | ```yaml 63 | logging: { 64 | time: true, 65 | debug: false, 66 | trace: false, 67 | colors: true, 68 | pid: false, 69 | } 70 | ``` 71 | 72 | These properties are configured for: 73 | 74 | * `time` - include the time in logging statements 75 | * `debug` - include debug logging 76 | * `trace` - include verbose, or trace, logging 77 | * `colors` - colorize the logging statements 78 | * `pid` - include the process id in logging statements 79 | 80 | 81 | 82 | ## Monitoring 83 | 84 | The monitoring section: 85 | 86 | ```yaml 87 | monitoring: { 88 | httpsport: -1, 89 | tls: { 90 | cert: /a/server-cert.pem, 91 | key: /a/server-key.pem, 92 | } 93 | } 94 | ``` 95 | 96 | Is used to configure an HTTP or HTTPS port, as well as TLS settings when HTTPS is used. 97 | 98 | * `httphost` or `http_host` - the network interface to publish monitoring on, valid for HTTP or HTTPS. An empty value will tell the replicator to use all available network interfaces. 99 | * `httpport` or `http_port` - the port for HTTP monitoring, no TLS configuration is expected, a value of -1 will tell the replicator to use an ephemeral port, the port will be logged on startup. 100 | 101 | `2019/03/20 12:06:38.027822 [INF] starting http monitor on :59744` 102 | 103 | * `httpsport` or `https_port` - the port for HTTPS monitoring, a TLS configuration is expected, a value of -1 will tell the server to use an ephemeral port, the port will be logged on startup. 104 | * `tls` - a [TLS configuration](#tls). 105 | 106 | The `httpport` and `httpsport` settings are mutually exclusive, if both are set to a non-zero value the replicator will not start. 107 | 108 | 109 | 110 | ## NATS 111 | 112 | The replicator can make multiple connections to NATS. Each connection has a name so that connectors can refer to it. Streaming connections also use the name of a NATS connection to refer to it. Configuration is through the `nats` section of the config file, which defines an array of connection parameters: 113 | 114 | ```yaml 115 | nats: [ 116 | { 117 | Name: "connection_one", 118 | Servers: ["localhost:4222"], 119 | ConnectTimeout: 5000, 120 | MaxReconnects: 5, 121 | ReconnectWait: 5000, 122 | } 123 | ] 124 | ``` 125 | 126 | NATS can be configured with the following properties: 127 | 128 | * `name` - the unique name used to refer to this configuration/connection 129 | * `servers` - an array of server URLS 130 | * `connecttimeout` or `connect_timeout` - the time, in milliseconds, to wait before failing to connect to the NATS server 131 | * `reconnectwait` or `reconnect_wait` - the time, in milliseconds, to wait between reconnect attempts 132 | * `maxreconnects` or `max_reconnects` - the maximum number of reconnects to try before exiting the replicator with an error. 133 | * `noecho` or `no_echo` - don't echo messages back from this client 134 | * `norandom` or `no_random` - don't randomize servers in the connect list 135 | * `tls` - (optional) [TLS configuration](#tls). If the NATS server uses unverified TLS with a valid certificate, this setting isn't required. 136 | * `usercredentials` or `user_credentials` - (optional) the path to a credentials file for connecting to NATs. 137 | 138 | 139 | 140 | ## NATS Streaming 141 | 142 | The replicator can make multiple connections to streaming servers. Each streaming connection must refer to a NATS connection by name. This reduces the configuration clutter by keeping NATS parameters in the NATS section. Configuration is through the `stan` section of the config file, which defines an array: 143 | 144 | ```yaml 145 | stan: [ 146 | { 147 | Name: "stan_one", 148 | NATSConnection: "connection_one", 149 | ClusterID: "test-cluster" 150 | ClientID: "replicator_one" 151 | } 152 | ] 153 | ``` 154 | 155 | Multiple streaming connections could use the same NATS connection. 156 | 157 | NATS streaming can be configured with the following properties: 158 | 159 | * `name` - the unique name used to refer to this configuration/connection 160 | * `natsconnection` or `nats_connection` - the unique name of the nats connection to use for this streaming connection 161 | * `clusterid` or `cluster_id` - the cluster id for the NATS streaming server. 162 | * `clientid` or `client_id` - the client id for the connection. 163 | * `pubackwait` or `pub_ack_wait` - the time, in milliseconds, to wait before a publish fails due to a timeout. 164 | * `discoverprefix` or `discover_prefix` - the discover prefix for the streaming server. 165 | * `maxpubacksinflight` or `max_pubacks_inflight` - maximum pub ACK messages that can be in flight for this connection, defaults to streaming default. 166 | * `connectwait` or `connect_wait` - the time, in milliseconds, to wait before failing to connect to the streaming server. 167 | 168 | 169 | 170 | ## Connectors 171 | 172 | The final piece of the replicator configuration is the `connect` section. Connect specifies an array of connector configurations. All connector configs use the same format, relying on optional settings to determine what the do. 173 | 174 | ```yaml 175 | connect: [ 176 | { 177 | type: NATSToNATS, 178 | id: "alpha", 179 | IncomingSubject: "in", 180 | IncomingConnection: "connection_one", 181 | OutgoingSubject: "out", 182 | OutgoingConnection: "connection_one", 183 | }, 184 | ], 185 | ``` 186 | 187 | The most important property in the connector configuration is the `type`. The type determines which kind of connector is instantiated. Available, uni-directional, types include: 188 | 189 | * `NATSToNATS` - a subject to subject connector 190 | * `NATSToStan` - a subject to streaming connector 191 | * `StanToNATS` - a streaming to subject connector 192 | * `StanToStan` - a streaming to streaming connector 193 | 194 | These types are case insensitive, so "natstonats" is the same as "NATSToNATS". 195 | 196 | All connectors can have an optional id, which is used in monitoring: 197 | 198 | * `id` - (optional) user defined id that will tag the connection in monitoring JSON. 199 | 200 | All connectors require a configuration for the connection to use: 201 | 202 | * `incomingconnection` or `incoming_connection` - the name of a NATS or streaming connection to subscribe to 203 | * `outgoingconnection` or `outgoing_connection` - the name of a NATS or streaming connection to publish to 204 | 205 | For NATS connections, specify: 206 | 207 | * `incomingsubject` or `incoming_subject` - the subject to subscribe to, depending on the connections direction. 208 | * `incomingqueuename` or `incoming_queue_name` - the queue group to use in subscriptions, this is optional but useful for load balancing. 209 | * `outgoingsubject` or `outgoing_subject` - the subject to publish to, depending on the connections direction. 210 | 211 | Keep in mind that NATS queue groups do not guarantee ordering, since the queue subscribers can be on different nats-servers in a cluster. So if you have to replicators running with connectors on the same NATS queue/subject pair and have a high message rate you may get messages to the receiver "out of order." Also, note that there is no outgoing queue. 212 | 213 | These settings are directional depending so a `NATSToStan` connector would use an `incomingsubject` while a `StanToNATS` connector would use an `outgoingsubject`. Connectors ignore settings they don't need. 214 | 215 | For streaming connections, the channel setting is required (directionality dependent), the others are optional: 216 | 217 | * `incomingchannel` or `incoming_channel` - the streaming channel to subscribe to. 218 | * `outgoingchannel` or `outgoing_channel` - the streaming channel to publish to. 219 | * `incomingdurablename` or `incoming_durable_name` - (optional) durable name for the streaming subscription (if appropriate.) 220 | * `incomingstartatsequence` or `incoming_startat_sequence` - (optional) start position, use -1 for start with last received, 0 for deliver all available (the default.) 221 | * `incomingstartattime` or `incoming_startat_time` - (optional) the start position as a time, in Unix seconds since the epoch, mutually exclusive with `startatsequence`. 222 | 223 | For example, a simple configuration may look something like: 224 | 225 | ```yaml 226 | 227 | reconnectinterval: 5000, 228 | 229 | logging: { 230 | Time: true, 231 | Debug: true, 232 | Trace: false, 233 | Colors: true, 234 | PID: false, 235 | }, 236 | 237 | monitoring: { 238 | httpport: 9090, 239 | } 240 | 241 | nats: [ 242 | { 243 | Name: "nats", 244 | Servers: ["localhost:4222"], 245 | ConnectTimeout: 5000, 246 | MaxReconnects: 5, 247 | ReconnectWait: 5000, 248 | } 249 | ] 250 | 251 | connect: [ 252 | { 253 | type: NATSToNATS, 254 | id: "alpha", 255 | IncomingSubject: "in", 256 | IncomingConnection: "nats", 257 | OutgoingSubject: "out", 258 | OutgoingConnection: "nats", 259 | }, 260 | ], 261 | 262 | ``` 263 | -------------------------------------------------------------------------------- /performance/nats_nats_bench/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 The NATS Authors 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "encoding/json" 21 | "flag" 22 | "log" 23 | "strings" 24 | "sync" 25 | "time" 26 | 27 | "github.com/nats-io/nats-replicator/server/conf" 28 | "github.com/nats-io/nats-replicator/server/core" 29 | "github.com/nats-io/nats.go" 30 | "github.com/nats-io/nuid" 31 | ) 32 | 33 | var iterations int 34 | var messageSize int 35 | var natsURL string 36 | var natsURL2 string 37 | var direct bool 38 | var pubOnly bool 39 | var subOnly bool 40 | var repOnly bool 41 | var showStats bool 42 | var hideReplicatorLog bool 43 | var showProgress bool 44 | var in string 45 | var out string 46 | 47 | func startReplicator(connections []conf.ConnectorConfig) (*core.NATSReplicator, error) { 48 | config := conf.DefaultConfig() 49 | config.Logging.Debug = false 50 | config.Logging.Trace = false 51 | config.Logging.Colors = false 52 | config.Logging.Hide = hideReplicatorLog 53 | config.Monitoring = conf.HTTPConfig{ 54 | HTTPPort: -1, 55 | } 56 | 57 | config.NATS = []conf.NATSConfig{} 58 | config.NATS = append(config.NATS, conf.NATSConfig{ 59 | Servers: []string{natsURL}, 60 | ConnectTimeout: 5000, 61 | ReconnectWait: 5000, 62 | MaxReconnects: 5, 63 | Name: "nats", 64 | }) 65 | config.NATS = append(config.NATS, conf.NATSConfig{ 66 | Servers: []string{natsURL2}, 67 | ConnectTimeout: 5000, 68 | ReconnectWait: 5000, 69 | MaxReconnects: 5, 70 | Name: "nats2", 71 | }) 72 | 73 | config.Connect = connections 74 | 75 | replicator := core.NewNATSReplicator() 76 | err := replicator.InitializeFromConfig(config) 77 | if err != nil { 78 | return nil, err 79 | } 80 | err = replicator.Start() 81 | if err != nil { 82 | replicator.Stop() 83 | return nil, err 84 | } 85 | 86 | return replicator, nil 87 | } 88 | 89 | func main() { 90 | flag.IntVar(&iterations, "n", 1000, "messages to send, defaults to 1000") 91 | flag.IntVar(&messageSize, "ms", 1024, "message size, defaults to 1024") 92 | flag.StringVar(&natsURL, "nats", "nats://localhost:4222", "nats url, defaults to nats://localhost:4222") 93 | flag.StringVar(&natsURL2, "nats2", "", "nats url for the subscriber side, defaults to nats://localhost:4222") 94 | flag.BoolVar(&direct, "direct", false, "skip the replicator and just use nats") 95 | flag.BoolVar(&repOnly, "rep", false, "only run the replicator") 96 | flag.BoolVar(&pubOnly, "pub", false, "only publish, don't subscribe, useful for testing send times across a long connection") 97 | flag.BoolVar(&subOnly, "sub", false, "only time the reads, useful for testing read times across a long connection, timer starts with first receive") 98 | flag.BoolVar(&showStats, "stats", false, "print replicator stats, if not direct") 99 | flag.BoolVar(&hideReplicatorLog, "hide", false, "hide the replicator log") 100 | flag.BoolVar(&showProgress, "progress", true, "show progress") 101 | flag.StringVar(&in, "in", "", "channel to publish to, and replicate, defaults to a random string") 102 | flag.StringVar(&out, "out", "", "channel to read from and replicate to, defaults to a random string") 103 | flag.Parse() 104 | 105 | var replicator *core.NATSReplicator 106 | var startPub time.Time 107 | var endPub time.Time 108 | var startSub time.Time 109 | var endSub time.Time 110 | var startRep time.Time 111 | var endRep time.Time 112 | 113 | incoming := nuid.Next() 114 | outgoing := nuid.Next() 115 | msgString := strings.Repeat("a", messageSize) 116 | msg := []byte(msgString) 117 | msgLen := len(msg) 118 | pubwg := sync.WaitGroup{} 119 | repwg := sync.WaitGroup{} 120 | subwg := sync.WaitGroup{} 121 | interval := int(iterations / 10) 122 | repTimeout := make(chan bool, 1) 123 | 124 | if natsURL2 == "" { 125 | natsURL2 = natsURL 126 | } 127 | 128 | if in != "" { 129 | incoming = in 130 | } 131 | 132 | if out != "" { 133 | outgoing = out 134 | } 135 | 136 | if pubOnly || subOnly { 137 | direct = true 138 | log.Printf("Pub and sub only mode runs with direct mode, no replicator is used") 139 | } 140 | 141 | if direct { 142 | log.Printf("Direct mode uses the same nats url for both connections") 143 | if in == "" && out == "" { 144 | log.Printf("Unless custom subjects are set, the same subject is used for read/write in direct mode") 145 | outgoing = incoming 146 | } 147 | natsURL2 = natsURL 148 | } 149 | 150 | nc, err := nats.Connect(natsURL, nats.Timeout(time.Second*5), nats.MaxReconnects(5), nats.ReconnectWait(time.Second*5)) 151 | if err != nil { 152 | log.Fatalf("error connecting to nats, %s", err.Error()) 153 | } 154 | defer nc.Close() 155 | 156 | nc2, err := nats.Connect(natsURL2, nats.Timeout(time.Second*5), nats.MaxReconnects(5), nats.ReconnectWait(time.Second*5)) 157 | if err != nil { 158 | log.Fatalf("error connecting to nats, %s", err.Error()) 159 | } 160 | defer nc.Close() 161 | 162 | log.Printf("Incoming/Replicated subject %s : Outgoing/Subscribed subject: %s", incoming, outgoing) 163 | 164 | if !pubOnly && !repOnly { 165 | subwg.Add(iterations) 166 | subCount := 0 167 | _, err := nc.Subscribe(outgoing, func(msg *nats.Msg) { 168 | if subCount == 0 { 169 | startSub = time.Now() // start timing on the first message 170 | } 171 | subCount++ 172 | if (subCount%interval == 0 || subCount == iterations) && showProgress { 173 | log.Printf("received count = %d", subCount) 174 | } 175 | 176 | if len(msg.Data) != msgLen { 177 | log.Fatalf("received message that is the wrong size %d != %d", len(msg.Data), msgLen) 178 | } 179 | 180 | if subCount <= iterations { 181 | subwg.Done() 182 | } 183 | }) 184 | 185 | if err != nil { 186 | log.Fatalf("error subscribing to %s, %s", outgoing, err.Error()) 187 | } 188 | } 189 | 190 | if !direct { 191 | connect := []conf.ConnectorConfig{ 192 | { 193 | Type: "NATSToNATS", 194 | IncomingConnection: "nats", 195 | OutgoingConnection: "nats2", 196 | IncomingSubject: incoming, 197 | OutgoingSubject: outgoing, 198 | }, 199 | } 200 | 201 | var err error 202 | replicator, err = startReplicator(connect) 203 | if err != nil { 204 | log.Fatalf("error starting replicator, %s", err.Error()) 205 | } 206 | 207 | // Start trying to capture the replicator ack activity 208 | repwg.Add(1) 209 | go func() { 210 | lastRepInterval := int64(0) 211 | repTicker := time.NewTicker(50 * time.Millisecond) 212 | loop: 213 | for { 214 | select { 215 | case t := <-repTicker.C: 216 | reqcount := replicator.SafeStats().RequestCount 217 | if reqcount >= 0 && startRep.IsZero() { 218 | startRep = t 219 | } 220 | 221 | curInterval := reqcount / int64(interval) 222 | 223 | if curInterval > lastRepInterval && showProgress { 224 | lastRepInterval = curInterval 225 | log.Printf("replicated count = %d", reqcount) 226 | } 227 | 228 | if reqcount >= int64(iterations) { 229 | endRep = t 230 | break loop 231 | } 232 | case <-repTimeout: 233 | break loop 234 | } 235 | } 236 | repwg.Done() 237 | repTicker.Stop() 238 | }() 239 | } 240 | 241 | if !subOnly && !repOnly { 242 | log.Printf("Sending %d messages of size %d bytes...", iterations, messageSize) 243 | pubwg.Add(iterations) 244 | startPub = time.Now() 245 | for i := 0; i < iterations; i++ { 246 | err := nc.Publish(incoming, msg) 247 | if err != nil { 248 | log.Fatalf("error publishing message, %s", err.Error()) 249 | } 250 | if (i%interval == 0) && showProgress { 251 | log.Printf("pub count = %d", i) 252 | } 253 | } 254 | if showProgress { 255 | log.Printf("pub count = %d", iterations) 256 | } 257 | endPub = time.Now() 258 | } 259 | 260 | if !pubOnly && !repOnly { 261 | subwg.Wait() 262 | endSub = time.Now() 263 | } 264 | 265 | if !direct { 266 | 267 | if !repOnly { 268 | log.Printf("Waiting for replicator before we shut it down, timeout of %d seconds", 10) 269 | go func() { 270 | stop := time.Now().Add(10 * time.Second) 271 | ticker := time.NewTicker(500 * time.Millisecond) 272 | for t := range ticker.C { 273 | if t.After(stop) { 274 | repTimeout <- true 275 | break 276 | } 277 | } 278 | ticker.Stop() 279 | }() 280 | } else { 281 | log.Printf("Waiting for replicator without timeout") 282 | } 283 | 284 | repwg.Wait() 285 | 286 | stats := replicator.SafeStats() 287 | statsJSON, _ := json.MarshalIndent(stats, "", " ") 288 | 289 | replicator.Stop() 290 | 291 | if showStats { 292 | log.Printf("Replicator Stats:\n\n%s\n", statsJSON) 293 | } 294 | } 295 | 296 | nc2.Close() 297 | nc.Close() 298 | 299 | if !direct && endRep.IsZero() { 300 | log.Printf("Test Failed, replicator did replicate all the messages within the timeout") 301 | return 302 | } 303 | 304 | var totalDiff time.Duration 305 | 306 | if pubOnly { 307 | totalDiff = endPub.Sub(startPub) 308 | log.Printf("Sent %d messages to a subject %s", iterations, totalDiff) 309 | } else if subOnly { 310 | totalDiff = endSub.Sub(startSub) 311 | log.Printf("Read %d messages from a subject in %s", iterations, totalDiff) 312 | } else if direct { 313 | totalDiff = endSub.Sub(startPub) 314 | log.Printf("Sent %d messages through a subject to a subscriber in %s", iterations, totalDiff) 315 | log.Printf("Total messages moved were %d", 2*iterations) 316 | } else if repOnly { 317 | totalDiff = endRep.Sub(startRep) 318 | log.Printf("Replicated %d messages %s", iterations, totalDiff) 319 | log.Printf("Total messages moved were %d", 2*iterations) 320 | } else { 321 | totalDiff = endSub.Sub(startPub) 322 | log.Printf("Sent %d messages through a subject to the replicator and read from another subject in %s", iterations, totalDiff) 323 | log.Printf("Total messages moved were %d", 4*iterations) 324 | } 325 | 326 | totalRate := float64(iterations) / float64(totalDiff.Seconds()) 327 | totalSizeRate := float64(messageSize) * totalRate / (1024 * 1024) 328 | log.Printf("Total stats - %.2f msgs/sec ~ %.2f MB/sec (using %d full paths)", totalRate, totalSizeRate, iterations) 329 | 330 | if !repOnly { 331 | pubDiff := endPub.Sub(startPub) 332 | pubRate := float64(iterations) / float64(pubDiff.Seconds()) 333 | pubSizeRate := float64(messageSize) * pubRate / (1024 * 1024) 334 | log.Printf(" Pub stats - %.2f msgs/sec ~ %.2f MB/sec", pubRate, pubSizeRate) 335 | } 336 | 337 | if !pubOnly && !repOnly { 338 | subDiff := endSub.Sub(startSub) 339 | subRate := float64(iterations) / float64(subDiff.Seconds()) 340 | subSizeRate := float64(messageSize) * subRate / (1024 * 1024) 341 | log.Printf(" Sub stats - %.2f msgs/sec ~ %.2f MB/sec", subRate, subSizeRate) 342 | } 343 | 344 | if !direct { 345 | repDiff := endRep.Sub(startRep) 346 | repRate := float64(iterations) / float64(repDiff.Seconds()) 347 | repSizeRate := float64(messageSize) * repRate / (1024 * 1024) 348 | log.Printf(" Rep stats - %.2f msgs/sec ~ %.2f MB/sec", repRate, repSizeRate) 349 | } 350 | } 351 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------