├── .github └── workflows │ └── test.yml ├── .gitignore ├── AUTHORS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── ChangeLog.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── apps ├── nsq_stat │ └── nsq_stat.go ├── nsq_tail │ └── nsq_tail.go ├── nsq_to_file │ ├── file_logger.go │ ├── nsq_to_file.go │ ├── options.go │ ├── strftime.go │ └── topic_discoverer.go ├── nsq_to_http │ ├── http.go │ ├── nsq_to_http.go │ └── nsq_to_http_test.go ├── nsq_to_nsq │ └── nsq_to_nsq.go ├── nsqadmin │ ├── main.go │ ├── main_test.go │ └── options.go ├── nsqd │ ├── README.md │ ├── main.go │ ├── main_test.go │ └── options.go ├── nsqlookupd │ ├── README.md │ ├── main.go │ ├── main_test.go │ └── options.go └── to_nsq │ ├── README.md │ └── to_nsq.go ├── bench.sh ├── bench ├── bench.py ├── bench_channels │ └── bench_channels.go ├── bench_reader │ └── bench_reader.go ├── bench_writer │ └── bench_writer.go └── requirements.txt ├── contrib ├── nsq.spec ├── nsqadmin.cfg.example ├── nsqd.cfg.example └── nsqlookupd.cfg.example ├── coverage.sh ├── dist.sh ├── fmt.sh ├── go.mod ├── go.sum ├── internal ├── app │ ├── float_array.go │ └── string_array.go ├── auth │ └── authorizations.go ├── clusterinfo │ ├── data.go │ ├── producer_test.go │ └── types.go ├── dirlock │ ├── dirlock.go │ ├── dirlock_illumos.go │ └── dirlock_windows.go ├── http_api │ ├── api_request.go │ ├── api_response.go │ ├── compress.go │ ├── http_server.go │ ├── req_params.go │ └── topic_channel_args.go ├── lg │ ├── lg.go │ └── lg_test.go ├── pqueue │ ├── pqueue.go │ └── pqueue_test.go ├── protocol │ ├── byte_base10.go │ ├── byte_base10_test.go │ ├── errors.go │ ├── names.go │ ├── protocol.go │ └── tcp_server.go ├── quantile │ ├── aggregate.go │ └── quantile.go ├── statsd │ ├── client.go │ └── host.go ├── stringy │ ├── slice.go │ ├── slice_test.go │ └── template.go ├── test │ ├── assertions.go │ ├── fakes.go │ └── logger.go ├── util │ ├── rand.go │ ├── unix_socket.go │ ├── util_test.go │ └── wait_group_wrapper.go ├── version │ └── binary.go └── writers │ ├── boundary_buffered_writer.go │ └── spread_writer.go ├── nsqadmin ├── .eslintrc ├── README.md ├── gulp ├── gulpfile.js ├── http.go ├── http_test.go ├── logger.go ├── notify.go ├── nsqadmin.go ├── nsqadmin_test.go ├── options.go ├── package-lock.json ├── package.json ├── static.go ├── static │ ├── build │ │ ├── base.css │ │ ├── bootstrap.min.css │ │ ├── favicon.png │ │ ├── glyphicons-halflings-regular.eot │ │ ├── glyphicons-halflings-regular.svg │ │ ├── glyphicons-halflings-regular.ttf │ │ ├── glyphicons-halflings-regular.woff │ │ ├── glyphicons-halflings-regular.woff2 │ │ ├── index.html │ │ ├── main.js │ │ ├── main.js.map │ │ ├── nsq_blue.png │ │ └── vendor.js │ ├── css │ │ ├── base.scss │ │ └── bootstrap.min.css │ ├── fonts │ │ ├── glyphicons-halflings-regular.eot │ │ ├── glyphicons-halflings-regular.svg │ │ ├── glyphicons-halflings-regular.ttf │ │ ├── glyphicons-halflings-regular.woff │ │ └── glyphicons-halflings-regular.woff2 │ ├── html │ │ └── index.html │ ├── img │ │ ├── favicon.png │ │ └── nsq_blue.png │ └── js │ │ ├── app_state.js │ │ ├── collections │ │ ├── nodes.js │ │ └── topics.js │ │ ├── lib │ │ ├── ajax_setup.js │ │ ├── handlebars_helpers.js │ │ └── pubsub.js │ │ ├── main.js │ │ ├── models │ │ ├── channel.js │ │ ├── node.js │ │ └── topic.js │ │ ├── router.js │ │ └── views │ │ ├── app.js │ │ ├── base.js │ │ ├── channel.hbs │ │ ├── channel.js │ │ ├── counter.hbs │ │ ├── counter.js │ │ ├── error.hbs │ │ ├── header.hbs │ │ ├── header.js │ │ ├── lookup.hbs │ │ ├── lookup.js │ │ ├── node.hbs │ │ ├── node.js │ │ ├── nodes.hbs │ │ ├── nodes.js │ │ ├── spinner.hbs │ │ ├── topic.hbs │ │ ├── topic.js │ │ ├── topics.hbs │ │ ├── topics.js │ │ └── warning.hbs └── test │ ├── ca.key │ ├── ca.pem │ ├── ca.srl │ ├── cert.pem │ ├── client.key │ ├── client.pem │ ├── client.req │ ├── key.pem │ ├── server.key │ ├── server.pem │ └── server.req ├── nsqd ├── README.md ├── backend_queue.go ├── buffer_pool.go ├── channel.go ├── channel_test.go ├── client_v2.go ├── dqname.go ├── dqname_windows.go ├── dummy_backend_queue.go ├── guid.go ├── guid_test.go ├── http.go ├── http_test.go ├── in_flight_pqueue.go ├── in_flight_pqueue_test.go ├── logger.go ├── lookup.go ├── lookup_peer.go ├── message.go ├── nsqd.go ├── nsqd_test.go ├── options.go ├── protocol_v2.go ├── protocol_v2_test.go ├── protocol_v2_unixsocket_test.go ├── stats.go ├── stats_test.go ├── statsd.go ├── tcp.go ├── test │ ├── cert.sh │ ├── certs │ │ ├── ca.key │ │ ├── ca.pem │ │ ├── ca.srl │ │ ├── cert.pem │ │ ├── client.key │ │ ├── client.pem │ │ ├── client.req │ │ ├── key.pem │ │ ├── server.key │ │ ├── server.pem │ │ └── server.req │ └── openssl.conf ├── topic.go └── topic_test.go ├── nsqlookupd ├── README.md ├── client_v1.go ├── http.go ├── http_test.go ├── logger.go ├── lookup_protocol_v1.go ├── lookup_protocol_v1_test.go ├── nsqlookupd.go ├── nsqlookupd_test.go ├── options.go ├── registration_db.go ├── registration_db_test.go └── tcp.go └── test.sh /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | 3 | on: 4 | push: {branches: [master]} 5 | pull_request: {branches: [master]} 6 | 7 | jobs: 8 | test: 9 | runs-on: ubuntu-20.04 10 | timeout-minutes: 30 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | go: ["1.21.x", "1.22.x", "1.23.x"] 15 | arch: ["amd64", "386"] 16 | 17 | env: 18 | GOARCH: "${{matrix.arch}}" 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | 23 | - uses: WillAbides/setup-go-faster@v1.7.0 24 | with: 25 | go-version: ${{matrix.go}} 26 | 27 | - name: build 28 | run: make all 29 | 30 | - name: test 31 | run: ./test.sh 32 | 33 | staticcheck: 34 | runs-on: ubuntu-20.04 35 | steps: 36 | - uses: actions/checkout@v2 37 | 38 | - uses: dominikh/staticcheck-action@v1.3.1 39 | with: 40 | version: "2024.1.1" 41 | install-go: false 42 | 43 | code-coverage: 44 | runs-on: ubuntu-20.04 45 | steps: 46 | - uses: actions/checkout@v2 47 | 48 | - name: install goveralls 49 | run: go install github.com/mattn/goveralls@latest 50 | 51 | - name: send coverage 52 | env: 53 | COVERALLS_TOKEN: ${{secrets.GITHUB_TOKEN}} 54 | run: ./coverage.sh --coveralls 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | dist/ 3 | .cover/ 4 | profile/ 5 | 6 | # nsqd data from testing 7 | *.dat 8 | 9 | # nsqadmin 10 | node_modules 11 | 12 | # apps 13 | apps/nsqlookupd/nsqlookupd 14 | apps/nsqd/nsqd 15 | apps/nsqadmin/nsqadmin 16 | apps/nsq_to_nsq/nsq_to_nsq 17 | apps/nsq_to_file/nsq_to_file 18 | apps/nsq_pubsub/nsq_pubsub 19 | apps/nsq_to_http/nsq_to_http 20 | apps/nsq_tail/nsq_tail 21 | apps/nsq_stat/nsq_stat 22 | apps/to_nsq/to_nsq 23 | bench/bench_reader/bench_reader 24 | bench/bench_writer/bench_writer 25 | bench/bench_channels/bench_channels 26 | 27 | # Go.gitignore 28 | 29 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 30 | *.o 31 | *.a 32 | *.so 33 | 34 | # Folders 35 | _obj 36 | _test 37 | 38 | # Architecture specific extensions/prefixes 39 | *.[568vq] 40 | [568vq].out 41 | 42 | *.cgo1.go 43 | *.cgo2.c 44 | _cgo_defun.c 45 | _cgo_gotypes.go 46 | _cgo_export.* 47 | 48 | _testmain.go 49 | 50 | *.exe 51 | 52 | 53 | # vim stuff 54 | *.sw[op] 55 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # For a complete listing, see https://github.com/nsqio/nsq/graphs/contributors 2 | 3 | # Original Authors 4 | 5 | Matt Reiferson 6 | Jehiah Czebotar 7 | 8 | # Maintainers 9 | 10 | Pierce Lopez 11 | 12 | # Disclaimer 13 | 14 | Matt Reiferson's contributions to this project are being made solely in a personal capacity 15 | and does not convey any rights to any intellectual property of any third parties. 16 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of fostering an open and 4 | welcoming community, we pledge to respect all people who contribute through reporting issues, 5 | posting feature requests, updating documentation, submitting pull requests or patches, and other 6 | activities. 7 | 8 | We are committed to making participation in this project a harassment-free experience for everyone, 9 | regardless of level of experience, gender, gender identity and expression, sexual orientation, 10 | disability, personal appearance, body size, race, ethnicity, age, religion, or nationality. 11 | 12 | Examples of unacceptable behavior by participants include: 13 | 14 | * The use of sexualized language or imagery 15 | * Personal attacks 16 | * Trolling or insulting/derogatory comments 17 | * Public or private harassment 18 | * Publishing other's private information, such as physical or electronic addresses, without explicit permission 19 | * Other unethical or unprofessional conduct. 20 | 21 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, 22 | code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. By 23 | adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently 24 | applying these principles to every aspect of managing this project. Project maintainers who do not 25 | follow or enforce the Code of Conduct may be permanently removed from the project team. 26 | 27 | This code of conduct applies both within project spaces and in public spaces when an individual is 28 | representing the project or its community. 29 | 30 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an 31 | issue or contacting one or more of the project maintainers. 32 | 33 | This Code of Conduct is adapted from the [Contributor Covenant][1], version 1.2.0, available at 34 | [http://contributor-covenant.org/version/1/2/0/][2]. 35 | 36 | [1]: http://contributor-covenant.org 37 | [2]: http://contributor-covenant.org/version/1/2/0/ 38 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thanks for your interest in contributing to NSQ! 4 | 5 | ## Code of Conduct 6 | 7 | Help us keep NSQ open and inclusive. Please read and follow our [Code of Conduct](CODE_OF_CONDUCT.md). 8 | 9 | ## Getting Started 10 | 11 | * make sure you have a [GitHub account](https://github.com/signup/free) 12 | * submit a ticket for your issue, assuming one does not already exist 13 | * clearly describe the issue including steps to reproduce when it is a bug 14 | * identify specific versions of the binaries and client libraries 15 | * fork the repository on GitHub 16 | 17 | ## Making Changes 18 | 19 | * create a branch from where you want to base your work 20 | * we typically name branches according to the following format: `helpful_name_` 21 | * make commits of logical units 22 | * make sure your commit messages are in a clear and readable format, example: 23 | 24 | ``` 25 | nsqd: fixed bug in protocol_v2 26 | 27 | * update the message pump to properly account for RDYness 28 | * cleanup variable names 29 | * ... 30 | ``` 31 | 32 | * if you're fixing a bug or adding functionality it probably makes sense to write a test 33 | * make sure to run `fmt.sh` and `test.sh` in the root of the repo to ensure that your code is 34 | properly formatted and that tests pass (we use GitHub Actions for continuous integration) 35 | 36 | ## Submitting Changes 37 | 38 | * push your changes to your branch in your fork of the repository 39 | * submit a pull request against nsqio's repository 40 | * comment in the pull request when you're ready for the changes to be reviewed: `"ready for review"` 41 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine AS build 2 | 3 | RUN apk update && apk add make gcc musl-dev 4 | 5 | RUN mkdir -p /go/src/github.com/nsqio/nsq 6 | COPY . /go/src/github.com/nsqio/nsq 7 | WORKDIR /go/src/github.com/nsqio/nsq 8 | 9 | RUN CGO_ENABLED=0 make BLDDIR=/tmp/nsq PREFIX=/opt/nsq BLDFLAGS='-ldflags="-s -w"' install 10 | 11 | FROM alpine:latest 12 | 13 | EXPOSE 4150 4151 4160 4161 4170 4171 14 | 15 | RUN mkdir -p /data 16 | WORKDIR /data 17 | 18 | # Optional volumes (explicitly configure with "docker run -v ...") 19 | # /data - used by nsqd for persistent storage across restarts 20 | # /etc/ssl/certs - for SSL Root CA certificates from host 21 | 22 | COPY --from=build /opt/nsq/bin/ /usr/local/bin/ 23 | RUN ln -s /usr/local/bin/*nsq* / \ 24 | && ln -s /usr/local/bin/*nsq* /bin/ 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy 2 | of this software and associated documentation files (the "Software"), to deal 3 | in the Software without restriction, including without limitation the rights 4 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 5 | copies of the Software, and to permit persons to whom the Software is 6 | furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in 9 | all copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 17 | THE SOFTWARE. 18 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PREFIX=/usr/local 2 | BINDIR=${PREFIX}/bin 3 | DESTDIR= 4 | BLDDIR=build 5 | BLDFLAGS= 6 | EXT= 7 | ifeq (${GOOS},windows) 8 | EXT=.exe 9 | endif 10 | 11 | APPS = nsqd nsqlookupd nsqadmin nsq_to_nsq nsq_to_file nsq_to_http nsq_tail nsq_stat to_nsq 12 | all: $(APPS) 13 | 14 | $(BLDDIR)/nsqd: $(wildcard apps/nsqd/*.go nsqd/*.go nsq/*.go internal/*/*.go) 15 | $(BLDDIR)/nsqlookupd: $(wildcard apps/nsqlookupd/*.go nsqlookupd/*.go nsq/*.go internal/*/*.go) 16 | $(BLDDIR)/nsqadmin: $(wildcard apps/nsqadmin/*.go nsqadmin/*.go nsqadmin/templates/*.go internal/*/*.go) 17 | $(BLDDIR)/nsq_to_nsq: $(wildcard apps/nsq_to_nsq/*.go nsq/*.go internal/*/*.go) 18 | $(BLDDIR)/nsq_to_file: $(wildcard apps/nsq_to_file/*.go nsq/*.go internal/*/*.go) 19 | $(BLDDIR)/nsq_to_http: $(wildcard apps/nsq_to_http/*.go nsq/*.go internal/*/*.go) 20 | $(BLDDIR)/nsq_tail: $(wildcard apps/nsq_tail/*.go nsq/*.go internal/*/*.go) 21 | $(BLDDIR)/nsq_stat: $(wildcard apps/nsq_stat/*.go internal/*/*.go) 22 | $(BLDDIR)/to_nsq: $(wildcard apps/to_nsq/*.go internal/*/*.go) 23 | 24 | $(BLDDIR)/%: 25 | @mkdir -p $(dir $@) 26 | go build ${BLDFLAGS} -o $@ ./apps/$* 27 | 28 | $(APPS): %: $(BLDDIR)/% 29 | 30 | clean: 31 | rm -fr $(BLDDIR) 32 | 33 | .PHONY: install clean all 34 | .PHONY: $(APPS) 35 | 36 | install: $(APPS) 37 | install -m 755 -d ${DESTDIR}${BINDIR} 38 | for APP in $^ ; do install -m 755 ${BLDDIR}/$$APP ${DESTDIR}${BINDIR}/$$APP${EXT} ; done 39 | -------------------------------------------------------------------------------- /apps/nsq_to_file/options.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "time" 4 | 5 | type Options struct { 6 | Topics []string `flag:"topic"` 7 | TopicPattern string `flag:"topic-pattern"` 8 | TopicRefreshInterval time.Duration `flag:"topic-refresh"` 9 | Channel string `flag:"channel"` 10 | 11 | NSQDTCPAddrs []string `flag:"nsqd-tcp-address"` 12 | NSQLookupdHTTPAddrs []string `flag:"lookupd-http-address"` 13 | ConsumerOpts []string `flag:"consumer-opt"` 14 | MaxInFlight int `flag:"max-in-flight"` 15 | HTTPClientConnectTimeout time.Duration `flag:"http-client-connect-timeout"` 16 | HTTPClientRequestTimeout time.Duration `flag:"http-client-request-timeout"` 17 | 18 | LogPrefix string `flag:"log-prefix"` 19 | LogLevel string `flag:"log-level"` 20 | OutputDir string `flag:"output-dir"` 21 | WorkDir string `flag:"work-dir"` 22 | DatetimeFormat string `flag:"datetime-format"` 23 | FilenameFormat string `flag:"filename-format"` 24 | HostIdentifier string `flag:"host-identifier"` 25 | GZIPLevel int `flag:"gzip-level"` 26 | GZIP bool `flag:"gzip"` 27 | SkipEmptyFiles bool `flag:"skip-empty-files"` 28 | RotateSize int64 `flag:"rotate-size"` 29 | RotateInterval time.Duration `flag:"rotate-interval"` 30 | SyncInterval time.Duration `flag:"sync-interval"` 31 | } 32 | 33 | func NewOptions() *Options { 34 | return &Options{ 35 | LogPrefix: "[nsq_to_file] ", 36 | LogLevel: "info", 37 | Channel: "nsq_to_file", 38 | MaxInFlight: 200, 39 | OutputDir: "/tmp", 40 | DatetimeFormat: "%Y-%m-%d_%H", 41 | FilenameFormat: "...log", 42 | GZIPLevel: 6, 43 | TopicRefreshInterval: time.Minute, 44 | SyncInterval: 30 * time.Second, 45 | HTTPClientConnectTimeout: 2 * time.Second, 46 | HTTPClientRequestTimeout: 5 * time.Second, 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /apps/nsq_to_file/strftime.go: -------------------------------------------------------------------------------- 1 | // COPIED FROM https://github.com/jehiah/go-strftime 2 | package main 3 | 4 | import ( 5 | "time" 6 | ) 7 | 8 | // taken from time/format.go 9 | var conversion = map[string]string{ 10 | /*stdLongMonth */ "B": "January", 11 | /*stdMonth */ "b": "Jan", 12 | // stdNumMonth */ "m": "1", 13 | /*stdZeroMonth */ "m": "01", 14 | /*stdLongWeekDay */ "A": "Monday", 15 | /*stdWeekDay */ "a": "Mon", 16 | // stdDay */ "d": "2", 17 | // stdUnderDay */ "d": "_2", 18 | /*stdZeroDay */ "d": "02", 19 | /*stdHour */ "H": "15", 20 | // stdHour12 */ "I": "3", 21 | /*stdZeroHour12 */ "I": "03", 22 | // stdMinute */ "M": "4", 23 | /*stdZeroMinute */ "M": "04", 24 | // stdSecond */ "S": "5", 25 | /*stdZeroSecond */ "S": "05", 26 | /*stdLongYear */ "Y": "2006", 27 | /*stdYear */ "y": "06", 28 | /*stdPM */ "p": "PM", 29 | // stdpm */ "p": "pm", 30 | /*stdTZ */ "Z": "MST", 31 | // stdISO8601TZ */ "z": "Z0700", // prints Z for UTC 32 | // stdISO8601ColonTZ */ "z": "Z07:00", // prints Z for UTC 33 | /*stdNumTZ */ "z": "-0700", // always numeric 34 | // stdNumShortTZ */ "b": "-07", // always numeric 35 | // stdNumColonTZ */ "b": "-07:00", // always numeric 36 | "%": "%", 37 | } 38 | 39 | // This is an alternative to time.Format because no one knows 40 | // what date 040305 is supposed to create when used as a 'layout' string 41 | // this takes standard strftime format options. For a complete list 42 | // of format options see http://strftime.org/ 43 | func strftime(format string, t time.Time) string { 44 | layout := "" 45 | length := len(format) 46 | for i := 0; i < length; i++ { 47 | if format[i] == '%' && i <= length-2 { 48 | if layoutCmd, ok := conversion[format[i+1:i+2]]; ok { 49 | layout = layout + layoutCmd 50 | i++ 51 | continue 52 | } 53 | } 54 | layout = layout + format[i:i+1] 55 | } 56 | return t.Format(layout) 57 | } 58 | -------------------------------------------------------------------------------- /apps/nsq_to_file/topic_discoverer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "regexp" 6 | "sync" 7 | "time" 8 | 9 | "github.com/nsqio/go-nsq" 10 | "github.com/nsqio/nsq/internal/clusterinfo" 11 | "github.com/nsqio/nsq/internal/http_api" 12 | "github.com/nsqio/nsq/internal/lg" 13 | ) 14 | 15 | type TopicDiscoverer struct { 16 | logf lg.AppLogFunc 17 | opts *Options 18 | ci *clusterinfo.ClusterInfo 19 | topics map[string]*FileLogger 20 | hupChan chan os.Signal 21 | termChan chan os.Signal 22 | wg sync.WaitGroup 23 | cfg *nsq.Config 24 | } 25 | 26 | func newTopicDiscoverer(logf lg.AppLogFunc, opts *Options, cfg *nsq.Config, hupChan chan os.Signal, termChan chan os.Signal) *TopicDiscoverer { 27 | client := http_api.NewClient(nil, opts.HTTPClientConnectTimeout, opts.HTTPClientRequestTimeout) 28 | return &TopicDiscoverer{ 29 | logf: logf, 30 | opts: opts, 31 | ci: clusterinfo.New(nil, client), 32 | topics: make(map[string]*FileLogger), 33 | hupChan: hupChan, 34 | termChan: termChan, 35 | cfg: cfg, 36 | } 37 | } 38 | 39 | func (t *TopicDiscoverer) updateTopics(topics []string) { 40 | for _, topic := range topics { 41 | if _, ok := t.topics[topic]; ok { 42 | continue 43 | } 44 | 45 | if !t.isTopicAllowed(topic) { 46 | t.logf(lg.WARN, "skipping topic %s (doesn't match pattern %s)", topic, t.opts.TopicPattern) 47 | continue 48 | } 49 | 50 | fl, err := NewFileLogger(t.logf, t.opts, topic, t.cfg) 51 | if err != nil { 52 | t.logf(lg.ERROR, "couldn't create logger for new topic %s: %s", topic, err) 53 | continue 54 | } 55 | t.topics[topic] = fl 56 | 57 | t.wg.Add(1) 58 | go func(fl *FileLogger) { 59 | fl.router() 60 | t.wg.Done() 61 | }(fl) 62 | } 63 | } 64 | 65 | func (t *TopicDiscoverer) run() { 66 | var ticker <-chan time.Time 67 | if len(t.opts.Topics) == 0 { 68 | ticker = time.Tick(t.opts.TopicRefreshInterval) 69 | } 70 | t.updateTopics(t.opts.Topics) 71 | forloop: 72 | for { 73 | select { 74 | case <-ticker: 75 | newTopics, err := t.ci.GetLookupdTopics(t.opts.NSQLookupdHTTPAddrs) 76 | if err != nil { 77 | t.logf(lg.ERROR, "could not retrieve topic list: %s", err) 78 | continue 79 | } 80 | t.updateTopics(newTopics) 81 | case <-t.termChan: 82 | for _, fl := range t.topics { 83 | close(fl.termChan) 84 | } 85 | break forloop 86 | case <-t.hupChan: 87 | for _, fl := range t.topics { 88 | fl.hupChan <- true 89 | } 90 | } 91 | } 92 | t.wg.Wait() 93 | } 94 | 95 | func (t *TopicDiscoverer) isTopicAllowed(topic string) bool { 96 | if t.opts.TopicPattern == "" { 97 | return true 98 | } 99 | match, err := regexp.MatchString(t.opts.TopicPattern, topic) 100 | if err != nil { 101 | return false 102 | } 103 | return match 104 | } 105 | -------------------------------------------------------------------------------- /apps/nsq_to_http/http.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/nsqio/nsq/internal/version" 9 | ) 10 | 11 | var httpclient *http.Client 12 | var userAgent string 13 | 14 | func init() { 15 | userAgent = fmt.Sprintf("nsq_to_http v%s", version.Binary) 16 | } 17 | 18 | func HTTPGet(endpoint string) (*http.Response, error) { 19 | req, err := http.NewRequest("GET", endpoint, nil) 20 | if err != nil { 21 | return nil, err 22 | } 23 | req.Header.Set("User-Agent", userAgent) 24 | for key, val := range validCustomHeaders { 25 | req.Header.Set(key, val) 26 | } 27 | return httpclient.Do(req) 28 | } 29 | 30 | func HTTPPost(endpoint string, body *bytes.Buffer) (*http.Response, error) { 31 | req, err := http.NewRequest("POST", endpoint, body) 32 | if err != nil { 33 | return nil, err 34 | } 35 | req.Header.Set("User-Agent", userAgent) 36 | req.Header.Set("Content-Type", *contentType) 37 | for key, val := range validCustomHeaders { 38 | req.Header.Set(key, val) 39 | } 40 | return httpclient.Do(req) 41 | } 42 | -------------------------------------------------------------------------------- /apps/nsq_to_http/nsq_to_http_test.go: -------------------------------------------------------------------------------- 1 | // This is an NSQ client that reads the specified topic/channel 2 | // and performs HTTP requests (GET/POST) to the specified endpoints 3 | 4 | package main 5 | 6 | import ( 7 | "reflect" 8 | "testing" 9 | ) 10 | 11 | func TestParseCustomHeaders(t *testing.T) { 12 | type args struct { 13 | strs []string 14 | } 15 | tests := []struct { 16 | name string 17 | args args 18 | want map[string]string 19 | wantErr bool 20 | }{ 21 | { 22 | "Valid Custom Headers", 23 | args{[]string{"header1: value1", "header2:value2", "header3:value3", "header4:value4"}}, 24 | map[string]string{"header1": "value1", "header2": "value2", "header3": "value3", "header4": "value4"}, 25 | false, 26 | }, 27 | { 28 | "Invalid Custom Headers where key is present but no value", 29 | args{[]string{"header1:", "header2:value2", "header3: value3", "header4:value4"}}, 30 | nil, 31 | true, 32 | }, 33 | { 34 | "Invalid Custom Headers where key is not present but value is present", 35 | args{[]string{"header1: value1", ": value2", "header3:value3", "header4:value4"}}, 36 | nil, 37 | true, 38 | }, 39 | { 40 | "Invalid Custom Headers where key and value are not present but ':' is specified", 41 | args{[]string{"header1:value1", "header2:value2", ":", "header4:value4"}}, 42 | nil, 43 | true, 44 | }, 45 | } 46 | for _, tt := range tests { 47 | t.Run(tt.name, func(t *testing.T) { 48 | got, err := parseCustomHeaders(tt.args.strs) 49 | if (err != nil) != tt.wantErr { 50 | t.Errorf("parseCustomHeaders() error = %v, wantErr %v", err, tt.wantErr) 51 | return 52 | } 53 | if !reflect.DeepEqual(got, tt.want) { 54 | t.Errorf("parseCustomHeaders() = %v, want %v", got, tt.want) 55 | } 56 | }) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /apps/nsqadmin/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mreiferson/go-options" 7 | "github.com/nsqio/nsq/internal/lg" 8 | "github.com/nsqio/nsq/internal/test" 9 | "github.com/nsqio/nsq/nsqadmin" 10 | ) 11 | 12 | func TestConfigFlagParsing(t *testing.T) { 13 | opts := nsqadmin.NewOptions() 14 | opts.Logger = test.NewTestLogger(t) 15 | 16 | flagSet := nsqadminFlagSet(opts) 17 | flagSet.Parse([]string{}) 18 | 19 | cfg := config{"log_level": "debug"} 20 | cfg.Validate() 21 | 22 | options.Resolve(opts, flagSet, cfg) 23 | if opts.LogLevel != lg.DEBUG { 24 | t.Fatalf("log level: want debug, got %s", opts.LogLevel.String()) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /apps/nsqadmin/options.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/nsqio/nsq/internal/lg" 7 | ) 8 | 9 | type config map[string]interface{} 10 | 11 | // Validate settings in the config file, and fatal on errors 12 | func (cfg config) Validate() { 13 | if v, exists := cfg["log_level"]; exists { 14 | var t lg.LogLevel 15 | err := t.Set(fmt.Sprintf("%v", v)) 16 | if err == nil { 17 | cfg["log_level"] = t 18 | } else { 19 | logFatal("failed parsing log_level %+v", v) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /apps/nsqd/README.md: -------------------------------------------------------------------------------- 1 | ## nsqd 2 | 3 | `nsqd` is the daemon that receives, queues, and delivers messages to clients. 4 | 5 | Read the [docs](https://nsq.io/components/nsqd.html). 6 | -------------------------------------------------------------------------------- /apps/nsqd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "math/rand" 8 | "os" 9 | "sync" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/BurntSushi/toml" 14 | "github.com/judwhite/go-svc" 15 | "github.com/mreiferson/go-options" 16 | "github.com/nsqio/nsq/internal/lg" 17 | "github.com/nsqio/nsq/internal/version" 18 | "github.com/nsqio/nsq/nsqd" 19 | ) 20 | 21 | type program struct { 22 | once sync.Once 23 | nsqd *nsqd.NSQD 24 | } 25 | 26 | func main() { 27 | prg := &program{} 28 | if err := svc.Run(prg, syscall.SIGINT, syscall.SIGTERM); err != nil { 29 | logFatal("%s", err) 30 | } 31 | } 32 | 33 | func (p *program) Init(env svc.Environment) error { 34 | opts := nsqd.NewOptions() 35 | 36 | flagSet := nsqdFlagSet(opts) 37 | flagSet.Parse(os.Args[1:]) 38 | 39 | rand.Seed(time.Now().UTC().UnixNano()) 40 | 41 | if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) { 42 | fmt.Println(version.String("nsqd")) 43 | os.Exit(0) 44 | } 45 | 46 | var cfg config 47 | configFile := flagSet.Lookup("config").Value.String() 48 | if configFile != "" { 49 | _, err := toml.DecodeFile(configFile, &cfg) 50 | if err != nil { 51 | logFatal("failed to load config file %s - %s", configFile, err) 52 | } 53 | } 54 | cfg.Validate() 55 | 56 | options.Resolve(opts, flagSet, cfg) 57 | 58 | nsqd, err := nsqd.New(opts) 59 | if err != nil { 60 | logFatal("failed to instantiate nsqd - %s", err) 61 | } 62 | p.nsqd = nsqd 63 | 64 | return nil 65 | } 66 | 67 | func (p *program) Start() error { 68 | err := p.nsqd.LoadMetadata() 69 | if err != nil { 70 | logFatal("failed to load metadata - %s", err) 71 | } 72 | err = p.nsqd.PersistMetadata() 73 | if err != nil { 74 | logFatal("failed to persist metadata - %s", err) 75 | } 76 | 77 | go func() { 78 | err := p.nsqd.Main() 79 | if err != nil { 80 | p.Stop() 81 | os.Exit(1) 82 | } 83 | }() 84 | 85 | return nil 86 | } 87 | 88 | func (p *program) Stop() error { 89 | p.once.Do(func() { 90 | p.nsqd.Exit() 91 | }) 92 | return nil 93 | } 94 | 95 | func (p *program) Handle(s os.Signal) error { 96 | return svc.ErrStop 97 | } 98 | 99 | // Context returns a context that will be canceled when nsqd initiates the shutdown 100 | func (p *program) Context() context.Context { 101 | return p.nsqd.Context() 102 | } 103 | 104 | func logFatal(f string, args ...interface{}) { 105 | lg.LogFatal("[nsqd] ", f, args...) 106 | } 107 | -------------------------------------------------------------------------------- /apps/nsqd/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/tls" 5 | "os" 6 | "testing" 7 | 8 | "github.com/BurntSushi/toml" 9 | "github.com/mreiferson/go-options" 10 | "github.com/nsqio/nsq/internal/lg" 11 | "github.com/nsqio/nsq/internal/test" 12 | "github.com/nsqio/nsq/nsqd" 13 | ) 14 | 15 | func TestConfigFlagParsing(t *testing.T) { 16 | opts := nsqd.NewOptions() 17 | opts.Logger = test.NewTestLogger(t) 18 | 19 | flagSet := nsqdFlagSet(opts) 20 | flagSet.Parse([]string{}) 21 | 22 | var cfg config 23 | f, err := os.Open("../../contrib/nsqd.cfg.example") 24 | if err != nil { 25 | t.Fatalf("%s", err) 26 | } 27 | defer f.Close() 28 | toml.NewDecoder(f).Decode(&cfg) 29 | cfg["log_level"] = "debug" 30 | cfg.Validate() 31 | 32 | options.Resolve(opts, flagSet, cfg) 33 | nsqd.New(opts) 34 | 35 | if opts.TLSMinVersion != tls.VersionTLS10 { 36 | t.Errorf("min %#v not expected %#v", opts.TLSMinVersion, tls.VersionTLS10) 37 | } 38 | if opts.LogLevel != lg.DEBUG { 39 | t.Fatalf("log level: want debug, got %s", opts.LogLevel.String()) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /apps/nsqlookupd/README.md: -------------------------------------------------------------------------------- 1 | ## nsqlookupd 2 | 3 | `nsqlookupd` is the daemon that manages topology metadata and serves client requests to 4 | discover the location of topics at runtime. 5 | 6 | Read the [docs](https://nsq.io/components/nsqlookupd.html). 7 | -------------------------------------------------------------------------------- /apps/nsqlookupd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "sync" 9 | "syscall" 10 | 11 | "github.com/BurntSushi/toml" 12 | "github.com/judwhite/go-svc" 13 | "github.com/mreiferson/go-options" 14 | "github.com/nsqio/nsq/internal/lg" 15 | "github.com/nsqio/nsq/internal/version" 16 | "github.com/nsqio/nsq/nsqlookupd" 17 | ) 18 | 19 | func nsqlookupdFlagSet(opts *nsqlookupd.Options) *flag.FlagSet { 20 | flagSet := flag.NewFlagSet("nsqlookupd", flag.ExitOnError) 21 | 22 | flagSet.String("config", "", "path to config file") 23 | flagSet.Bool("version", false, "print version string") 24 | 25 | logLevel := opts.LogLevel 26 | flagSet.Var(&logLevel, "log-level", "set log verbosity: debug, info, warn, error, or fatal") 27 | flagSet.String("log-prefix", "[nsqlookupd] ", "log message prefix") 28 | flagSet.Bool("verbose", false, "[deprecated] has no effect, use --log-level") 29 | 30 | flagSet.String("tcp-address", opts.TCPAddress, ": to listen on for TCP clients") 31 | flagSet.String("http-address", opts.HTTPAddress, ": to listen on for HTTP clients") 32 | flagSet.String("broadcast-address", opts.BroadcastAddress, "address of this lookupd node, (default to the OS hostname)") 33 | 34 | flagSet.Duration("inactive-producer-timeout", opts.InactiveProducerTimeout, "duration of time a producer will remain in the active list since its last ping") 35 | flagSet.Duration("tombstone-lifetime", opts.TombstoneLifetime, "duration of time a producer will remain tombstoned if registration remains") 36 | 37 | return flagSet 38 | } 39 | 40 | type program struct { 41 | once sync.Once 42 | nsqlookupd *nsqlookupd.NSQLookupd 43 | } 44 | 45 | func main() { 46 | prg := &program{} 47 | if err := svc.Run(prg, syscall.SIGINT, syscall.SIGTERM); err != nil { 48 | logFatal("%s", err) 49 | } 50 | } 51 | 52 | func (p *program) Init(env svc.Environment) error { 53 | if env.IsWindowsService() { 54 | dir := filepath.Dir(os.Args[0]) 55 | return os.Chdir(dir) 56 | } 57 | return nil 58 | } 59 | 60 | func (p *program) Start() error { 61 | opts := nsqlookupd.NewOptions() 62 | 63 | flagSet := nsqlookupdFlagSet(opts) 64 | flagSet.Parse(os.Args[1:]) 65 | 66 | if flagSet.Lookup("version").Value.(flag.Getter).Get().(bool) { 67 | fmt.Println(version.String("nsqlookupd")) 68 | os.Exit(0) 69 | } 70 | 71 | var cfg config 72 | configFile := flagSet.Lookup("config").Value.String() 73 | if configFile != "" { 74 | _, err := toml.DecodeFile(configFile, &cfg) 75 | if err != nil { 76 | logFatal("failed to load config file %s - %s", configFile, err) 77 | } 78 | } 79 | cfg.Validate() 80 | 81 | options.Resolve(opts, flagSet, cfg) 82 | nsqlookupd, err := nsqlookupd.New(opts) 83 | if err != nil { 84 | logFatal("failed to instantiate nsqlookupd", err) 85 | } 86 | p.nsqlookupd = nsqlookupd 87 | 88 | go func() { 89 | err := p.nsqlookupd.Main() 90 | if err != nil { 91 | p.Stop() 92 | os.Exit(1) 93 | } 94 | }() 95 | 96 | return nil 97 | } 98 | 99 | func (p *program) Stop() error { 100 | p.once.Do(func() { 101 | p.nsqlookupd.Exit() 102 | }) 103 | return nil 104 | } 105 | 106 | func logFatal(f string, args ...interface{}) { 107 | lg.LogFatal("[nsqlookupd] ", f, args...) 108 | } 109 | -------------------------------------------------------------------------------- /apps/nsqlookupd/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mreiferson/go-options" 7 | "github.com/nsqio/nsq/internal/lg" 8 | "github.com/nsqio/nsq/internal/test" 9 | "github.com/nsqio/nsq/nsqlookupd" 10 | ) 11 | 12 | func TestConfigFlagParsing(t *testing.T) { 13 | opts := nsqlookupd.NewOptions() 14 | opts.Logger = test.NewTestLogger(t) 15 | 16 | flagSet := nsqlookupdFlagSet(opts) 17 | flagSet.Parse([]string{}) 18 | 19 | cfg := config{"log_level": "debug"} 20 | cfg.Validate() 21 | 22 | options.Resolve(opts, flagSet, cfg) 23 | if opts.LogLevel != lg.DEBUG { 24 | t.Fatalf("log level: want debug, got %s", opts.LogLevel.String()) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /apps/nsqlookupd/options.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/nsqio/nsq/internal/lg" 7 | ) 8 | 9 | type config map[string]interface{} 10 | 11 | // Validate settings in the config file, and fatal on errors 12 | func (cfg config) Validate() { 13 | if v, exists := cfg["log_level"]; exists { 14 | var t lg.LogLevel 15 | err := t.Set(fmt.Sprintf("%v", v)) 16 | if err == nil { 17 | cfg["log_level"] = t 18 | } else { 19 | logFatal("failed parsing log_level %+v", v) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /apps/to_nsq/README.md: -------------------------------------------------------------------------------- 1 | # to_nsq 2 | 3 | A tool for publishing to an nsq topic with data from `stdin`. 4 | 5 | ## Usage 6 | 7 | ``` 8 | Usage of ./to_nsq: 9 | -delimiter string 10 | character to split input from stdin (default "\n") 11 | -nsqd-tcp-address value 12 | destination nsqd TCP address (may be given multiple times) 13 | -producer-opt value 14 | option to passthrough to nsq.Producer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config) 15 | -rate int 16 | Throttle messages to n/second. 0 to disable 17 | -topic string 18 | NSQ topic to publish to 19 | ``` 20 | 21 | ### Examples 22 | 23 | Publish each line of a file: 24 | 25 | ```bash 26 | $ cat source.txt | to_nsq -topic="topic" -nsqd-tcp-address="127.0.0.1:4150" 27 | ``` 28 | 29 | Publish three messages, in one go: 30 | 31 | ```bash 32 | $ echo "one,two,three" | to_nsq -delimiter="," -topic="topic" -nsqd-tcp-address="127.0.0.1:4150" 33 | ``` -------------------------------------------------------------------------------- /apps/to_nsq/to_nsq.go: -------------------------------------------------------------------------------- 1 | // This is an NSQ client that publishes incoming messages from 2 | // stdin to the specified topic. 3 | 4 | package main 5 | 6 | import ( 7 | "bufio" 8 | "flag" 9 | "fmt" 10 | "io" 11 | "log" 12 | "os" 13 | "os/signal" 14 | "sync/atomic" 15 | "syscall" 16 | "time" 17 | 18 | "github.com/nsqio/go-nsq" 19 | "github.com/nsqio/nsq/internal/app" 20 | "github.com/nsqio/nsq/internal/version" 21 | ) 22 | 23 | var ( 24 | topic = flag.String("topic", "", "NSQ topic to publish to") 25 | delimiter = flag.String("delimiter", "\n", "character to split input from stdin") 26 | 27 | destNsqdTCPAddrs = app.StringArray{} 28 | ) 29 | 30 | func init() { 31 | flag.Var(&destNsqdTCPAddrs, "nsqd-tcp-address", "destination nsqd TCP address (may be given multiple times)") 32 | } 33 | 34 | func main() { 35 | cfg := nsq.NewConfig() 36 | flag.Var(&nsq.ConfigFlag{cfg}, "producer-opt", "option to passthrough to nsq.Producer (may be given multiple times, http://godoc.org/github.com/nsqio/go-nsq#Config)") 37 | rate := flag.Int64("rate", 0, "Throttle messages to n/second. 0 to disable") 38 | 39 | flag.Parse() 40 | 41 | if len(*topic) == 0 { 42 | log.Fatal("--topic required") 43 | } 44 | 45 | if len(*delimiter) != 1 { 46 | log.Fatal("--delimiter must be a single byte") 47 | } 48 | 49 | stopChan := make(chan bool) 50 | termChan := make(chan os.Signal, 1) 51 | signal.Notify(termChan, syscall.SIGINT, syscall.SIGTERM) 52 | 53 | cfg.UserAgent = fmt.Sprintf("to_nsq/%s go-nsq/%s", version.Binary, nsq.VERSION) 54 | 55 | // make the producers 56 | producers := make(map[string]*nsq.Producer) 57 | for _, addr := range destNsqdTCPAddrs { 58 | producer, err := nsq.NewProducer(addr, cfg) 59 | if err != nil { 60 | log.Fatalf("failed to create nsq.Producer - %s", err) 61 | } 62 | producers[addr] = producer 63 | } 64 | 65 | if len(producers) == 0 { 66 | log.Fatal("--nsqd-tcp-address required") 67 | } 68 | 69 | throttleEnabled := *rate >= 1 70 | balance := int64(1) 71 | // avoid divide by 0 if !throttleEnabled 72 | var interval time.Duration 73 | if throttleEnabled { 74 | interval = time.Second / time.Duration(*rate) 75 | } 76 | go func() { 77 | if !throttleEnabled { 78 | return 79 | } 80 | log.Printf("Throttling messages rate to max:%d/second", *rate) 81 | // every tick increase the number of messages we can send 82 | for range time.Tick(interval) { 83 | n := atomic.AddInt64(&balance, 1) 84 | // if we build up more than 1s of capacity just bound to that 85 | if n > int64(*rate) { 86 | atomic.StoreInt64(&balance, int64(*rate)) 87 | } 88 | } 89 | }() 90 | 91 | r := bufio.NewReader(os.Stdin) 92 | delim := (*delimiter)[0] 93 | go func() { 94 | for { 95 | var err error 96 | if throttleEnabled { 97 | currentBalance := atomic.LoadInt64(&balance) 98 | if currentBalance <= 0 { 99 | time.Sleep(interval) 100 | } 101 | err = readAndPublish(r, delim, producers) 102 | atomic.AddInt64(&balance, -1) 103 | } else { 104 | err = readAndPublish(r, delim, producers) 105 | } 106 | if err != nil { 107 | if err != io.EOF { 108 | log.Fatal(err) 109 | } 110 | close(stopChan) 111 | break 112 | } 113 | } 114 | }() 115 | 116 | select { 117 | case <-termChan: 118 | case <-stopChan: 119 | } 120 | 121 | for _, producer := range producers { 122 | producer.Stop() 123 | } 124 | } 125 | 126 | // readAndPublish reads to the delim from r and publishes the bytes 127 | // to the map of producers. 128 | func readAndPublish(r *bufio.Reader, delim byte, producers map[string]*nsq.Producer) error { 129 | line, readErr := r.ReadBytes(delim) 130 | 131 | if len(line) > 0 { 132 | // trim the delimiter 133 | line = line[:len(line)-1] 134 | } 135 | 136 | if len(line) == 0 { 137 | return readErr 138 | } 139 | 140 | for _, producer := range producers { 141 | err := producer.Publish(*topic, line) 142 | if err != nil { 143 | return err 144 | } 145 | } 146 | 147 | return readErr 148 | } 149 | -------------------------------------------------------------------------------- /bench.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | readonly messageSize="${1:-200}" 3 | readonly batchSize="${2:-200}" 4 | readonly memQueueSize="${3:-1000000}" 5 | readonly dataPath="${4:-}" 6 | set -e 7 | set -u 8 | 9 | echo "# using --mem-queue-size=$memQueueSize --data-path=$dataPath --size=$messageSize --batch-size=$batchSize" 10 | echo "# compiling/running nsqd" 11 | pushd apps/nsqd >/dev/null 12 | go build 13 | rm -f *.dat 14 | ./nsqd --mem-queue-size=$memQueueSize --data-path=$dataPath >/dev/null 2>&1 & 15 | nsqd_pid=$! 16 | popd >/dev/null 17 | 18 | cleanup() { 19 | kill -9 $nsqd_pid 20 | rm -f nsqd/*.dat 21 | } 22 | trap cleanup INT TERM EXIT 23 | 24 | sleep 0.3 25 | echo "# creating topic/channel" 26 | curl --silent 'http://127.0.0.1:4151/create_topic?topic=sub_bench' >/dev/null 2>&1 27 | curl --silent 'http://127.0.0.1:4151/create_channel?topic=sub_bench&channel=ch' >/dev/null 2>&1 28 | 29 | echo "# compiling bench_reader/bench_writer" 30 | pushd bench >/dev/null 31 | for app in bench_reader bench_writer; do 32 | pushd $app >/dev/null 33 | go build 34 | popd >/dev/null 35 | done 36 | popd >/dev/null 37 | 38 | echo -n "PUB: " 39 | bench/bench_writer/bench_writer --size=$messageSize --batch-size=$batchSize 2>&1 40 | 41 | curl -s -o cpu.pprof http://127.0.0.1:4151/debug/pprof/profile & 42 | pprof_pid=$! 43 | 44 | echo -n "SUB: " 45 | bench/bench_reader/bench_reader --size=$messageSize --channel=ch 2>&1 46 | 47 | echo "waiting for pprof..." 48 | wait $pprof_pid 49 | -------------------------------------------------------------------------------- /bench/bench_channels/bench_channels.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "net" 8 | "sync" 9 | "time" 10 | 11 | "github.com/nsqio/go-nsq" 12 | ) 13 | 14 | var ( 15 | num = flag.Int("num", 10000, "num channels") 16 | tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", ": to connect to nsqd") 17 | ) 18 | 19 | func main() { 20 | flag.Parse() 21 | var wg sync.WaitGroup 22 | 23 | goChan := make(chan int) 24 | rdyChan := make(chan int) 25 | for j := 0; j < *num; j++ { 26 | wg.Add(1) 27 | go func(id int) { 28 | subWorker(*num, *tcpAddress, fmt.Sprintf("t%d", j), "ch", rdyChan, goChan, id) 29 | wg.Done() 30 | }(j) 31 | <-rdyChan 32 | time.Sleep(5 * time.Millisecond) 33 | } 34 | 35 | close(goChan) 36 | wg.Wait() 37 | } 38 | 39 | func subWorker(n int, tcpAddr string, 40 | topic string, channel string, 41 | rdyChan chan int, goChan chan int, id int) { 42 | conn, err := net.DialTimeout("tcp", tcpAddr, time.Second) 43 | if err != nil { 44 | panic(err.Error()) 45 | } 46 | conn.Write(nsq.MagicV2) 47 | rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) 48 | ci := make(map[string]interface{}) 49 | ci["client_id"] = "test" 50 | cmd, _ := nsq.Identify(ci) 51 | cmd.WriteTo(rw) 52 | nsq.Subscribe(topic, channel).WriteTo(rw) 53 | rdyCount := 1 54 | rdy := rdyCount 55 | rdyChan <- 1 56 | <-goChan 57 | nsq.Ready(rdyCount).WriteTo(rw) 58 | rw.Flush() 59 | nsq.ReadResponse(rw) 60 | nsq.ReadResponse(rw) 61 | for { 62 | resp, err := nsq.ReadResponse(rw) 63 | if err != nil { 64 | panic(err.Error()) 65 | } 66 | frameType, data, err := nsq.UnpackResponse(resp) 67 | if err != nil { 68 | panic(err.Error()) 69 | } 70 | if frameType == nsq.FrameTypeError { 71 | panic(string(data)) 72 | } else if frameType == nsq.FrameTypeResponse { 73 | nsq.Nop().WriteTo(rw) 74 | rw.Flush() 75 | continue 76 | } 77 | msg, err := nsq.DecodeMessage(data) 78 | if err != nil { 79 | panic(err.Error()) 80 | } 81 | nsq.Finish(msg.ID).WriteTo(rw) 82 | rdy-- 83 | if rdy == 0 { 84 | nsq.Ready(rdyCount).WriteTo(rw) 85 | rdy = rdyCount 86 | rw.Flush() 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /bench/bench_reader/bench_reader.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "net" 10 | "runtime" 11 | "sync" 12 | "sync/atomic" 13 | "time" 14 | 15 | "github.com/nsqio/go-nsq" 16 | ) 17 | 18 | var ( 19 | runfor = flag.Duration("runfor", 10*time.Second, "duration of time to run") 20 | tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", ": to connect to nsqd") 21 | size = flag.Int("size", 200, "size of messages") 22 | topic = flag.String("topic", "sub_bench", "topic to receive messages on") 23 | channel = flag.String("channel", "ch", "channel to receive messages on") 24 | deadline = flag.String("deadline", "", "deadline to start the benchmark run") 25 | rdy = flag.Int("rdy", 2500, "RDY count to use") 26 | ) 27 | 28 | var totalMsgCount int64 29 | 30 | func main() { 31 | flag.Parse() 32 | var wg sync.WaitGroup 33 | 34 | log.SetPrefix("[bench_reader] ") 35 | 36 | goChan := make(chan int) 37 | rdyChan := make(chan int) 38 | workers := runtime.GOMAXPROCS(0) 39 | for j := 0; j < workers; j++ { 40 | wg.Add(1) 41 | go func(id int) { 42 | subWorker(*runfor, workers, *tcpAddress, *topic, *channel, rdyChan, goChan, id) 43 | wg.Done() 44 | }(j) 45 | <-rdyChan 46 | } 47 | 48 | if *deadline != "" { 49 | t, err := time.Parse("2006-01-02 15:04:05", *deadline) 50 | if err != nil { 51 | log.Fatal(err) 52 | } 53 | d := time.Until(t) 54 | log.Printf("sleeping until %s (%s)", t, d) 55 | time.Sleep(d) 56 | } 57 | 58 | start := time.Now() 59 | close(goChan) 60 | wg.Wait() 61 | end := time.Now() 62 | duration := end.Sub(start) 63 | tmc := atomic.LoadInt64(&totalMsgCount) 64 | log.Printf("duration: %s - %.03fmb/s - %.03fops/s - %.03fus/op", 65 | duration, 66 | float64(tmc*int64(*size))/duration.Seconds()/1024/1024, 67 | float64(tmc)/duration.Seconds(), 68 | float64(duration/time.Microsecond)/float64(tmc)) 69 | } 70 | 71 | func subWorker(td time.Duration, workers int, tcpAddr string, topic string, channel string, rdyChan chan int, goChan chan int, id int) { 72 | conn, err := net.DialTimeout("tcp", tcpAddr, time.Second) 73 | if err != nil { 74 | panic(err.Error()) 75 | } 76 | conn.Write(nsq.MagicV2) 77 | rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) 78 | ci := make(map[string]interface{}) 79 | ci["client_id"] = "reader" 80 | ci["hostname"] = "reader" 81 | ci["user_agent"] = fmt.Sprintf("bench_reader/%s", nsq.VERSION) 82 | cmd, _ := nsq.Identify(ci) 83 | cmd.WriteTo(rw) 84 | nsq.Subscribe(topic, channel).WriteTo(rw) 85 | rdyChan <- 1 86 | <-goChan 87 | nsq.Ready(*rdy).WriteTo(rw) 88 | rw.Flush() 89 | nsq.ReadResponse(rw) 90 | nsq.ReadResponse(rw) 91 | var msgCount int64 92 | go func() { 93 | time.Sleep(td) 94 | conn.Close() 95 | }() 96 | for { 97 | resp, err := nsq.ReadResponse(rw) 98 | if err != nil { 99 | if errors.Is(err, net.ErrClosed) { 100 | break 101 | } 102 | panic(err.Error()) 103 | } 104 | frameType, data, err := nsq.UnpackResponse(resp) 105 | if err != nil { 106 | panic(err.Error()) 107 | } 108 | if frameType == nsq.FrameTypeError { 109 | panic(string(data)) 110 | } else if frameType == nsq.FrameTypeResponse { 111 | continue 112 | } 113 | msg, err := nsq.DecodeMessage(data) 114 | if err != nil { 115 | panic(err.Error()) 116 | } 117 | nsq.Finish(msg.ID).WriteTo(rw) 118 | msgCount++ 119 | if float64(msgCount%int64(*rdy)) > float64(*rdy)*0.75 { 120 | rw.Flush() 121 | } 122 | } 123 | atomic.AddInt64(&totalMsgCount, msgCount) 124 | } 125 | -------------------------------------------------------------------------------- /bench/bench_writer/bench_writer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "net" 9 | "runtime" 10 | "sync" 11 | "sync/atomic" 12 | "time" 13 | 14 | "github.com/nsqio/go-nsq" 15 | ) 16 | 17 | var ( 18 | runfor = flag.Duration("runfor", 10*time.Second, "duration of time to run") 19 | tcpAddress = flag.String("nsqd-tcp-address", "127.0.0.1:4150", ": to connect to nsqd") 20 | topic = flag.String("topic", "sub_bench", "topic to receive messages on") 21 | size = flag.Int("size", 200, "size of messages") 22 | batchSize = flag.Int("batch-size", 200, "batch size of messages") 23 | deadline = flag.String("deadline", "", "deadline to start the benchmark run") 24 | ) 25 | 26 | var totalMsgCount int64 27 | 28 | func main() { 29 | flag.Parse() 30 | var wg sync.WaitGroup 31 | 32 | log.SetPrefix("[bench_writer] ") 33 | 34 | msg := make([]byte, *size) 35 | batch := make([][]byte, *batchSize) 36 | for i := range batch { 37 | batch[i] = msg 38 | } 39 | 40 | goChan := make(chan int) 41 | rdyChan := make(chan int) 42 | for j := 0; j < runtime.GOMAXPROCS(0); j++ { 43 | wg.Add(1) 44 | go func() { 45 | pubWorker(*runfor, *tcpAddress, *batchSize, batch, *topic, rdyChan, goChan) 46 | wg.Done() 47 | }() 48 | <-rdyChan 49 | } 50 | 51 | if *deadline != "" { 52 | t, err := time.Parse("2006-01-02 15:04:05", *deadline) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | d := time.Until(t) 57 | log.Printf("sleeping until %s (%s)", t, d) 58 | time.Sleep(d) 59 | } 60 | 61 | start := time.Now() 62 | close(goChan) 63 | wg.Wait() 64 | end := time.Now() 65 | duration := end.Sub(start) 66 | tmc := atomic.LoadInt64(&totalMsgCount) 67 | log.Printf("duration: %s - %.03fmb/s - %.03fops/s - %.03fus/op", 68 | duration, 69 | float64(tmc*int64(*size))/duration.Seconds()/1024/1024, 70 | float64(tmc)/duration.Seconds(), 71 | float64(duration/time.Microsecond)/float64(tmc)) 72 | } 73 | 74 | func pubWorker(td time.Duration, tcpAddr string, batchSize int, batch [][]byte, topic string, rdyChan chan int, goChan chan int) { 75 | conn, err := net.DialTimeout("tcp", tcpAddr, time.Second) 76 | if err != nil { 77 | panic(err.Error()) 78 | } 79 | conn.Write(nsq.MagicV2) 80 | rw := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn)) 81 | ci := make(map[string]interface{}) 82 | ci["client_id"] = "writer" 83 | ci["hostname"] = "writer" 84 | ci["user_agent"] = fmt.Sprintf("bench_writer/%s", nsq.VERSION) 85 | cmd, _ := nsq.Identify(ci) 86 | cmd.WriteTo(rw) 87 | rdyChan <- 1 88 | <-goChan 89 | rw.Flush() 90 | nsq.ReadResponse(rw) 91 | var msgCount int64 92 | endTime := time.Now().Add(td) 93 | for { 94 | cmd, _ := nsq.MultiPublish(topic, batch) 95 | _, err := cmd.WriteTo(rw) 96 | if err != nil { 97 | panic(err.Error()) 98 | } 99 | err = rw.Flush() 100 | if err != nil { 101 | panic(err.Error()) 102 | } 103 | resp, err := nsq.ReadResponse(rw) 104 | if err != nil { 105 | panic(err.Error()) 106 | } 107 | frameType, data, err := nsq.UnpackResponse(resp) 108 | if err != nil { 109 | panic(err.Error()) 110 | } 111 | if frameType == nsq.FrameTypeError { 112 | panic(string(data)) 113 | } 114 | msgCount += int64(len(batch)) 115 | if time.Now().After(endTime) { 116 | break 117 | } 118 | } 119 | atomic.AddInt64(&totalMsgCount, msgCount) 120 | } 121 | -------------------------------------------------------------------------------- /bench/requirements.txt: -------------------------------------------------------------------------------- 1 | tornado==4.3 2 | paramiko==1.16.0 3 | boto==2.39.0 4 | -------------------------------------------------------------------------------- /contrib/nsq.spec: -------------------------------------------------------------------------------- 1 | %define name nsq 2 | %define version 1.1.1-alpha 3 | %define release 1 4 | %define path usr/local 5 | %define group Database/Applications 6 | %define __os_install_post %{nil} 7 | 8 | Summary: nsq 9 | Name: %{name} 10 | Version: %{version} 11 | Release: %{release} 12 | Group: %{group} 13 | Packager: Matt Reiferson 14 | License: Apache 15 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} 16 | AutoReqProv: no 17 | # we just assume you have go installed. You may or may not have an RPM to depend on. 18 | # BuildRequires: go 19 | 20 | %description 21 | NSQ - A realtime distributed messaging platform 22 | https://github.com/nsqio/nsq 23 | 24 | %prep 25 | mkdir -p $RPM_BUILD_DIR/%{name}-%{version}-%{release} 26 | cd $RPM_BUILD_DIR/%{name}-%{version}-%{release} 27 | git clone git@github.com:nsqio/nsq.git 28 | 29 | %build 30 | cd $RPM_BUILD_DIR/%{name}-%{version}-%{release}/nsq 31 | make PREFIX=/%{path} 32 | 33 | %install 34 | export DONT_STRIP=1 35 | rm -rf $RPM_BUILD_ROOT 36 | cd $RPM_BUILD_DIR/%{name}-%{version}-%{release}/nsq 37 | make PREFIX=/${path} DESTDIR=$RPM_BUILD_ROOT install 38 | 39 | %files 40 | /%{path}/bin/nsqadmin 41 | /%{path}/bin/nsqd 42 | /%{path}/bin/nsqlookupd 43 | /%{path}/bin/nsq_to_file 44 | /%{path}/bin/nsq_to_http 45 | /%{path}/bin/nsq_to_nsq 46 | /%{path}/bin/nsq_tail 47 | /%{path}/bin/nsq_stat 48 | /%{path}/bin/to_nsq 49 | -------------------------------------------------------------------------------- /contrib/nsqadmin.cfg.example: -------------------------------------------------------------------------------- 1 | ## log verbosity level: debug, info, warn, error, or fatal 2 | log_level = "info" 3 | 4 | ## log message prefix (default "[nsqadmin] ") 5 | # log_prefix = "" 6 | 7 | ## HTTP header to check for authenticated admin users (default "X_Forwarded_User") 8 | # acl_http_header = "" 9 | 10 | ## admin user (may be given multiple times; if specified, only these users will be able to perform privileged actions) 11 | # admin_users = [ 12 | # "admin" 13 | # ] 14 | 15 | ## A CIDR from which to allow HTTP requests to the /config endpoint (default "127.0.0.1/8") 16 | # allow_config_from_cidr = "" 17 | 18 | ## URL base path (default "/") 19 | # base_path = "" 20 | 21 | ## timeout for HTTP connect (default 2s) 22 | # http_client_connect_timeout = "2s" 23 | 24 | ## timeout for HTTP request (default 5s) 25 | # http_client_request_timeout = "5s" 26 | 27 | ## path to certificate file for the HTTP client 28 | # http_client_tls_cert = "" 29 | 30 | ## configure the HTTP client to skip verification of TLS certificates 31 | # http_client_tls_insecure_skip_verify = false 32 | 33 | ## path to key file for the HTTP client 34 | # http_client_tls_key = "" 35 | 36 | ## path to CA file for the HTTP client 37 | # http_client_tls_root_ca_file = "" 38 | 39 | ## : to listen on for HTTP clients 40 | http_address = "0.0.0.0:4171" 41 | 42 | ## graphite HTTP address 43 | graphite_url = "" 44 | 45 | ## proxy HTTP requests to graphite 46 | proxy_graphite = false 47 | 48 | ## prefix used for keys sent to statsd (%s for host replacement, must match nsqd) 49 | statsd_prefix = "nsq.%s" 50 | 51 | ## format of statsd counter stats 52 | statsd_counter_format = "stats.counters.%s.count" 53 | 54 | ## format of statsd gauge stats 55 | statsd_gauge_format = "stats.gauges.%s" 56 | 57 | ## time interval nsqd is configured to push to statsd (must match nsqd) 58 | statsd_interval = "60s" 59 | 60 | ## HTTP endpoint (fully qualified) to which POST notifications of admin actions will be sent 61 | notification_http_endpoint = "" 62 | 63 | 64 | ## nsqlookupd HTTP addresses 65 | nsqlookupd_http_addresses = [ 66 | "127.0.0.1:4161" 67 | ] 68 | 69 | ## nsqd HTTP addresses (optional) 70 | nsqd_http_addresses = [ 71 | "127.0.0.1:4151" 72 | ] 73 | -------------------------------------------------------------------------------- /contrib/nsqd.cfg.example: -------------------------------------------------------------------------------- 1 | ## log verbosity level: debug, info, warn, error, or fatal 2 | log_level = "info" 3 | 4 | ## unique identifier (int) for this worker (will default to a hash of hostname) 5 | # id = 5150 6 | 7 | ## : to listen on for TCP clients 8 | tcp_address = "0.0.0.0:4150" 9 | 10 | ## : to listen on for HTTP clients 11 | http_address = "0.0.0.0:4151" 12 | 13 | ## : to listen on for HTTPS clients 14 | # https_address = "0.0.0.0:4152" 15 | 16 | ## address that will be registered with lookupd (defaults to the OS hostname) 17 | # broadcast_address = "" 18 | 19 | ## cluster of nsqlookupd TCP addresses 20 | nsqlookupd_tcp_addresses = [ 21 | "127.0.0.1:4160" 22 | ] 23 | 24 | ## duration to wait before HTTP client connection timeout 25 | http_client_connect_timeout = "2s" 26 | 27 | ## duration to wait before HTTP client request timeout 28 | http_client_request_timeout = "5s" 29 | 30 | ## path to store disk-backed messages 31 | # data_path = "/var/lib/nsq" 32 | 33 | ## number of messages to keep in memory (per topic/channel) 34 | mem_queue_size = 10000 35 | 36 | ## number of bytes per diskqueue file before rolling 37 | max_bytes_per_file = 104857600 38 | 39 | ## number of messages per diskqueue fsync 40 | sync_every = 2500 41 | 42 | ## duration of time per diskqueue fsync (time.Duration) 43 | sync_timeout = "2s" 44 | 45 | 46 | ## duration to wait before auto-requeing a message 47 | msg_timeout = "60s" 48 | 49 | ## maximum duration before a message will timeout 50 | max_msg_timeout = "15m" 51 | 52 | ## maximum size of a single message in bytes 53 | max_msg_size = 1024768 54 | 55 | ## maximum requeuing timeout for a message 56 | max_req_timeout = "1h" 57 | 58 | ## maximum size of a single command body 59 | max_body_size = 5123840 60 | 61 | 62 | ## maximum client configurable duration of time between client heartbeats 63 | max_heartbeat_interval = "60s" 64 | 65 | ## maximum RDY count for a client 66 | max_rdy_count = 2500 67 | 68 | ## maximum client configurable size (in bytes) for a client output buffer 69 | max_output_buffer_size = 65536 70 | 71 | ## maximum client configurable duration of time between flushing to a client (time.Duration) 72 | max_output_buffer_timeout = "1s" 73 | 74 | 75 | ## UDP : of a statsd daemon for pushing stats 76 | # statsd_address = "127.0.0.1:8125" 77 | 78 | ## prefix used for keys sent to statsd (%s for host replacement) 79 | statsd_prefix = "nsq.%s" 80 | 81 | ## duration between pushing to statsd (time.Duration) 82 | statsd_interval = "60s" 83 | 84 | ## toggle sending memory and GC stats to statsd 85 | statsd_mem_stats = true 86 | 87 | ## the size in bytes of statsd UDP packets 88 | # statsd_udp_packet_size = 508 89 | 90 | 91 | ## message processing time percentiles to keep track of (float) 92 | e2e_processing_latency_percentiles = [ 93 | 1.0, 94 | 0.99, 95 | 0.95 96 | ] 97 | 98 | ## calculate end to end latency quantiles for this duration of time (time.Duration) 99 | e2e_processing_latency_window_time = "10m" 100 | 101 | 102 | ## path to certificate file 103 | tls_cert = "" 104 | 105 | ## path to private key file 106 | tls_key = "" 107 | 108 | ## set policy on client certificate (require - client must provide certificate, 109 | ## require-verify - client must provide verifiable signed certificate) 110 | # tls_client_auth_policy = "require-verify" 111 | 112 | ## set custom root Certificate Authority 113 | # tls_root_ca_file = "" 114 | 115 | ## require client TLS upgrades 116 | tls_required = false 117 | 118 | ## minimum TLS version ("ssl3.0", "tls1.0," "tls1.1", "tls1.2") 119 | tls_min_version = "" 120 | 121 | ## enable deflate feature negotiation (client compression) 122 | deflate = true 123 | 124 | ## max deflate compression level a client can negotiate (> values == > nsqd CPU usage) 125 | max_deflate_level = 6 126 | 127 | ## enable snappy feature negotiation (client compression) 128 | snappy = true 129 | -------------------------------------------------------------------------------- /contrib/nsqlookupd.cfg.example: -------------------------------------------------------------------------------- 1 | ## log verbosity level: debug, info, warn, error, or fatal 2 | log_level = "info" 3 | 4 | ## : to listen on for TCP clients 5 | tcp_address = "0.0.0.0:4160" 6 | 7 | ## : to listen on for HTTP clients 8 | http_address = "0.0.0.0:4161" 9 | 10 | ## address that will be registered with lookupd (defaults to the OS hostname) 11 | # broadcast_address = "" 12 | 13 | 14 | ## duration of time a producer will remain in the active list since its last ping 15 | inactive_producer_timeout = "300s" 16 | 17 | ## duration of time a producer will remain tombstoned if registration remains 18 | tombstone_lifetime = "45s" 19 | -------------------------------------------------------------------------------- /coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Generate test coverage statistics for Go packages. 3 | # 4 | # Works around the fact that `go test -coverprofile` currently does not work 5 | # with multiple packages, see https://code.google.com/p/go/issues/detail?id=6909 6 | # 7 | # Usage: coverage.sh [--html|--coveralls] 8 | # 9 | # --html Additionally create HTML report 10 | # --coveralls Push coverage statistics to coveralls.io 11 | # 12 | 13 | set -e 14 | 15 | workdir=.cover 16 | profile="$workdir/cover.out" 17 | mode=count 18 | 19 | generate_cover_data() { 20 | rm -rf "$workdir" 21 | mkdir "$workdir" 22 | 23 | for pkg in "$@"; do 24 | f="$workdir/$(echo $pkg | tr / -).cover" 25 | go test -covermode="$mode" -coverprofile="$f" "$pkg" 26 | done 27 | 28 | echo "mode: $mode" >"$profile" 29 | grep -h -v "^mode:" "$workdir"/*.cover >>"$profile" 30 | } 31 | 32 | show_html_report() { 33 | go tool cover -html="$profile" -o="$workdir"/coverage.html 34 | } 35 | 36 | show_csv_report() { 37 | go tool cover -func="$profile" -o="$workdir"/coverage.csv 38 | } 39 | 40 | push_to_coveralls() { 41 | echo "Pushing coverage statistics to coveralls.io" 42 | # ignore failure to push - it happens 43 | $GOPATH/bin/goveralls -coverprofile="$profile" \ 44 | -service=github \ 45 | -ignore="nsqadmin/bindata.go" || true 46 | } 47 | 48 | generate_cover_data $(go list ./... | grep -v /vendor/) 49 | show_csv_report 50 | 51 | case "$1" in 52 | "") 53 | ;; 54 | --html) 55 | show_html_report ;; 56 | --coveralls) 57 | push_to_coveralls ;; 58 | *) 59 | echo >&2 "error: invalid option: $1"; exit 1 ;; 60 | esac 61 | -------------------------------------------------------------------------------- /dist.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 1. commit to bump the version and update the changelog/readme 4 | # 2. tag that commit 5 | # 3. use dist.sh to produce tar.gz for all platforms 6 | # 4. aws s3 cp dist s3://bitly-downloads/nsq/ --recursive --exclude "*" --include "nsq-1.3.0*" --profile bitly --acl public-read 7 | # 5. docker manifest push nsqio/nsq:latest 8 | # 6. push to nsqio/master 9 | # 7. update the release metadata on github / upload the binaries 10 | # 8. update nsqio/nsqio.github.io/_posts/2014-03-01-installing.md 11 | # 9. send release announcement emails 12 | # 10. update IRC channel topic 13 | # 11. tweet 14 | 15 | set -e 16 | 17 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 18 | rm -rf $DIR/dist/docker 19 | mkdir -p $DIR/dist/docker 20 | 21 | GOFLAGS='-ldflags="-s -w"' 22 | version=$(awk '/const Binary/ {print $NF}' < $DIR/internal/version/binary.go | sed 's/"//g') 23 | goversion=$(go version | awk '{print $3}') 24 | 25 | echo "... running tests" 26 | ./test.sh 27 | 28 | export GO111MODULE=on 29 | for target in "linux/amd64" "linux/arm64" "darwin/amd64" "darwin/arm64" "freebsd/amd64" "windows/amd64"; do 30 | os=${target%/*} 31 | arch=${target##*/} 32 | echo "... building v$version for $os/$arch" 33 | BUILD=$(mktemp -d ${TMPDIR:-/tmp}/nsq-XXXXX) 34 | TARGET="nsq-$version.$os-$arch.$goversion" 35 | GOOS=$os GOARCH=$arch CGO_ENABLED=0 \ 36 | make DESTDIR=$BUILD PREFIX=/$TARGET BLDFLAGS="$GOFLAGS" install 37 | pushd $BUILD 38 | sudo chown -R 0:0 $TARGET 39 | tar czvf $TARGET.tar.gz $TARGET 40 | mv $TARGET.tar.gz $DIR/dist 41 | popd 42 | make clean 43 | sudo rm -r $BUILD 44 | done 45 | 46 | rnd=$(LC_ALL=C tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c10) 47 | docker buildx create --use --name nsq-$rnd 48 | docker buildx build .\ 49 | --tag nsqio/nsq:v$version \ 50 | --platform linux/amd64,linux/arm64 \ 51 | --output type=image,push=true 52 | if [[ ! $version == *"-"* ]]; then 53 | echo "Tagging nsqio/nsq:v$version as the latest release" 54 | shas=$(docker manifest inspect nsqio/nsq:v$version |\ 55 | grep digest | awk '{print $2}' | sed 's/[",]//g' | sed 's/^/nsqio\/nsq@/') 56 | docker manifest create --amend nsqio/nsq:latest $shas 57 | fi 58 | -------------------------------------------------------------------------------- /fmt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | find . -name "*.go" | xargs goimports -w 3 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/nsqio/nsq 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/BurntSushi/toml v1.3.2 7 | github.com/bitly/go-hostpool v0.1.0 8 | github.com/bitly/timer_metrics v1.0.0 9 | github.com/blang/semver v3.5.1+incompatible 10 | github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b 11 | github.com/golang/snappy v0.0.4 12 | github.com/judwhite/go-svc v1.2.1 13 | github.com/julienschmidt/httprouter v1.3.0 14 | github.com/mreiferson/go-options v1.0.0 15 | github.com/nsqio/go-diskqueue v1.1.0 16 | github.com/nsqio/go-nsq v1.1.0 17 | ) 18 | 19 | require ( 20 | github.com/stretchr/testify v1.9.0 // indirect 21 | golang.org/x/sys v0.10.0 // indirect 22 | ) 23 | 24 | replace github.com/judwhite/go-svc => github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6 25 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= 2 | github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 3 | github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= 4 | github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= 5 | github.com/bitly/timer_metrics v1.0.0 h1:bbszVIl0vT5+/cdZx8L4KOQmM8mC/0y3EBICGSxyhCk= 6 | github.com/bitly/timer_metrics v1.0.0/go.mod h1:87z4/LSg3f++tMqZwZlsLwPuJu6xloyJ7Qm40NyEkLs= 7 | github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= 8 | github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= 9 | github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE= 10 | github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 15 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 16 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 17 | github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= 18 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 19 | github.com/mreiferson/go-options v1.0.0 h1:RMLidydGlDWpL+lQTXo0bVIf/XT2CTq7AEJMoz5/VWs= 20 | github.com/mreiferson/go-options v1.0.0/go.mod h1:zHtCks/HQvOt8ATyfwVe3JJq2PPuImzXINPRTC03+9w= 21 | github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6 h1:NbuBXARvEXrYZ1SzN53ZpObeuwGhl1zvs/C+kzCggrQ= 22 | github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6/go.mod h1:mo/P2JNX8C07ywpP9YtO2gnBgnUiFTHqtsZekJrUuTk= 23 | github.com/nsqio/go-diskqueue v1.1.0 h1:r0dJ0DMXT3+2mOq+79cvCjnhoBxyGC2S9O+OjQrpe4Q= 24 | github.com/nsqio/go-diskqueue v1.1.0/go.mod h1:INuJIxl4ayUsyoNtHL5+9MFPDfSZ0zY93hNY6vhBRsI= 25 | github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= 26 | github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= 27 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 28 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 29 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 30 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 31 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 32 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 33 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 34 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 35 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 36 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 37 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 38 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 39 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 40 | golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= 41 | golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 42 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 43 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 44 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 45 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 46 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 47 | -------------------------------------------------------------------------------- /internal/app/float_array.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "sort" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | type FloatArray []float64 12 | 13 | func (a *FloatArray) Get() interface{} { return []float64(*a) } 14 | 15 | func (a *FloatArray) Set(param string) error { 16 | for _, s := range strings.Split(param, ",") { 17 | v, err := strconv.ParseFloat(s, 64) 18 | if err != nil { 19 | log.Fatalf("Could not parse: %s", s) 20 | return nil 21 | } 22 | *a = append(*a, v) 23 | } 24 | sort.Sort(*a) 25 | return nil 26 | } 27 | 28 | func (a FloatArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 29 | func (a FloatArray) Less(i, j int) bool { return a[i] > a[j] } 30 | func (a FloatArray) Len() int { return len(a) } 31 | 32 | func (a *FloatArray) String() string { 33 | var s []string 34 | for _, v := range *a { 35 | s = append(s, fmt.Sprintf("%f", v)) 36 | } 37 | return strings.Join(s, ",") 38 | } 39 | -------------------------------------------------------------------------------- /internal/app/string_array.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | type StringArray []string 8 | 9 | func (a *StringArray) Get() interface{} { return []string(*a) } 10 | 11 | func (a *StringArray) Set(s string) error { 12 | *a = append(*a, s) 13 | return nil 14 | } 15 | 16 | func (a *StringArray) String() string { 17 | return strings.Join(*a, ",") 18 | } 19 | -------------------------------------------------------------------------------- /internal/clusterinfo/producer_test.go: -------------------------------------------------------------------------------- 1 | package clusterinfo 2 | 3 | import "testing" 4 | 5 | func TestHostNameAddresses(t *testing.T) { 6 | p := &Producer{ 7 | BroadcastAddress: "host.domain.com", 8 | TCPPort: 4150, 9 | HTTPPort: 4151, 10 | } 11 | 12 | if p.HTTPAddress() != "host.domain.com:4151" { 13 | t.Errorf("Incorrect HTTPAddress: %s", p.HTTPAddress()) 14 | } 15 | if p.TCPAddress() != "host.domain.com:4150" { 16 | t.Errorf("Incorrect TCPAddress: %s", p.TCPAddress()) 17 | } 18 | } 19 | 20 | func TestIPv4Addresses(t *testing.T) { 21 | p := &Producer{ 22 | BroadcastAddress: "192.168.1.17", 23 | TCPPort: 4150, 24 | HTTPPort: 4151, 25 | } 26 | 27 | if p.HTTPAddress() != "192.168.1.17:4151" { 28 | t.Errorf("Incorrect IPv4 HTTPAddress: %s", p.HTTPAddress()) 29 | } 30 | if p.TCPAddress() != "192.168.1.17:4150" { 31 | t.Errorf("Incorrect IPv4 TCPAddress: %s", p.TCPAddress()) 32 | } 33 | } 34 | 35 | func TestIPv6Addresses(t *testing.T) { 36 | p := &Producer{ 37 | BroadcastAddress: "fd4a:622f:d2f2::1", 38 | TCPPort: 4150, 39 | HTTPPort: 4151, 40 | } 41 | if p.HTTPAddress() != "[fd4a:622f:d2f2::1]:4151" { 42 | t.Errorf("Incorrect IPv6 HTTPAddress: %s", p.HTTPAddress()) 43 | } 44 | if p.TCPAddress() != "[fd4a:622f:d2f2::1]:4150" { 45 | t.Errorf("Incorrect IPv6 TCPAddress: %s", p.TCPAddress()) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /internal/dirlock/dirlock.go: -------------------------------------------------------------------------------- 1 | //go:build !windows && !illumos 2 | // +build !windows,!illumos 3 | 4 | package dirlock 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | "syscall" 10 | ) 11 | 12 | type DirLock struct { 13 | dir string 14 | f *os.File 15 | } 16 | 17 | func New(dir string) *DirLock { 18 | return &DirLock{ 19 | dir: dir, 20 | } 21 | } 22 | 23 | func (l *DirLock) Lock() error { 24 | f, err := os.Open(l.dir) 25 | if err != nil { 26 | return err 27 | } 28 | l.f = f 29 | err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) 30 | if err != nil { 31 | return fmt.Errorf("cannot flock directory %s - %s (possibly in use by another instance of nsqd)", l.dir, err) 32 | } 33 | return nil 34 | } 35 | 36 | func (l *DirLock) Unlock() error { 37 | defer l.f.Close() 38 | return syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN) 39 | } 40 | -------------------------------------------------------------------------------- /internal/dirlock/dirlock_illumos.go: -------------------------------------------------------------------------------- 1 | //go:build illumos 2 | // +build illumos 3 | 4 | package dirlock 5 | 6 | type DirLock struct { 7 | dir string 8 | } 9 | 10 | func New(dir string) *DirLock { 11 | return &DirLock{ 12 | dir: dir, 13 | } 14 | } 15 | 16 | func (l *DirLock) Lock() error { 17 | return nil 18 | } 19 | 20 | func (l *DirLock) Unlock() error { 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /internal/dirlock/dirlock_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | package dirlock 5 | 6 | type DirLock struct { 7 | dir string 8 | } 9 | 10 | func New(dir string) *DirLock { 11 | return &DirLock{ 12 | dir: dir, 13 | } 14 | } 15 | 16 | func (l *DirLock) Lock() error { 17 | return nil 18 | } 19 | 20 | func (l *DirLock) Unlock() error { 21 | return nil 22 | } 23 | -------------------------------------------------------------------------------- /internal/http_api/compress.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Gorilla Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // copied from https://github.com/gorilla/handlers/blob/master/compress.go 6 | 7 | package http_api 8 | 9 | import ( 10 | "compress/flate" 11 | "compress/gzip" 12 | "io" 13 | "net/http" 14 | "strings" 15 | ) 16 | 17 | type compressResponseWriter struct { 18 | io.Writer 19 | http.ResponseWriter 20 | http.Hijacker 21 | } 22 | 23 | func (w *compressResponseWriter) Header() http.Header { 24 | return w.ResponseWriter.Header() 25 | } 26 | 27 | func (w *compressResponseWriter) WriteHeader(c int) { 28 | w.ResponseWriter.Header().Del("Content-Length") 29 | w.ResponseWriter.WriteHeader(c) 30 | } 31 | 32 | func (w *compressResponseWriter) Write(b []byte) (int, error) { 33 | h := w.ResponseWriter.Header() 34 | if h.Get("Content-Type") == "" { 35 | h.Set("Content-Type", http.DetectContentType(b)) 36 | } 37 | h.Del("Content-Length") 38 | return w.Writer.Write(b) 39 | } 40 | 41 | // CompressHandler gzip compresses HTTP responses for clients that support it 42 | // via the 'Accept-Encoding' header. 43 | func CompressHandler(h http.Handler) http.Handler { 44 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 45 | L: 46 | for _, enc := range strings.Split(r.Header.Get("Accept-Encoding"), ",") { 47 | switch strings.TrimSpace(enc) { 48 | case "gzip": 49 | w.Header().Set("Content-Encoding", "gzip") 50 | w.Header().Add("Vary", "Accept-Encoding") 51 | 52 | gw := gzip.NewWriter(w) 53 | defer gw.Close() 54 | 55 | h, hok := w.(http.Hijacker) 56 | if !hok { /* w is not Hijacker... oh well... */ 57 | h = nil 58 | } 59 | 60 | w = &compressResponseWriter{ 61 | Writer: gw, 62 | ResponseWriter: w, 63 | Hijacker: h, 64 | } 65 | 66 | break L 67 | case "deflate": 68 | w.Header().Set("Content-Encoding", "deflate") 69 | w.Header().Add("Vary", "Accept-Encoding") 70 | 71 | fw, _ := flate.NewWriter(w, flate.DefaultCompression) 72 | defer fw.Close() 73 | 74 | h, hok := w.(http.Hijacker) 75 | if !hok { /* w is not Hijacker... oh well... */ 76 | h = nil 77 | } 78 | 79 | w = &compressResponseWriter{ 80 | Writer: fw, 81 | ResponseWriter: w, 82 | Hijacker: h, 83 | } 84 | 85 | break L 86 | } 87 | } 88 | 89 | h.ServeHTTP(w, r) 90 | }) 91 | } 92 | -------------------------------------------------------------------------------- /internal/http_api/http_server.go: -------------------------------------------------------------------------------- 1 | package http_api 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "net" 8 | "net/http" 9 | 10 | "github.com/nsqio/nsq/internal/lg" 11 | ) 12 | 13 | type logWriter struct { 14 | logf lg.AppLogFunc 15 | } 16 | 17 | func (l logWriter) Write(p []byte) (int, error) { 18 | l.logf(lg.WARN, "%s", string(p)) 19 | return len(p), nil 20 | } 21 | 22 | func Serve(listener net.Listener, handler http.Handler, proto string, logf lg.AppLogFunc) error { 23 | logf(lg.INFO, "%s: listening on %s", proto, listener.Addr()) 24 | 25 | server := &http.Server{ 26 | Handler: handler, 27 | ErrorLog: log.New(logWriter{logf}, "", 0), 28 | } 29 | err := server.Serve(listener) 30 | // theres no direct way to detect this error because it is not exposed 31 | if err != nil && !errors.Is(err, net.ErrClosed) { 32 | return fmt.Errorf("http.Serve() error - %s", err) 33 | } 34 | 35 | logf(lg.INFO, "%s: closing %s", proto, listener.Addr()) 36 | 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /internal/http_api/req_params.go: -------------------------------------------------------------------------------- 1 | package http_api 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "net/http" 7 | "net/url" 8 | ) 9 | 10 | type ReqParams struct { 11 | url.Values 12 | Body []byte 13 | } 14 | 15 | func NewReqParams(req *http.Request) (*ReqParams, error) { 16 | reqParams, err := url.ParseQuery(req.URL.RawQuery) 17 | if err != nil { 18 | return nil, err 19 | } 20 | 21 | data, err := io.ReadAll(req.Body) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | return &ReqParams{reqParams, data}, nil 27 | } 28 | 29 | func (r *ReqParams) Get(key string) (string, error) { 30 | v, ok := r.Values[key] 31 | if !ok { 32 | return "", errors.New("key not in query params") 33 | } 34 | return v[0], nil 35 | } 36 | 37 | func (r *ReqParams) GetAll(key string) ([]string, error) { 38 | v, ok := r.Values[key] 39 | if !ok { 40 | return nil, errors.New("key not in query params") 41 | } 42 | return v, nil 43 | } 44 | -------------------------------------------------------------------------------- /internal/http_api/topic_channel_args.go: -------------------------------------------------------------------------------- 1 | package http_api 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/nsqio/nsq/internal/protocol" 7 | ) 8 | 9 | type getter interface { 10 | Get(key string) (string, error) 11 | } 12 | 13 | func GetTopicChannelArgs(rp getter) (string, string, error) { 14 | topicName, err := rp.Get("topic") 15 | if err != nil { 16 | return "", "", errors.New("MISSING_ARG_TOPIC") 17 | } 18 | 19 | if !protocol.IsValidTopicName(topicName) { 20 | return "", "", errors.New("INVALID_ARG_TOPIC") 21 | } 22 | 23 | channelName, err := rp.Get("channel") 24 | if err != nil { 25 | return "", "", errors.New("MISSING_ARG_CHANNEL") 26 | } 27 | 28 | if !protocol.IsValidChannelName(channelName) { 29 | return "", "", errors.New("INVALID_ARG_CHANNEL") 30 | } 31 | 32 | return topicName, channelName, nil 33 | } 34 | -------------------------------------------------------------------------------- /internal/lg/lg.go: -------------------------------------------------------------------------------- 1 | // Package lg provides leveled logging 2 | package lg 3 | 4 | import ( 5 | "fmt" 6 | "log" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | const ( 12 | DEBUG = LogLevel(1) 13 | INFO = LogLevel(2) 14 | WARN = LogLevel(3) 15 | ERROR = LogLevel(4) 16 | FATAL = LogLevel(5) 17 | ) 18 | 19 | type AppLogFunc func(lvl LogLevel, f string, args ...interface{}) 20 | 21 | type Logger interface { 22 | Output(maxdepth int, s string) error 23 | } 24 | 25 | type NilLogger struct{} 26 | 27 | func (l NilLogger) Output(maxdepth int, s string) error { 28 | return nil 29 | } 30 | 31 | type LogLevel int 32 | 33 | func (l *LogLevel) Get() interface{} { return *l } 34 | 35 | func (l *LogLevel) Set(s string) error { 36 | lvl, err := ParseLogLevel(s) 37 | if err != nil { 38 | return err 39 | } 40 | *l = lvl 41 | return nil 42 | } 43 | 44 | func (l *LogLevel) String() string { 45 | switch *l { 46 | case DEBUG: 47 | return "DEBUG" 48 | case INFO: 49 | return "INFO" 50 | case WARN: 51 | return "WARNING" 52 | case ERROR: 53 | return "ERROR" 54 | case FATAL: 55 | return "FATAL" 56 | } 57 | return "invalid" 58 | } 59 | 60 | func ParseLogLevel(levelstr string) (LogLevel, error) { 61 | switch strings.ToLower(levelstr) { 62 | case "debug": 63 | return DEBUG, nil 64 | case "info": 65 | return INFO, nil 66 | case "warn": 67 | return WARN, nil 68 | case "error": 69 | return ERROR, nil 70 | case "fatal": 71 | return FATAL, nil 72 | } 73 | return 0, fmt.Errorf("invalid log level '%s' (debug, info, warn, error, fatal)", levelstr) 74 | } 75 | 76 | func Logf(logger Logger, cfgLevel LogLevel, msgLevel LogLevel, f string, args ...interface{}) { 77 | if cfgLevel > msgLevel { 78 | return 79 | } 80 | logger.Output(3, fmt.Sprintf(msgLevel.String()+": "+f, args...)) 81 | } 82 | 83 | func LogFatal(prefix string, f string, args ...interface{}) { 84 | logger := log.New(os.Stderr, prefix, log.Ldate|log.Ltime|log.Lmicroseconds) 85 | Logf(logger, FATAL, FATAL, f, args...) 86 | os.Exit(1) 87 | } 88 | -------------------------------------------------------------------------------- /internal/lg/lg_test.go: -------------------------------------------------------------------------------- 1 | package lg 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/nsqio/nsq/internal/test" 7 | ) 8 | 9 | type mockLogger struct { 10 | Count int 11 | } 12 | 13 | func (l *mockLogger) Output(maxdepth int, s string) error { 14 | l.Count++ 15 | return nil 16 | } 17 | 18 | func TestLogging(t *testing.T) { 19 | logger := &mockLogger{} 20 | 21 | // Test only fatal get through 22 | logger.Count = 0 23 | for i := 1; i <= 5; i++ { 24 | Logf(logger, FATAL, LogLevel(i), "Test") 25 | } 26 | test.Equal(t, 1, logger.Count) 27 | 28 | // Test only warnings or higher get through 29 | logger.Count = 0 30 | for i := 1; i <= 5; i++ { 31 | Logf(logger, WARN, LogLevel(i), "Test") 32 | } 33 | test.Equal(t, 3, logger.Count) 34 | 35 | // Test everything gets through 36 | logger.Count = 0 37 | for i := 1; i <= 5; i++ { 38 | Logf(logger, DEBUG, LogLevel(i), "Test") 39 | } 40 | test.Equal(t, 5, logger.Count) 41 | } 42 | -------------------------------------------------------------------------------- /internal/pqueue/pqueue.go: -------------------------------------------------------------------------------- 1 | package pqueue 2 | 3 | import ( 4 | "container/heap" 5 | ) 6 | 7 | type Item struct { 8 | Value interface{} 9 | Priority int64 10 | Index int 11 | } 12 | 13 | // this is a priority queue as implemented by a min heap 14 | // ie. the 0th element is the *lowest* value 15 | type PriorityQueue []*Item 16 | 17 | func New(capacity int) PriorityQueue { 18 | return make(PriorityQueue, 0, capacity) 19 | } 20 | 21 | func (pq PriorityQueue) Len() int { 22 | return len(pq) 23 | } 24 | 25 | func (pq PriorityQueue) Less(i, j int) bool { 26 | return pq[i].Priority < pq[j].Priority 27 | } 28 | 29 | func (pq PriorityQueue) Swap(i, j int) { 30 | pq[i], pq[j] = pq[j], pq[i] 31 | pq[i].Index = i 32 | pq[j].Index = j 33 | } 34 | 35 | func (pq *PriorityQueue) Push(x interface{}) { 36 | n := len(*pq) 37 | c := cap(*pq) 38 | if n+1 > c { 39 | npq := make(PriorityQueue, n, c*2) 40 | copy(npq, *pq) 41 | *pq = npq 42 | } 43 | *pq = (*pq)[0 : n+1] 44 | item := x.(*Item) 45 | item.Index = n 46 | (*pq)[n] = item 47 | } 48 | 49 | func (pq *PriorityQueue) Pop() interface{} { 50 | n := len(*pq) 51 | c := cap(*pq) 52 | if n < (c/2) && c > 25 { 53 | npq := make(PriorityQueue, n, c/2) 54 | copy(npq, *pq) 55 | *pq = npq 56 | } 57 | item := (*pq)[n-1] 58 | item.Index = -1 59 | *pq = (*pq)[0 : n-1] 60 | return item 61 | } 62 | 63 | func (pq *PriorityQueue) PeekAndShift(max int64) (*Item, int64) { 64 | if pq.Len() == 0 { 65 | return nil, 0 66 | } 67 | 68 | item := (*pq)[0] 69 | if item.Priority > max { 70 | return nil, item.Priority - max 71 | } 72 | heap.Remove(pq, 0) 73 | 74 | return item, 0 75 | } 76 | -------------------------------------------------------------------------------- /internal/pqueue/pqueue_test.go: -------------------------------------------------------------------------------- 1 | package pqueue 2 | 3 | import ( 4 | "container/heap" 5 | "math/rand" 6 | "path/filepath" 7 | "reflect" 8 | "runtime" 9 | "sort" 10 | "testing" 11 | ) 12 | 13 | func equal(t *testing.T, act, exp interface{}) { 14 | if !reflect.DeepEqual(exp, act) { 15 | _, file, line, _ := runtime.Caller(1) 16 | t.Logf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", 17 | filepath.Base(file), line, exp, act) 18 | t.FailNow() 19 | } 20 | } 21 | 22 | func TestPriorityQueue(t *testing.T) { 23 | c := 100 24 | pq := New(c) 25 | 26 | for i := 0; i < c+1; i++ { 27 | heap.Push(&pq, &Item{Value: i, Priority: int64(i)}) 28 | } 29 | equal(t, pq.Len(), c+1) 30 | equal(t, cap(pq), c*2) 31 | 32 | for i := 0; i < c+1; i++ { 33 | item := heap.Pop(&pq) 34 | equal(t, item.(*Item).Value.(int), i) 35 | } 36 | equal(t, cap(pq), c/4) 37 | } 38 | 39 | func TestUnsortedInsert(t *testing.T) { 40 | c := 100 41 | pq := New(c) 42 | ints := make([]int, 0, c) 43 | 44 | for i := 0; i < c; i++ { 45 | v := rand.Int() 46 | ints = append(ints, v) 47 | heap.Push(&pq, &Item{Value: i, Priority: int64(v)}) 48 | } 49 | equal(t, pq.Len(), c) 50 | equal(t, cap(pq), c) 51 | 52 | sort.Ints(ints) 53 | 54 | for i := 0; i < c; i++ { 55 | item, _ := pq.PeekAndShift(int64(ints[len(ints)-1])) 56 | equal(t, item.Priority, int64(ints[i])) 57 | } 58 | } 59 | 60 | func TestRemove(t *testing.T) { 61 | c := 100 62 | pq := New(c) 63 | 64 | for i := 0; i < c; i++ { 65 | v := rand.Int() 66 | heap.Push(&pq, &Item{Value: "test", Priority: int64(v)}) 67 | } 68 | 69 | for i := 0; i < 10; i++ { 70 | heap.Remove(&pq, rand.Intn((c-1)-i)) 71 | } 72 | 73 | lastPriority := heap.Pop(&pq).(*Item).Priority 74 | for i := 0; i < (c - 10 - 1); i++ { 75 | item := heap.Pop(&pq) 76 | equal(t, lastPriority < item.(*Item).Priority, true) 77 | lastPriority = item.(*Item).Priority 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /internal/protocol/byte_base10.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var errBase10 = errors.New("failed to convert to Base10") 8 | 9 | func ByteToBase10(b []byte) (n uint64, err error) { 10 | base := uint64(10) 11 | 12 | n = 0 13 | for i := 0; i < len(b); i++ { 14 | var v byte 15 | d := b[i] 16 | switch { 17 | case '0' <= d && d <= '9': 18 | v = d - '0' 19 | default: 20 | n = 0 21 | err = errBase10 22 | return 23 | } 24 | n *= base 25 | n += uint64(v) 26 | } 27 | 28 | return n, err 29 | } 30 | -------------------------------------------------------------------------------- /internal/protocol/byte_base10_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var result uint64 8 | 9 | func BenchmarkByteToBase10Valid(b *testing.B) { 10 | bt := []byte{'3', '1', '4', '1', '5', '9', '2', '5'} 11 | var n uint64 12 | for i := 0; i < b.N; i++ { 13 | n, _ = ByteToBase10(bt) 14 | } 15 | result = n 16 | } 17 | 18 | func BenchmarkByteToBase10Invalid(b *testing.B) { 19 | bt := []byte{'?', '1', '4', '1', '5', '9', '2', '5'} 20 | var n uint64 21 | for i := 0; i < b.N; i++ { 22 | n, _ = ByteToBase10(bt) 23 | } 24 | result = n 25 | } 26 | -------------------------------------------------------------------------------- /internal/protocol/errors.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | type ChildErr interface { 4 | Parent() error 5 | } 6 | 7 | // ClientErr provides a way for NSQ daemons to log a human reabable 8 | // error string and return a machine readable string to the client. 9 | // 10 | // see docs/protocol.md for error codes by command 11 | type ClientErr struct { 12 | ParentErr error 13 | Code string 14 | Desc string 15 | } 16 | 17 | // Error returns the machine readable form 18 | func (e *ClientErr) Error() string { 19 | return e.Code + " " + e.Desc 20 | } 21 | 22 | // Parent returns the parent error 23 | func (e *ClientErr) Parent() error { 24 | return e.ParentErr 25 | } 26 | 27 | // NewClientErr creates a ClientErr with the supplied human and machine readable strings 28 | func NewClientErr(parent error, code string, description string) *ClientErr { 29 | return &ClientErr{parent, code, description} 30 | } 31 | 32 | type FatalClientErr struct { 33 | ParentErr error 34 | Code string 35 | Desc string 36 | } 37 | 38 | // Error returns the machine readable form 39 | func (e *FatalClientErr) Error() string { 40 | return e.Code + " " + e.Desc 41 | } 42 | 43 | // Parent returns the parent error 44 | func (e *FatalClientErr) Parent() error { 45 | return e.ParentErr 46 | } 47 | 48 | // NewFatalClientErr creates a ClientErr with the supplied human and machine readable strings 49 | func NewFatalClientErr(parent error, code string, description string) *FatalClientErr { 50 | return &FatalClientErr{parent, code, description} 51 | } 52 | -------------------------------------------------------------------------------- /internal/protocol/names.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | var validTopicChannelNameRegex = regexp.MustCompile(`^[.a-zA-Z0-9_-]+(#ephemeral)?$`) 8 | 9 | // IsValidTopicName checks a topic name for correctness 10 | func IsValidTopicName(name string) bool { 11 | return isValidName(name) 12 | } 13 | 14 | // IsValidChannelName checks a channel name for correctness 15 | func IsValidChannelName(name string) bool { 16 | return isValidName(name) 17 | } 18 | 19 | func isValidName(name string) bool { 20 | if len(name) > 64 || len(name) < 1 { 21 | return false 22 | } 23 | return validTopicChannelNameRegex.MatchString(name) 24 | } 25 | -------------------------------------------------------------------------------- /internal/protocol/protocol.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "encoding/binary" 5 | "io" 6 | "net" 7 | ) 8 | 9 | type Client interface { 10 | Close() error 11 | } 12 | 13 | // Protocol describes the basic behavior of any protocol in the system 14 | type Protocol interface { 15 | NewClient(net.Conn) Client 16 | IOLoop(Client) error 17 | } 18 | 19 | // SendResponse is a server side utility function to prefix data with a length header 20 | // and write to the supplied Writer 21 | func SendResponse(w io.Writer, data []byte) (int, error) { 22 | err := binary.Write(w, binary.BigEndian, int32(len(data))) 23 | if err != nil { 24 | return 0, err 25 | } 26 | 27 | n, err := w.Write(data) 28 | if err != nil { 29 | return 0, err 30 | } 31 | 32 | return (n + 4), nil 33 | } 34 | 35 | // SendFramedResponse is a server side utility function to prefix data with a length header 36 | // and frame header and write to the supplied Writer 37 | func SendFramedResponse(w io.Writer, frameType int32, data []byte) (int, error) { 38 | beBuf := make([]byte, 4) 39 | size := uint32(len(data)) + 4 40 | 41 | binary.BigEndian.PutUint32(beBuf, size) 42 | n, err := w.Write(beBuf) 43 | if err != nil { 44 | return n, err 45 | } 46 | 47 | binary.BigEndian.PutUint32(beBuf, uint32(frameType)) 48 | n, err = w.Write(beBuf) 49 | if err != nil { 50 | return n + 4, err 51 | } 52 | 53 | n, err = w.Write(data) 54 | return n + 8, err 55 | } 56 | -------------------------------------------------------------------------------- /internal/protocol/tcp_server.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | "runtime" 8 | "sync" 9 | 10 | "github.com/nsqio/nsq/internal/lg" 11 | ) 12 | 13 | type TCPHandler interface { 14 | Handle(net.Conn) 15 | } 16 | 17 | func TCPServer(listener net.Listener, handler TCPHandler, logf lg.AppLogFunc) error { 18 | logf(lg.INFO, "TCP: listening on %s", listener.Addr()) 19 | 20 | var wg sync.WaitGroup 21 | 22 | for { 23 | clientConn, err := listener.Accept() 24 | if err != nil { 25 | // net.Error.Temporary() is deprecated, but is valid for accept 26 | // this is a hack to avoid a staticcheck error 27 | if te, ok := err.(interface{ Temporary() bool }); ok && te.Temporary() { 28 | logf(lg.WARN, "temporary Accept() failure - %s", err) 29 | runtime.Gosched() 30 | continue 31 | } 32 | // theres no direct way to detect this error because it is not exposed 33 | if !errors.Is(err, net.ErrClosed) { 34 | return fmt.Errorf("listener.Accept() error - %s", err) 35 | } 36 | break 37 | } 38 | 39 | wg.Add(1) 40 | go func() { 41 | handler.Handle(clientConn) 42 | wg.Done() 43 | }() 44 | } 45 | 46 | // wait to return until all handler goroutines complete 47 | wg.Wait() 48 | 49 | logf(lg.INFO, "TCP: closing %s", listener.Addr()) 50 | 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /internal/quantile/aggregate.go: -------------------------------------------------------------------------------- 1 | package quantile 2 | 3 | import ( 4 | "encoding/json" 5 | "math" 6 | "sort" 7 | ) 8 | 9 | type E2eProcessingLatencyAggregate struct { 10 | Count int `json:"count"` 11 | Percentiles []map[string]float64 `json:"percentiles"` 12 | Topic string `json:"topic"` 13 | Channel string `json:"channel"` 14 | Addr string `json:"host"` 15 | } 16 | 17 | func (e *E2eProcessingLatencyAggregate) UnmarshalJSON(b []byte) error { 18 | var resp struct { 19 | Count int `json:"count"` 20 | Percentiles []map[string]float64 `json:"percentiles"` 21 | Topic string `json:"topic"` 22 | Channel string `json:"channel"` 23 | Addr string `json:"host"` 24 | } 25 | err := json.Unmarshal(b, &resp) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | for _, p := range resp.Percentiles { 31 | p["min"] = p["value"] 32 | p["max"] = p["value"] 33 | p["average"] = p["value"] 34 | p["count"] = float64(resp.Count) 35 | } 36 | 37 | e.Count = resp.Count 38 | e.Percentiles = resp.Percentiles 39 | e.Topic = resp.Topic 40 | e.Channel = resp.Channel 41 | e.Addr = resp.Addr 42 | 43 | return nil 44 | } 45 | 46 | func (e *E2eProcessingLatencyAggregate) Len() int { return len(e.Percentiles) } 47 | func (e *E2eProcessingLatencyAggregate) Swap(i, j int) { 48 | e.Percentiles[i], e.Percentiles[j] = e.Percentiles[j], e.Percentiles[i] 49 | } 50 | func (e *E2eProcessingLatencyAggregate) Less(i, j int) bool { 51 | return e.Percentiles[i]["percentile"] > e.Percentiles[j]["percentile"] 52 | } 53 | 54 | // Add merges e2 into e by averaging the percentiles 55 | func (e *E2eProcessingLatencyAggregate) Add(e2 *E2eProcessingLatencyAggregate) { 56 | e.Addr = "*" 57 | p := e.Percentiles 58 | e.Count += e2.Count 59 | for _, value := range e2.Percentiles { 60 | i := -1 61 | for j, v := range p { 62 | if value["quantile"] == v["quantile"] { 63 | i = j 64 | break 65 | } 66 | } 67 | if i == -1 { 68 | i = len(p) 69 | e.Percentiles = append(p, make(map[string]float64)) 70 | p = e.Percentiles 71 | p[i]["quantile"] = value["quantile"] 72 | } 73 | p[i]["max"] = math.Max(value["max"], p[i]["max"]) 74 | p[i]["min"] = math.Min(value["max"], p[i]["max"]) 75 | p[i]["count"] += value["count"] 76 | if p[i]["count"] == 0 { 77 | p[i]["average"] = 0 78 | continue 79 | } 80 | delta := value["average"] - p[i]["average"] 81 | R := delta * value["count"] / p[i]["count"] 82 | p[i]["average"] = p[i]["average"] + R 83 | } 84 | sort.Sort(e) 85 | } 86 | -------------------------------------------------------------------------------- /internal/quantile/quantile.go: -------------------------------------------------------------------------------- 1 | package quantile 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | "time" 7 | 8 | "github.com/bmizerany/perks/quantile" 9 | "github.com/nsqio/nsq/internal/stringy" 10 | ) 11 | 12 | type Result struct { 13 | Count int `json:"count"` 14 | Percentiles []map[string]float64 `json:"percentiles"` 15 | } 16 | 17 | func (r *Result) String() string { 18 | var s []string 19 | for _, item := range r.Percentiles { 20 | s = append(s, stringy.NanoSecondToHuman(item["value"])) 21 | } 22 | return strings.Join(s, ", ") 23 | } 24 | 25 | type Quantile struct { 26 | sync.Mutex 27 | streams [2]quantile.Stream 28 | currentIndex uint8 29 | lastMoveWindow time.Time 30 | currentStream *quantile.Stream 31 | 32 | Percentiles []float64 33 | MoveWindowTime time.Duration 34 | } 35 | 36 | func New(WindowTime time.Duration, Percentiles []float64) *Quantile { 37 | q := Quantile{ 38 | currentIndex: 0, 39 | lastMoveWindow: time.Now(), 40 | MoveWindowTime: WindowTime / 2, 41 | Percentiles: Percentiles, 42 | } 43 | for i := 0; i < 2; i++ { 44 | q.streams[i] = *quantile.NewTargeted(Percentiles...) 45 | } 46 | q.currentStream = &q.streams[0] 47 | return &q 48 | } 49 | 50 | func (q *Quantile) Result() *Result { 51 | if q == nil { 52 | return &Result{} 53 | } 54 | queryHandler := q.QueryHandler() 55 | result := Result{ 56 | Count: queryHandler.Count(), 57 | Percentiles: make([]map[string]float64, len(q.Percentiles)), 58 | } 59 | for i, p := range q.Percentiles { 60 | value := queryHandler.Query(p) 61 | result.Percentiles[i] = map[string]float64{"quantile": p, "value": value} 62 | } 63 | return &result 64 | } 65 | 66 | func (q *Quantile) Insert(msgStartTime int64) { 67 | q.Lock() 68 | 69 | now := time.Now() 70 | for q.IsDataStale(now) { 71 | q.moveWindow() 72 | } 73 | 74 | q.currentStream.Insert(float64(now.UnixNano() - msgStartTime)) 75 | q.Unlock() 76 | } 77 | 78 | func (q *Quantile) QueryHandler() *quantile.Stream { 79 | q.Lock() 80 | now := time.Now() 81 | for q.IsDataStale(now) { 82 | q.moveWindow() 83 | } 84 | 85 | merged := quantile.NewTargeted(q.Percentiles...) 86 | merged.Merge(q.streams[0].Samples()) 87 | merged.Merge(q.streams[1].Samples()) 88 | q.Unlock() 89 | return merged 90 | } 91 | 92 | func (q *Quantile) IsDataStale(now time.Time) bool { 93 | return now.After(q.lastMoveWindow.Add(q.MoveWindowTime)) 94 | } 95 | 96 | func (q *Quantile) Merge(them *Quantile) { 97 | q.Lock() 98 | them.Lock() 99 | iUs := q.currentIndex 100 | iThem := them.currentIndex 101 | 102 | q.streams[iUs].Merge(them.streams[iThem].Samples()) 103 | 104 | iUs ^= 0x1 105 | iThem ^= 0x1 106 | q.streams[iUs].Merge(them.streams[iThem].Samples()) 107 | 108 | if q.lastMoveWindow.Before(them.lastMoveWindow) { 109 | q.lastMoveWindow = them.lastMoveWindow 110 | } 111 | q.Unlock() 112 | them.Unlock() 113 | } 114 | 115 | func (q *Quantile) moveWindow() { 116 | q.currentIndex ^= 0x1 117 | q.currentStream = &q.streams[q.currentIndex] 118 | q.lastMoveWindow = q.lastMoveWindow.Add(q.MoveWindowTime) 119 | q.currentStream.Reset() 120 | } 121 | -------------------------------------------------------------------------------- /internal/statsd/client.go: -------------------------------------------------------------------------------- 1 | package statsd 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | ) 7 | 8 | type Client struct { 9 | w io.Writer 10 | prefix string 11 | } 12 | 13 | func NewClient(w io.Writer, prefix string) *Client { 14 | return &Client{ 15 | w: w, 16 | prefix: prefix, 17 | } 18 | } 19 | 20 | func (c *Client) Incr(stat string, count int64) error { 21 | return c.send(stat, "%d|c", count) 22 | } 23 | 24 | func (c *Client) Decr(stat string, count int64) error { 25 | return c.send(stat, "%d|c", -count) 26 | } 27 | 28 | func (c *Client) Timing(stat string, delta int64) error { 29 | return c.send(stat, "%d|ms", delta) 30 | } 31 | 32 | func (c *Client) Gauge(stat string, value int64) error { 33 | return c.send(stat, "%d|g", value) 34 | } 35 | 36 | func (c *Client) send(stat string, format string, value int64) error { 37 | format = fmt.Sprintf("%s%s:%s\n", c.prefix, stat, format) 38 | _, err := fmt.Fprintf(c.w, format, value) 39 | return err 40 | } 41 | -------------------------------------------------------------------------------- /internal/statsd/host.go: -------------------------------------------------------------------------------- 1 | package statsd 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func HostKey(h string) string { 8 | return strings.Replace(strings.Replace(h, ".", "_", -1), ":", "_", -1) 9 | } 10 | -------------------------------------------------------------------------------- /internal/stringy/slice.go: -------------------------------------------------------------------------------- 1 | package stringy 2 | 3 | func Add(s []string, a string) []string { 4 | for _, existing := range s { 5 | if a == existing { 6 | return s 7 | } 8 | } 9 | return append(s, a) 10 | 11 | } 12 | 13 | func Union(s []string, a []string) []string { 14 | for _, entry := range a { 15 | found := false 16 | for _, existing := range s { 17 | if entry == existing { 18 | found = true 19 | break 20 | } 21 | } 22 | if !found { 23 | s = append(s, entry) 24 | } 25 | } 26 | return s 27 | } 28 | 29 | func Uniq(s []string) (r []string) { 30 | outerLoop: 31 | for _, entry := range s { 32 | for _, existing := range r { 33 | if existing == entry { 34 | continue outerLoop 35 | } 36 | } 37 | r = append(r, entry) 38 | } 39 | return 40 | } 41 | -------------------------------------------------------------------------------- /internal/stringy/slice_test.go: -------------------------------------------------------------------------------- 1 | package stringy_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/nsqio/nsq/internal/stringy" 7 | ) 8 | 9 | func BenchmarkUniq(b *testing.B) { 10 | values := []string{"a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "b"} 11 | for i := 0; i < b.N; i++ { 12 | values = stringy.Uniq(values) 13 | if len(values) != 2 { 14 | b.Fatal("values len is incorrect") 15 | } 16 | } 17 | } 18 | 19 | func TestUniq(t *testing.T) { 20 | values := []string{"a", "a", "a", "b", "b", "b", "c", "c", "c"} 21 | values = stringy.Uniq(values) 22 | if len(values) != 3 { 23 | t.Fatal("values len is incorrect") 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /internal/stringy/template.go: -------------------------------------------------------------------------------- 1 | package stringy 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | func NanoSecondToHuman(v float64) string { 8 | var suffix string 9 | switch { 10 | case v > 1000000000: 11 | v /= 1000000000 12 | suffix = "s" 13 | case v > 1000000: 14 | v /= 1000000 15 | suffix = "ms" 16 | case v > 1000: 17 | v /= 1000 18 | suffix = "us" 19 | default: 20 | suffix = "ns" 21 | } 22 | return fmt.Sprintf("%0.1f%s", v, suffix) 23 | } 24 | -------------------------------------------------------------------------------- /internal/test/assertions.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "path/filepath" 5 | "reflect" 6 | "runtime" 7 | "testing" 8 | ) 9 | 10 | func Equal(t *testing.T, expected, actual interface{}) { 11 | if !reflect.DeepEqual(expected, actual) { 12 | _, file, line, _ := runtime.Caller(1) 13 | t.Logf("\033[31m%s:%d:\n\n\t %#v (expected)\n\n\t!= %#v (actual)\033[39m\n\n", 14 | filepath.Base(file), line, expected, actual) 15 | t.FailNow() 16 | } 17 | } 18 | 19 | func NotEqual(t *testing.T, expected, actual interface{}) { 20 | if reflect.DeepEqual(expected, actual) { 21 | _, file, line, _ := runtime.Caller(1) 22 | t.Logf("\033[31m%s:%d:\n\n\tnexp: %#v\n\n\tgot: %#v\033[39m\n\n", 23 | filepath.Base(file), line, expected, actual) 24 | t.FailNow() 25 | } 26 | } 27 | 28 | func Nil(t *testing.T, object interface{}) { 29 | if !isNil(object) { 30 | _, file, line, _ := runtime.Caller(1) 31 | t.Logf("\033[31m%s:%d:\n\n\t (expected)\n\n\t!= %#v (actual)\033[39m\n\n", 32 | filepath.Base(file), line, object) 33 | t.FailNow() 34 | } 35 | } 36 | 37 | func NotNil(t *testing.T, object interface{}) { 38 | if isNil(object) { 39 | _, file, line, _ := runtime.Caller(1) 40 | t.Logf("\033[31m%s:%d:\n\n\tExpected value not to be \033[39m\n\n", 41 | filepath.Base(file), line) 42 | t.FailNow() 43 | } 44 | } 45 | 46 | func isNil(object interface{}) bool { 47 | if object == nil { 48 | return true 49 | } 50 | 51 | value := reflect.ValueOf(object) 52 | kind := value.Kind() 53 | if kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() { 54 | return true 55 | } 56 | 57 | return false 58 | } 59 | -------------------------------------------------------------------------------- /internal/test/fakes.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "net" 5 | "time" 6 | ) 7 | 8 | type FakeNetConn struct { 9 | ReadFunc func([]byte) (int, error) 10 | WriteFunc func([]byte) (int, error) 11 | CloseFunc func() error 12 | LocalAddrFunc func() net.Addr 13 | RemoteAddrFunc func() net.Addr 14 | SetDeadlineFunc func(time.Time) error 15 | SetReadDeadlineFunc func(time.Time) error 16 | SetWriteDeadlineFunc func(time.Time) error 17 | } 18 | 19 | func (f FakeNetConn) Read(b []byte) (int, error) { return f.ReadFunc(b) } 20 | func (f FakeNetConn) Write(b []byte) (int, error) { return f.WriteFunc(b) } 21 | func (f FakeNetConn) Close() error { return f.CloseFunc() } 22 | func (f FakeNetConn) LocalAddr() net.Addr { return f.LocalAddrFunc() } 23 | func (f FakeNetConn) RemoteAddr() net.Addr { return f.RemoteAddrFunc() } 24 | func (f FakeNetConn) SetDeadline(t time.Time) error { return f.SetDeadlineFunc(t) } 25 | func (f FakeNetConn) SetReadDeadline(t time.Time) error { return f.SetReadDeadlineFunc(t) } 26 | func (f FakeNetConn) SetWriteDeadline(t time.Time) error { return f.SetWriteDeadlineFunc(t) } 27 | 28 | type fakeNetAddr struct{} 29 | 30 | func (fakeNetAddr) Network() string { return "" } 31 | func (fakeNetAddr) String() string { return "" } 32 | 33 | func NewFakeNetConn() FakeNetConn { 34 | netAddr := fakeNetAddr{} 35 | return FakeNetConn{ 36 | ReadFunc: func(b []byte) (int, error) { return 0, nil }, 37 | WriteFunc: func(b []byte) (int, error) { return len(b), nil }, 38 | CloseFunc: func() error { return nil }, 39 | LocalAddrFunc: func() net.Addr { return netAddr }, 40 | RemoteAddrFunc: func() net.Addr { return netAddr }, 41 | SetDeadlineFunc: func(time.Time) error { return nil }, 42 | SetWriteDeadlineFunc: func(time.Time) error { return nil }, 43 | SetReadDeadlineFunc: func(time.Time) error { return nil }, 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /internal/test/logger.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | type Logger interface { 4 | Output(maxdepth int, s string) error 5 | } 6 | 7 | type tbLog interface { 8 | Log(...interface{}) 9 | } 10 | 11 | type testLogger struct { 12 | tbLog 13 | } 14 | 15 | func (tl *testLogger) Output(maxdepth int, s string) error { 16 | tl.Log(s) 17 | return nil 18 | } 19 | 20 | func NewTestLogger(tbl tbLog) Logger { 21 | return &testLogger{tbl} 22 | } 23 | -------------------------------------------------------------------------------- /internal/util/rand.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "math/rand" 5 | ) 6 | 7 | func UniqRands(quantity int, maxval int) []int { 8 | if maxval < quantity { 9 | quantity = maxval 10 | } 11 | 12 | intSlice := make([]int, maxval) 13 | for i := 0; i < maxval; i++ { 14 | intSlice[i] = i 15 | } 16 | 17 | for i := 0; i < quantity; i++ { 18 | j := rand.Int()%maxval + i 19 | // swap 20 | intSlice[i], intSlice[j] = intSlice[j], intSlice[i] 21 | maxval-- 22 | 23 | } 24 | return intSlice[0:quantity] 25 | } 26 | -------------------------------------------------------------------------------- /internal/util/unix_socket.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | func TypeOfAddr(addr string) string { 8 | if _, _, err := net.SplitHostPort(addr); err == nil { 9 | return "tcp" 10 | } 11 | return "unix" 12 | } 13 | -------------------------------------------------------------------------------- /internal/util/util_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/nsqio/nsq/internal/test" 7 | ) 8 | 9 | func BenchmarkUniqRands5of5(b *testing.B) { 10 | for i := 0; i < b.N; i++ { 11 | UniqRands(5, 5) 12 | } 13 | } 14 | func BenchmarkUniqRands20of20(b *testing.B) { 15 | for i := 0; i < b.N; i++ { 16 | UniqRands(20, 20) 17 | } 18 | } 19 | 20 | func BenchmarkUniqRands20of50(b *testing.B) { 21 | for i := 0; i < b.N; i++ { 22 | UniqRands(20, 50) 23 | } 24 | } 25 | 26 | func TestUniqRands(t *testing.T) { 27 | var x []int 28 | x = UniqRands(3, 10) 29 | test.Equal(t, 3, len(x)) 30 | 31 | x = UniqRands(10, 5) 32 | test.Equal(t, 5, len(x)) 33 | 34 | x = UniqRands(10, 20) 35 | test.Equal(t, 10, len(x)) 36 | } 37 | 38 | func TestTypeOfAddr(t *testing.T) { 39 | var x string 40 | x = TypeOfAddr("127.0.0.1:80") 41 | test.Equal(t, "tcp", x) 42 | 43 | x = TypeOfAddr("test:80") 44 | test.Equal(t, "tcp", x) 45 | 46 | x = TypeOfAddr("/var/run/nsqd.sock") 47 | test.Equal(t, "unix", x) 48 | 49 | x = TypeOfAddr("[::1%lo0]:80") 50 | test.Equal(t, "tcp", x) 51 | } 52 | -------------------------------------------------------------------------------- /internal/util/wait_group_wrapper.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type WaitGroupWrapper struct { 8 | sync.WaitGroup 9 | } 10 | 11 | func (w *WaitGroupWrapper) Wrap(cb func()) { 12 | w.Add(1) 13 | go func() { 14 | cb() 15 | w.Done() 16 | }() 17 | } 18 | -------------------------------------------------------------------------------- /internal/version/binary.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | ) 7 | 8 | const Binary = "1.3.0" 9 | 10 | func String(app string) string { 11 | return fmt.Sprintf("%s v%s (built w/%s)", app, Binary, runtime.Version()) 12 | } 13 | -------------------------------------------------------------------------------- /internal/writers/boundary_buffered_writer.go: -------------------------------------------------------------------------------- 1 | package writers 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | ) 7 | 8 | type BoundaryBufferedWriter struct { 9 | bw *bufio.Writer 10 | } 11 | 12 | func NewBoundaryBufferedWriter(w io.Writer, size int) *BoundaryBufferedWriter { 13 | return &BoundaryBufferedWriter{ 14 | bw: bufio.NewWriterSize(w, size), 15 | } 16 | } 17 | 18 | func (b *BoundaryBufferedWriter) Write(p []byte) (int, error) { 19 | if len(p) > b.bw.Available() { 20 | err := b.bw.Flush() 21 | if err != nil { 22 | return 0, err 23 | } 24 | } 25 | return b.bw.Write(p) 26 | } 27 | 28 | func (b *BoundaryBufferedWriter) Flush() error { 29 | return b.bw.Flush() 30 | } 31 | -------------------------------------------------------------------------------- /internal/writers/spread_writer.go: -------------------------------------------------------------------------------- 1 | package writers 2 | 3 | import ( 4 | "io" 5 | "time" 6 | ) 7 | 8 | type SpreadWriter struct { 9 | w io.Writer 10 | interval time.Duration 11 | buf [][]byte 12 | exitCh chan int 13 | } 14 | 15 | func NewSpreadWriter(w io.Writer, interval time.Duration, exitCh chan int) *SpreadWriter { 16 | return &SpreadWriter{ 17 | w: w, 18 | interval: interval, 19 | buf: make([][]byte, 0), 20 | exitCh: exitCh, 21 | } 22 | } 23 | 24 | func (s *SpreadWriter) Write(p []byte) (int, error) { 25 | b := make([]byte, len(p)) 26 | copy(b, p) 27 | s.buf = append(s.buf, b) 28 | return len(p), nil 29 | } 30 | 31 | func (s *SpreadWriter) Flush() { 32 | if len(s.buf) == 0 { 33 | // nothing to write, just wait 34 | select { 35 | case <-time.After(s.interval): 36 | case <-s.exitCh: 37 | } 38 | return 39 | } 40 | sleep := s.interval / time.Duration(len(s.buf)) 41 | ticker := time.NewTicker(sleep) 42 | for _, b := range s.buf { 43 | s.w.Write(b) 44 | select { 45 | case <-ticker.C: 46 | case <-s.exitCh: // skip sleeps finish writes 47 | } 48 | } 49 | ticker.Stop() 50 | s.buf = s.buf[:0] 51 | } 52 | -------------------------------------------------------------------------------- /nsqadmin/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2020": true 5 | }, 6 | // Rule docs: http://eslint.org/docs/rules/ 7 | "rules": { 8 | "block-scoped-var": [1], 9 | "brace-style": [1, "1tbs", {"allowSingleLine": true}], 10 | "camelcase": [1], 11 | "comma-spacing": [1], 12 | "comma-style": [1], 13 | "computed-property-spacing": [1, "never"], 14 | "consistent-return": [1], 15 | "consistent-this": [1, "self"], 16 | "curly": [2], 17 | "dot-notation": [0], 18 | "eol-last": [1], 19 | "eqeqeq": [1], 20 | "indent": [1, 4], 21 | "key-spacing": [1], 22 | "max-len": [1, 100], 23 | "max-nested-callbacks": [2, 3], // ???? 24 | "new-cap": [1], 25 | "new-parens": [1], 26 | "no-caller": [2], 27 | "no-console": [0], 28 | "no-eval": [2], 29 | "no-extend-native": [2], 30 | "no-extra-bind": [1], 31 | "no-floating-decimal": [1], 32 | "no-iterator": [1], 33 | "no-lone-blocks": [1], 34 | "no-lonely-if": [1], 35 | "no-mixed-requires": [0], 36 | "no-mixed-spaces-and-tabs": [1], 37 | "no-multi-spaces": [1], 38 | "no-multi-str": [1], 39 | "no-multiple-empty-lines": [2, {"max": 2}], 40 | "no-native-reassign": [1], 41 | "no-new": [0], 42 | "no-redeclare": [1], 43 | "no-shadow": [1], 44 | "no-spaced-func": [1], 45 | "no-throw-literal": [1], 46 | "no-trailing-spaces": [1], 47 | "no-undef": [1], 48 | "no-underscore-dangle": [0], 49 | "no-unneeded-ternary": [1], 50 | "no-unused-vars": [1], 51 | "no-use-before-define": [1, "nofunc"], 52 | "no-with": [2], 53 | "one-var": [1, "never"], 54 | "quotes": [1, "single"], 55 | "radix": [1], 56 | "semi": [1, "always"], 57 | "semi-spacing": [1], 58 | "keyword-spacing": [1, {"before": true, "after": true}], 59 | "space-before-blocks": [1, "always"], 60 | "space-before-function-paren": [1, {"anonymous": "never", "named": "never", "asyncArrow": "always"}], 61 | "space-in-parens": [1, "never"], 62 | "space-infix-ops": [1], 63 | "space-unary-ops": [1], 64 | "strict": [0], 65 | "wrap-iife": [1] 66 | }, 67 | "globals": { 68 | "BASE_PATH": true, 69 | "GRAPHITE_URL": true, 70 | "GRAPH_ENABLED": true, 71 | "IS_ADMIN": true, 72 | "NSQLOOKUPD": true, 73 | "STATSD_COUNTER_FORMAT": true, 74 | "STATSD_GAUGE_FORMAT": true, 75 | "STATSD_INTERVAL": true, 76 | "STATSD_PREFIX": true, 77 | "USER_AGENT": true, 78 | "VERSION": true, 79 | "module": true, 80 | "require": true 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /nsqadmin/README.md: -------------------------------------------------------------------------------- 1 | ## nsqadmin 2 | 3 | `nsqadmin` is a Web UI to view aggregated cluster stats in realtime and perform various 4 | administrative tasks. 5 | 6 | Read the [docs](https://nsq.io/components/nsqadmin.html) 7 | 8 | 9 | ## Local Development 10 | 11 | ### Dependencies 12 | 13 | 1. Install NodeJS 16.x (includes `npm`) 14 | 15 | ### Live Reload Workflow 16 | 17 | 1. `$ npm install` 18 | 2. `$ ./gulp --series clean watch` 19 | 3. `$ cd .. && make && ./build/nsqadmin --dev-static-dir=nsqadmin/static/build --lookupd-http-address=<...>` 20 | 4. make changes to static assets (repeat step 3 only if you make changes to any Go code) 21 | 22 | ### Build 23 | 24 | 1. `$ ./gulp --series clean build` 25 | -------------------------------------------------------------------------------- /nsqadmin/gulp: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec ./node_modules/.bin/gulp "$@" 3 | -------------------------------------------------------------------------------- /nsqadmin/logger.go: -------------------------------------------------------------------------------- 1 | package nsqadmin 2 | 3 | import ( 4 | "github.com/nsqio/nsq/internal/lg" 5 | ) 6 | 7 | type Logger lg.Logger 8 | 9 | const ( 10 | LOG_DEBUG = lg.DEBUG 11 | LOG_INFO = lg.INFO 12 | LOG_WARN = lg.WARN 13 | LOG_ERROR = lg.ERROR 14 | LOG_FATAL = lg.FATAL 15 | ) 16 | 17 | func (n *NSQAdmin) logf(level lg.LogLevel, f string, args ...interface{}) { 18 | opts := n.getOpts() 19 | lg.Logf(opts.Logger, opts.LogLevel, level, f, args...) 20 | } 21 | -------------------------------------------------------------------------------- /nsqadmin/notify.go: -------------------------------------------------------------------------------- 1 | package nsqadmin 2 | 3 | import ( 4 | "encoding/base64" 5 | "net/http" 6 | "net/url" 7 | "os" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | type AdminAction struct { 13 | Action string `json:"action"` 14 | Topic string `json:"topic"` 15 | Channel string `json:"channel,omitempty"` 16 | Node string `json:"node,omitempty"` 17 | Timestamp int64 `json:"timestamp"` 18 | User string `json:"user,omitempty"` 19 | RemoteIP string `json:"remote_ip"` 20 | UserAgent string `json:"user_agent"` 21 | URL string `json:"url"` // The URL of the HTTP request that triggered this action 22 | Via string `json:"via"` // the Hostname of the nsqadmin performing this action 23 | } 24 | 25 | func basicAuthUser(req *http.Request) string { 26 | s := strings.SplitN(req.Header.Get("Authorization"), " ", 2) 27 | if len(s) != 2 || s[0] != "Basic" { 28 | return "" 29 | } 30 | b, err := base64.StdEncoding.DecodeString(s[1]) 31 | if err != nil { 32 | return "" 33 | } 34 | pair := strings.SplitN(string(b), ":", 2) 35 | if len(pair) != 2 { 36 | return "" 37 | } 38 | return pair[0] 39 | } 40 | 41 | func (s *httpServer) notifyAdminAction(action, topic, channel, node string, req *http.Request) { 42 | if s.nsqadmin.getOpts().NotificationHTTPEndpoint == "" { 43 | return 44 | } 45 | via, _ := os.Hostname() 46 | 47 | u := url.URL{ 48 | Scheme: "http", 49 | Host: req.Host, 50 | Path: req.URL.Path, 51 | RawQuery: req.URL.RawQuery, 52 | } 53 | if req.TLS != nil || req.Header.Get("X-Scheme") == "https" { 54 | u.Scheme = "https" 55 | } 56 | 57 | a := &AdminAction{ 58 | Action: action, 59 | Topic: topic, 60 | Channel: channel, 61 | Node: node, 62 | Timestamp: time.Now().Unix(), 63 | User: basicAuthUser(req), 64 | RemoteIP: req.RemoteAddr, 65 | UserAgent: req.UserAgent(), 66 | URL: u.String(), 67 | Via: via, 68 | } 69 | // Perform all work in a new goroutine so this never blocks 70 | go func() { s.nsqadmin.notifications <- a }() 71 | } 72 | -------------------------------------------------------------------------------- /nsqadmin/nsqadmin_test.go: -------------------------------------------------------------------------------- 1 | package nsqadmin 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/http" 7 | "net/url" 8 | "os" 9 | "testing" 10 | 11 | "github.com/nsqio/nsq/internal/lg" 12 | "github.com/nsqio/nsq/internal/test" 13 | "github.com/nsqio/nsq/nsqd" 14 | ) 15 | 16 | func TestNeitherNSQDAndNSQLookup(t *testing.T) { 17 | opts := NewOptions() 18 | opts.Logger = lg.NilLogger{} 19 | opts.HTTPAddress = "127.0.0.1:0" 20 | _, err := New(opts) 21 | test.NotNil(t, err) 22 | test.Equal(t, "--nsqd-http-address or --lookupd-http-address required", fmt.Sprintf("%s", err)) 23 | } 24 | 25 | func TestBothNSQDAndNSQLookup(t *testing.T) { 26 | opts := NewOptions() 27 | opts.Logger = lg.NilLogger{} 28 | opts.HTTPAddress = "127.0.0.1:0" 29 | opts.NSQLookupdHTTPAddresses = []string{"127.0.0.1:4161"} 30 | opts.NSQDHTTPAddresses = []string{"127.0.0.1:4151"} 31 | _, err := New(opts) 32 | test.NotNil(t, err) 33 | test.Equal(t, "use --nsqd-http-address or --lookupd-http-address not both", fmt.Sprintf("%s", err)) 34 | } 35 | 36 | func TestTLSHTTPClient(t *testing.T) { 37 | lgr := test.NewTestLogger(t) 38 | 39 | nsqdOpts := nsqd.NewOptions() 40 | nsqdOpts.TLSCert = "./test/server.pem" 41 | nsqdOpts.TLSKey = "./test/server.key" 42 | nsqdOpts.TLSRootCAFile = "./test/ca.pem" 43 | nsqdOpts.TLSClientAuthPolicy = "require-verify" 44 | nsqdOpts.Logger = lgr 45 | _, nsqdHTTPAddr, nsqd := mustStartNSQD(nsqdOpts) 46 | defer os.RemoveAll(nsqdOpts.DataPath) 47 | defer nsqd.Exit() 48 | 49 | opts := NewOptions() 50 | opts.HTTPAddress = "127.0.0.1:0" 51 | opts.NSQDHTTPAddresses = []string{nsqdHTTPAddr.String()} 52 | opts.HTTPClientTLSRootCAFile = "./test/ca.pem" 53 | opts.HTTPClientTLSCert = "./test/client.pem" 54 | opts.HTTPClientTLSKey = "./test/client.key" 55 | opts.Logger = lgr 56 | nsqadmin, err := New(opts) 57 | test.Nil(t, err) 58 | go func() { 59 | err := nsqadmin.Main() 60 | if err != nil { 61 | panic(err) 62 | } 63 | }() 64 | defer nsqadmin.Exit() 65 | 66 | httpAddr := nsqadmin.RealHTTPAddr() 67 | u := url.URL{ 68 | Scheme: "http", 69 | Host: httpAddr.String(), 70 | Path: "/api/nodes/" + nsqdHTTPAddr.String(), 71 | } 72 | 73 | resp, err := http.Get(u.String()) 74 | test.Nil(t, err) 75 | defer resp.Body.Close() 76 | 77 | test.Equal(t, resp.StatusCode < 500, true) 78 | } 79 | 80 | func mustStartNSQD(opts *nsqd.Options) (net.Addr, net.Addr, *nsqd.NSQD) { 81 | opts.TCPAddress = "127.0.0.1:0" 82 | opts.HTTPAddress = "127.0.0.1:0" 83 | opts.HTTPSAddress = "127.0.0.1:0" 84 | if opts.DataPath == "" { 85 | tmpDir, err := os.MkdirTemp("", "nsq-test-") 86 | if err != nil { 87 | panic(err) 88 | } 89 | opts.DataPath = tmpDir 90 | } 91 | nsqd, err := nsqd.New(opts) 92 | if err != nil { 93 | panic(err) 94 | } 95 | go func() { 96 | err := nsqd.Main() 97 | if err != nil { 98 | panic(err) 99 | } 100 | }() 101 | return nsqd.RealTCPAddr(), nsqd.RealHTTPAddr(), nsqd 102 | } 103 | -------------------------------------------------------------------------------- /nsqadmin/options.go: -------------------------------------------------------------------------------- 1 | package nsqadmin 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/nsqio/nsq/internal/lg" 7 | ) 8 | 9 | type Options struct { 10 | LogLevel lg.LogLevel `flag:"log-level"` 11 | LogPrefix string `flag:"log-prefix"` 12 | Logger Logger 13 | 14 | HTTPAddress string `flag:"http-address"` 15 | BasePath string `flag:"base-path"` 16 | 17 | DevStaticDir string `flag:"dev-static-dir"` 18 | 19 | GraphiteURL string `flag:"graphite-url"` 20 | ProxyGraphite bool `flag:"proxy-graphite"` 21 | 22 | StatsdPrefix string `flag:"statsd-prefix"` 23 | StatsdCounterFormat string `flag:"statsd-counter-format"` 24 | StatsdGaugeFormat string `flag:"statsd-gauge-format"` 25 | 26 | StatsdInterval time.Duration `flag:"statsd-interval"` 27 | 28 | NSQLookupdHTTPAddresses []string `flag:"lookupd-http-address" cfg:"nsqlookupd_http_addresses"` 29 | NSQDHTTPAddresses []string `flag:"nsqd-http-address" cfg:"nsqd_http_addresses"` 30 | 31 | HTTPClientConnectTimeout time.Duration `flag:"http-client-connect-timeout"` 32 | HTTPClientRequestTimeout time.Duration `flag:"http-client-request-timeout"` 33 | 34 | HTTPClientTLSInsecureSkipVerify bool `flag:"http-client-tls-insecure-skip-verify"` 35 | HTTPClientTLSRootCAFile string `flag:"http-client-tls-root-ca-file"` 36 | HTTPClientTLSCert string `flag:"http-client-tls-cert"` 37 | HTTPClientTLSKey string `flag:"http-client-tls-key"` 38 | 39 | AllowConfigFromCIDR string `flag:"allow-config-from-cidr"` 40 | 41 | NotificationHTTPEndpoint string `flag:"notification-http-endpoint"` 42 | 43 | ACLHTTPHeader string `flag:"acl-http-header"` 44 | AdminUsers []string `flag:"admin-user" cfg:"admin_users"` 45 | } 46 | 47 | func NewOptions() *Options { 48 | return &Options{ 49 | LogPrefix: "[nsqadmin] ", 50 | LogLevel: lg.INFO, 51 | HTTPAddress: "0.0.0.0:4171", 52 | BasePath: "/", 53 | StatsdPrefix: "nsq.%s", 54 | StatsdCounterFormat: "stats.counters.%s.count", 55 | StatsdGaugeFormat: "stats.gauges.%s", 56 | StatsdInterval: 60 * time.Second, 57 | HTTPClientConnectTimeout: 2 * time.Second, 58 | HTTPClientRequestTimeout: 5 * time.Second, 59 | AllowConfigFromCIDR: "127.0.0.1/8", 60 | ACLHTTPHeader: "X-Forwarded-User", 61 | AdminUsers: []string{}, 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /nsqadmin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nsqadmin", 3 | "version": "0.3.0", 4 | "description": "operational dashboard for NSQ (https://nsq.io/)", 5 | "repository": { 6 | "type": "git", 7 | "url": "" 8 | }, 9 | "scripts": { 10 | "lint": "eslint static/js" 11 | }, 12 | "devDependencies": { 13 | "browserify": "^17.0.0", 14 | "eslint": "^8.52.0", 15 | "gulp": "^4.0.2", 16 | "gulp-clean": "^0.4.0", 17 | "gulp-cli": "^2.3.0", 18 | "gulp-dart-sass": "^1.0.2", 19 | "gulp-notify": "^4.0.0", 20 | "gulp-sourcemaps": "^3.0.0", 21 | "gulp-task-listing": "^1.1.0", 22 | "gulp-uglify": "^3.0.2", 23 | "handlebars": "^4.7.7", 24 | "hbsfy": "^2.8.1", 25 | "vinyl-buffer": "^1.0.1", 26 | "vinyl-source-stream": "^2.0.0" 27 | }, 28 | "dependencies": { 29 | "backbone": "^1.4.0", 30 | "bootbox": "^5.5.2", 31 | "bootstrap": "^3.4.1", 32 | "jquery": "^3.6.0", 33 | "moment": "^2.29.1", 34 | "underscore": "^1.13.1" 35 | }, 36 | "browserify": { 37 | "transform": [ 38 | "hbsfy" 39 | ] 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /nsqadmin/static.go: -------------------------------------------------------------------------------- 1 | package nsqadmin 2 | 3 | import ( 4 | "embed" 5 | ) 6 | 7 | //go:embed static/build 8 | var static embed.FS 9 | 10 | func staticAsset(name string) ([]byte, error) { 11 | return static.ReadFile("static/build/" + name) 12 | } 13 | -------------------------------------------------------------------------------- /nsqadmin/static/build/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/build/favicon.png -------------------------------------------------------------------------------- /nsqadmin/static/build/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/build/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /nsqadmin/static/build/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/build/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /nsqadmin/static/build/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/build/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /nsqadmin/static/build/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/build/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /nsqadmin/static/build/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | nsqadmin 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |
15 | 16 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /nsqadmin/static/build/nsq_blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/build/nsq_blue.png -------------------------------------------------------------------------------- /nsqadmin/static/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /nsqadmin/static/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /nsqadmin/static/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /nsqadmin/static/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /nsqadmin/static/html/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | nsqadmin 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |
15 | 16 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /nsqadmin/static/img/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/img/favicon.png -------------------------------------------------------------------------------- /nsqadmin/static/img/nsq_blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nsqio/nsq/3103474b6c5afe8feca0a598797e82c06ed726e3/nsqadmin/static/img/nsq_blue.png -------------------------------------------------------------------------------- /nsqadmin/static/js/app_state.js: -------------------------------------------------------------------------------- 1 | var Backbone = require('backbone'); 2 | var _ = require('underscore'); 3 | 4 | var AppState = Backbone.Model.extend({ 5 | defaults: function() { 6 | return { 7 | 'VERSION': VERSION, 8 | 'GRAPHITE_URL': GRAPHITE_URL, 9 | 'GRAPH_ENABLED': GRAPH_ENABLED, 10 | 'STATSD_INTERVAL': STATSD_INTERVAL, 11 | 'STATSD_COUNTER_FORMAT': STATSD_COUNTER_FORMAT, 12 | 'STATSD_GAUGE_FORMAT': STATSD_GAUGE_FORMAT, 13 | 'STATSD_PREFIX': STATSD_PREFIX, 14 | 'NSQLOOKUPD': NSQLOOKUPD, 15 | 'graph_interval': '2h', 16 | 'IS_ADMIN': IS_ADMIN, 17 | 'BASE_PATH': BASE_PATH 18 | }; 19 | }, 20 | 21 | initialize: function() { 22 | this.on('change:graph_interval', function(model, v) { 23 | localStorage.setItem('graph_interval', v); 24 | }); 25 | 26 | var qp = _.object(_.compact(_.map(window.location.search.slice(1).split('&'), 27 | function(item) { return item ? item.split('=') : false; }))); 28 | 29 | var def = this.get('GRAPH_ENABLED') ? '2h' : 'off'; 30 | var interval = qp['t'] || localStorage.getItem('graph_interval') || def; 31 | this.set('graph_interval', interval); 32 | }, 33 | 34 | basePath: function(p) { 35 | // if base path is / then don't prefix 36 | var bp = this.get('BASE_PATH') === '/' ? '' : this.get('BASE_PATH'); 37 | // remove trailing /, but guarantee at least / 38 | return (bp + p).replace(/\/$/, '') || '/'; 39 | }, 40 | 41 | apiPath: function(p) { 42 | return this.basePath('/api' + p); 43 | } 44 | }); 45 | 46 | var appState = new AppState(); 47 | 48 | window.AppState = appState; 49 | 50 | module.exports = appState; 51 | -------------------------------------------------------------------------------- /nsqadmin/static/js/collections/nodes.js: -------------------------------------------------------------------------------- 1 | var Backbone = require('backbone'); 2 | 3 | var AppState = require('../app_state'); 4 | 5 | var NodeModel = require('../models/node'); 6 | 7 | var Nodes = Backbone.Collection.extend({ 8 | model: NodeModel, 9 | 10 | comparator: 'id', 11 | 12 | constructor: function Nodes() { 13 | Backbone.Collection.prototype.constructor.apply(this, arguments); 14 | }, 15 | 16 | url: function() { 17 | return AppState.apiPath('/nodes'); 18 | }, 19 | 20 | parse: function(resp) { 21 | resp['nodes'].forEach(function(n) { 22 | var jaddr = n['broadcast_address']; 23 | if (jaddr.includes(':')) { 24 | // ipv6 raw address contains ':' 25 | // it must be wrapped in '[ ]' when joined with port 26 | jaddr = '[' + jaddr + ']'; 27 | } 28 | n['broadcast_address_http'] = jaddr + ':' + n['http_port']; 29 | }); 30 | return resp['nodes']; 31 | } 32 | }); 33 | 34 | module.exports = Nodes; 35 | -------------------------------------------------------------------------------- /nsqadmin/static/js/collections/topics.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var Backbone = require('backbone'); 3 | 4 | var AppState = require('../app_state'); 5 | 6 | var Topic = require('../models/topic'); 7 | 8 | var Topics = Backbone.Collection.extend({ 9 | model: Topic, 10 | 11 | comparator: 'id', 12 | 13 | constructor: function Topics() { 14 | Backbone.Collection.prototype.constructor.apply(this, arguments); 15 | }, 16 | 17 | url: function() { 18 | return AppState.apiPath('/topics'); 19 | }, 20 | 21 | parse: function(resp) { 22 | var topics = _.map(resp['topics'], function(name) { 23 | return {'name': name}; 24 | }); 25 | return topics; 26 | } 27 | }); 28 | 29 | module.exports = Topics; 30 | -------------------------------------------------------------------------------- /nsqadmin/static/js/lib/ajax_setup.js: -------------------------------------------------------------------------------- 1 | var $ = require('jquery'); 2 | var _ = require('underscore'); 3 | 4 | // Set up some headers and options for every request. 5 | $.ajaxPrefilter(function(options) { 6 | options['headers'] = _.defaults(options['headers'] || {}, { 7 | 'X-UserAgent': USER_AGENT, 8 | 'Accept': 'application/vnd.nsq; version=1.0' 9 | }); 10 | options['timeout'] = 20 * 1000; 11 | options['contentType'] = 'application/json'; 12 | }); 13 | -------------------------------------------------------------------------------- /nsqadmin/static/js/lib/pubsub.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var Backbone = require('backbone'); 3 | 4 | var Pubsub = _.clone(Backbone.Events); 5 | 6 | // making this global to more easily trigger events from the console 7 | window.Pubsub = Pubsub; 8 | 9 | module.exports = Pubsub; 10 | -------------------------------------------------------------------------------- /nsqadmin/static/js/main.js: -------------------------------------------------------------------------------- 1 | var $ = require('jquery'); 2 | var Backbone = require('backbone'); 3 | 4 | // var Pubsub = require('./lib/pubsub'); 5 | var Router = require('./router'); 6 | 7 | var AppView = require('./views/app'); 8 | 9 | // When using browserify, we need to tell Backbone what jQuery to use. 10 | Backbone.$ = $; 11 | 12 | // Side effects: 13 | require('./lib/ajax_setup'); 14 | require('./lib/handlebars_helpers'); 15 | 16 | var start = function() { 17 | new AppView(); 18 | Router.start(); 19 | }; 20 | 21 | // Pubsub.on('all', function() { 22 | // console.log.apply(console, arguments); 23 | // }); 24 | 25 | start(); 26 | -------------------------------------------------------------------------------- /nsqadmin/static/js/models/channel.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | 3 | var AppState = require('../app_state'); 4 | var Backbone = require('backbone'); 5 | 6 | var Channel = Backbone.Model.extend({ 7 | idAttribute: 'name', 8 | 9 | constructor: function Channel() { 10 | Backbone.Model.prototype.constructor.apply(this, arguments); 11 | }, 12 | 13 | url: function() { 14 | return AppState.apiPath('/topics/' + 15 | encodeURIComponent(this.get('topic')) + '/' + 16 | encodeURIComponent(this.get('name'))); 17 | }, 18 | 19 | parse: function(response) { 20 | response['nodes'] = _.map(response['nodes'] || [], function(node) { 21 | var nodeParts = node['node'].split(':'); 22 | var port = nodeParts.pop(); 23 | var address = nodeParts.join(':'); 24 | var hostname = node['hostname']; 25 | var zonecount = node['zone_local_msg_count']; 26 | var deliverycount = node['delivery_msg_count']; 27 | var regioncount = node['region_local_msg_count']; 28 | var globalcount = node['global_msg_count']; 29 | node['show_broadcast_address'] = hostname.toLowerCase() !== address.toLowerCase(); 30 | node['hostname_port'] = hostname + ':' + port; 31 | node['zone_local_percentage'] = zonecount / deliverycount; 32 | node['region_local_percentage'] = regioncount / deliverycount; 33 | node['global_percentage'] = globalcount / deliverycount; 34 | if (isNaN(node['zone_local_percentage'])) { 35 | node['zone_local_percentage'] = 0; 36 | } 37 | if (isNaN(node['region_local_percentage'])) { 38 | node['region_local_percentage'] = 0; 39 | } 40 | if (isNaN(node['global_percentage'])) { 41 | node['global_percentage'] = 0; 42 | } 43 | return node; 44 | }); 45 | 46 | response['clients'] = _.map(response['clients'] || [], function(client) { 47 | var clientId = client['client_id']; 48 | var hostname = client['hostname']; 49 | var shortHostname = hostname.split('.')[0]; 50 | 51 | // ignore client_id if it's duplicative 52 | client['show_client_id'] = (clientId.toLowerCase() !== shortHostname.toLowerCase() 53 | && clientId.toLowerCase() !== hostname.toLowerCase()); 54 | 55 | var port = client['remote_address'].split(':').pop(); 56 | client['hostname_port'] = hostname + ':' + port; 57 | 58 | return client; 59 | }); 60 | 61 | return response; 62 | } 63 | }); 64 | 65 | module.exports = Channel; 66 | -------------------------------------------------------------------------------- /nsqadmin/static/js/models/node.js: -------------------------------------------------------------------------------- 1 | var AppState = require('../app_state'); 2 | var Backbone = require('backbone'); 3 | 4 | var NodeModel = Backbone.Model.extend({ 5 | idAttribute: 'name', 6 | 7 | constructor: function Node() { 8 | Backbone.Model.prototype.constructor.apply(this, arguments); 9 | }, 10 | 11 | urlRoot: function() { 12 | return AppState.apiPath('/nodes'); 13 | }, 14 | 15 | tombstoneTopic: function(topic) { 16 | return this.destroy({ 17 | 'data': JSON.stringify({'topic': topic}), 18 | 'dataType': 'text' 19 | }); 20 | } 21 | }); 22 | 23 | module.exports = NodeModel; 24 | -------------------------------------------------------------------------------- /nsqadmin/static/js/models/topic.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | 3 | var AppState = require('../app_state'); 4 | var Backbone = require('backbone'); 5 | 6 | var Topic = Backbone.Model.extend({ 7 | idAttribute: 'name', 8 | 9 | constructor: function Topic() { 10 | Backbone.Model.prototype.constructor.apply(this, arguments); 11 | }, 12 | 13 | url: function() { 14 | return AppState.apiPath('/topics/' + encodeURIComponent(this.get('name'))); 15 | }, 16 | 17 | parse: function(response) { 18 | response['nodes'] = _.map(response['nodes'] || [], function(node) { 19 | var nodeParts = node['node'].split(':'); 20 | var port = nodeParts.pop(); 21 | var address = nodeParts.join(':'); 22 | var hostname = node['hostname']; 23 | node['show_broadcast_address'] = hostname.toLowerCase() !== address.toLowerCase(); 24 | node['hostname_port'] = hostname + ':' + port; 25 | return node; 26 | }); 27 | return response; 28 | } 29 | }); 30 | 31 | module.exports = Topic; 32 | -------------------------------------------------------------------------------- /nsqadmin/static/js/router.js: -------------------------------------------------------------------------------- 1 | var Backbone = require('backbone'); 2 | 3 | var AppState = require('./app_state'); 4 | var Pubsub = require('./lib/pubsub'); 5 | 6 | 7 | var Router = Backbone.Router.extend({ 8 | initialize: function() { 9 | var bp = function(p) { 10 | // remove leading slash 11 | return AppState.basePath(p).substring(1); 12 | }; 13 | this.route(bp('/'), 'topics'); 14 | this.route(bp('/topics/(:topic)(/:channel)'), 'topic'); 15 | this.route(bp('/lookup'), 'lookup'); 16 | this.route(bp('/nodes(/:node)'), 'nodes'); 17 | this.route(bp('/counter'), 'counter'); 18 | // this.listenTo(this, 'route', function(route, params) { 19 | // console.log('Route: %o; params: %o', route, params); 20 | // }); 21 | }, 22 | 23 | start: function() { 24 | Backbone.history.start({ 25 | 'pushState': true 26 | }); 27 | }, 28 | 29 | topics: function() { 30 | Pubsub.trigger('topics:show'); 31 | }, 32 | 33 | topic: function(topic, channel) { 34 | if (channel !== null) { 35 | Pubsub.trigger('channel:show', topic, channel); 36 | return; 37 | } 38 | Pubsub.trigger('topic:show', topic); 39 | }, 40 | 41 | lookup: function() { 42 | Pubsub.trigger('lookup:show'); 43 | }, 44 | 45 | nodes: function(node) { 46 | if (node !== null) { 47 | Pubsub.trigger('node:show', node); 48 | return; 49 | } 50 | Pubsub.trigger('nodes:show'); 51 | }, 52 | 53 | counter: function() { 54 | Pubsub.trigger('counter:show'); 55 | } 56 | }); 57 | 58 | 59 | module.exports = new Router(); 60 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/base.js: -------------------------------------------------------------------------------- 1 | var $ = require('jquery'); 2 | var _ = require('underscore'); 3 | var Backbone = require('backbone'); 4 | 5 | var AppState = require('../app_state'); 6 | 7 | var errorTemplate = require('./error.hbs'); 8 | 9 | 10 | var BaseView = Backbone.View.extend({ 11 | constructor: function(options) { 12 | // As of 1.10, Backbone no longer automatically attaches options passed 13 | // to the constructor as this.options, but that's often useful in some 14 | // cases, like a className function, that happen before initialize() 15 | // would have a chance to attach the same options. 16 | this.options = options || {}; 17 | return Backbone.View.prototype.constructor.apply(this, arguments); 18 | }, 19 | 20 | initialize: function() { 21 | this.subviews = []; 22 | this.rendered = false; 23 | }, 24 | 25 | template: function() {}, 26 | 27 | skippedRender: function() {}, 28 | 29 | render: function(data) { 30 | if (this.renderOnce && this.rendered) { 31 | this.skippedRender(); 32 | return this; 33 | } 34 | this.removeSubviews(); 35 | var ctx = this.getRenderCtx(data); 36 | // console.log('render ctx: %o', ctx); 37 | var html = this.template(ctx); 38 | if (!this.removed) { 39 | this.$el.empty(); 40 | this.$el.append(html); 41 | this.postRender(ctx); 42 | } 43 | this.rendered = true; 44 | return this; 45 | }, 46 | 47 | getRenderCtx: function(data) { 48 | var ctx = { 49 | 'graph_enabled': AppState.get('GRAPH_ENABLED'), 50 | 'graph_interval': AppState.get('graph_interval'), 51 | 'graph_active': AppState.get('GRAPH_ENABLED') && 52 | AppState.get('graph_interval') !== 'off', 53 | 'nsqlookupd': AppState.get('NSQLOOKUPD'), 54 | 'version': AppState.get('VERSION') 55 | }; 56 | if (this.model) { 57 | ctx = _.extend(ctx, this.model.toJSON()); 58 | } else if (this.collection) { 59 | ctx = _.extend(ctx, {'collection': this.collection.toJSON()}); 60 | } 61 | if (data) { 62 | ctx = _.extend(ctx, data); 63 | } 64 | return ctx; 65 | }, 66 | 67 | postRender: function() {}, 68 | 69 | appendSubview: function(subview, selector) { 70 | return this.appendSubviews([subview], selector); 71 | }, 72 | 73 | appendSubviews: function(subviews, selector) { 74 | this.subviews.push.apply(this.subviews, subviews); 75 | var $el = selector ? this.$(selector) : this.$el; 76 | $el.append(subviews.map(function(subview) { 77 | return subview.render().delegateEvents().el; 78 | })); 79 | }, 80 | 81 | removeSubviews: function() { 82 | while (this.subviews.length) { 83 | this.subviews.pop().remove(); 84 | } 85 | }, 86 | 87 | remove: function() { 88 | this.removed = true; 89 | this.removeSubviews(); 90 | Backbone.View.prototype.remove.apply(this, arguments); 91 | }, 92 | 93 | parseErrorMessage: function(jqXHR) { 94 | var msg = 'ERROR: failed to connect to nsqadmin'; 95 | if (jqXHR.readyState === 4) { 96 | try { 97 | var parsed = JSON.parse(jqXHR.responseText); 98 | msg = parsed['message']; 99 | } catch (err) { 100 | msg = 'ERROR: failed to decode JSON - ' + err.message; 101 | } 102 | } 103 | return msg; 104 | }, 105 | 106 | handleAJAXError: function(jqXHR) { 107 | $('#warning, #error').hide(); 108 | $('#error .alert').text(this.parseErrorMessage(jqXHR)); 109 | $('#error').show(); 110 | }, 111 | 112 | handleViewError: function(jqXHR) { 113 | this.removeSubviews(); 114 | this.$el.html(errorTemplate({'message': this.parseErrorMessage(jqXHR)})); 115 | } 116 | }); 117 | 118 | module.exports = BaseView; 119 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/channel.js: -------------------------------------------------------------------------------- 1 | var $ = require('jquery'); 2 | 3 | window.jQuery = $; 4 | var bootstrap = require('bootstrap'); //eslint-disable-line no-unused-vars 5 | var bootbox = require('bootbox'); 6 | 7 | var Pubsub = require('../lib/pubsub'); 8 | var AppState = require('../app_state'); 9 | 10 | var BaseView = require('./base'); 11 | 12 | var ChannelView = BaseView.extend({ 13 | className: 'channel container-fluid', 14 | 15 | template: require('./spinner.hbs'), 16 | 17 | events: { 18 | 'click .channel-actions button': 'channelAction', 19 | 'click .popup': 'showDeliveryBreakdown' 20 | }, 21 | 22 | initialize: function() { 23 | BaseView.prototype.initialize.apply(this, arguments); 24 | this.listenTo(AppState, 'change:graph_interval', this.render); 25 | var isAdmin = this.model.get('isAdmin'); 26 | this.model.fetch() 27 | .done(function(data) { 28 | this.template = require('./channel.hbs'); 29 | this.render({'message': data['message'], 'isAdmin': isAdmin}); 30 | }.bind(this)) 31 | .fail(this.handleViewError.bind(this)) 32 | .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); 33 | }, 34 | 35 | showDeliveryBreakdown: function(e) { 36 | e.preventDefault(); 37 | e.stopPropagation(); 38 | var popup = document.getElementById($(e.currentTarget).data('id')); 39 | popup.classList.toggle("show"); 40 | }, 41 | 42 | channelAction: function(e) { 43 | e.preventDefault(); 44 | e.stopPropagation(); 45 | var action = $(e.currentTarget).data('action'); 46 | var txt = 'Are you sure you want to ' + 47 | action + ' ' + this.model.get('topic') + 48 | '/' + this.model.get('name') + '?'; 49 | bootbox.confirm(txt, function(result) { 50 | if (result !== true) { 51 | return; 52 | } 53 | if (action === 'delete') { 54 | var topic = this.model.get('topic'); 55 | $.ajax(this.model.url(), {'method': 'DELETE'}) 56 | .done(function() { 57 | window.location = AppState.basePath('/topics/' + 58 | encodeURIComponent(topic)); 59 | }) 60 | .fail(this.handleAJAXError.bind(this)); 61 | } else { 62 | $.post(this.model.url(), JSON.stringify({'action': action})) 63 | .done(function() { window.location.reload(true); }) 64 | .fail(this.handleAJAXError.bind(this)); 65 | } 66 | }.bind(this)); 67 | } 68 | }); 69 | 70 | module.exports = ChannelView; 71 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/error.hbs: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | {{message}} 5 |
6 |
7 |
8 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/header.hbs: -------------------------------------------------------------------------------- 1 | 39 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/header.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var $ = require('jquery'); 3 | 4 | var AppState = require('../app_state'); 5 | 6 | var BaseView = require('./base'); 7 | 8 | var HeaderView = BaseView.extend({ 9 | className: 'header', 10 | 11 | template: require('./header.hbs'), 12 | 13 | events: { 14 | 'click .dropdown-menu li': 'onGraphIntervalClick' 15 | }, 16 | 17 | initialize: function() { 18 | BaseView.prototype.initialize.apply(this, arguments); 19 | this.listenTo(AppState, 'change:graph_interval', this.render); 20 | }, 21 | 22 | getRenderCtx: function() { 23 | return _.extend(BaseView.prototype.getRenderCtx.apply(this, arguments), { 24 | 'graph_intervals': ['1h', '2h', '12h', '24h', '48h', '168h', 'off'], 25 | 'graph_interval': AppState.get('graph_interval') 26 | }); 27 | }, 28 | 29 | onReset: function() { 30 | this.render(); 31 | this.$('.dropdown-toggle').dropdown(); 32 | }, 33 | 34 | onGraphIntervalClick: function(e) { 35 | e.stopPropagation(); 36 | AppState.set('graph_interval', $(e.target).text()); 37 | } 38 | }); 39 | 40 | module.exports = HeaderView; 41 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/lookup.hbs: -------------------------------------------------------------------------------- 1 | {{> warning}} 2 | {{> error}} 3 | 4 |
5 |
6 |

Lookup

7 |
8 |
9 | 10 | {{#unless nsqlookupd.length}} 11 |
12 |

Notice

nsqadmin is not configured with nsqlookupd hosts 13 |
14 | {{else}} 15 |
16 |
17 | 18 | 19 | 20 | 21 | {{#each nsqlookupd}} 22 | 23 | {{/each}} 24 |
nsqlookupd Host
{{this}}
25 |
26 |
27 | 28 |
29 |
30 | {{#if topics}} 31 |
32 | Below is a tree of Topics/Channels that are currently inactive (i.e. not produced on any nsqd in the cluster but are present in the lookup data) 33 |
34 |
    35 | {{#each topics}} 36 |
  • 37 | {{name}} 38 |
      39 | {{#each channels}} 40 |
    • 41 | {{this}} 42 |
      • 43 | {{/each}} 44 |
    45 |
    • 46 | {{/each}} 47 |
48 | {{else}} 49 |

Notice

No inactive Topics
50 | {{/if}} 51 |
52 |
53 | 54 | {{#if isAdmin}} 55 |
56 |
57 |
58 | Create Topic/Channel 59 |
60 |

This provides a way to setup a stream hierarchy 61 | before services are deployed to production. 62 |

If Channel Name is empty, just the topic is created. 63 |

64 |
65 | 66 | 67 |
68 | 69 |
70 |
71 |
72 | {{/if}} 73 | {{/unless}} 74 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/lookup.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var $ = require('jquery'); 3 | 4 | var AppState = require('../app_state'); 5 | var Pubsub = require('../lib/pubsub'); 6 | var BaseView = require('./base'); 7 | 8 | var Topic = require('../models/topic'); 9 | var Channel = require('../models/channel'); 10 | 11 | var LookupView = BaseView.extend({ 12 | className: 'lookup container-fluid', 13 | 14 | template: require('./spinner.hbs'), 15 | 16 | events: { 17 | 'click .hierarchy button': 'onCreateTopicChannel', 18 | 'click .delete-topic-link': 'onDeleteTopic', 19 | 'click .delete-channel-link': 'onDeleteChannel' 20 | }, 21 | 22 | initialize: function() { 23 | BaseView.prototype.initialize.apply(this, arguments); 24 | var isAdmin = arguments[0]['isAdmin']; 25 | $.ajax(AppState.apiPath('/topics?inactive=true')) 26 | .done(function(data) { 27 | this.template = require('./lookup.hbs'); 28 | this.render({ 29 | 'topics': _.map(data['topics'], function(v, k) { 30 | return {'name': k, 'channels': v}; 31 | }), 32 | 'message': data['message'], 33 | 'isAdmin': isAdmin 34 | }); 35 | }.bind(this)) 36 | .fail(this.handleViewError.bind(this)) 37 | .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); 38 | }, 39 | 40 | onCreateTopicChannel: function(e) { 41 | e.preventDefault(); 42 | e.stopPropagation(); 43 | var topic = $(e.target.form.elements['topic']).val(); 44 | var channel = $(e.target.form.elements['channel']).val(); 45 | if (topic === '' && channel === '') { 46 | return; 47 | } 48 | $.post(AppState.apiPath('/topics'), JSON.stringify({ 49 | 'topic': topic, 50 | 'channel': channel 51 | })) 52 | .done(function() { window.location.reload(true); }) 53 | .fail(this.handleAJAXError.bind(this)); 54 | }, 55 | 56 | onDeleteTopic: function(e) { 57 | e.preventDefault(); 58 | e.stopPropagation(); 59 | var topic = new Topic({ 60 | 'name': $(e.target).data('topic') 61 | }); 62 | topic.destroy({'dataType': 'text'}) 63 | .done(function() { window.location.reload(true); }) 64 | .fail(this.handleAJAXError.bind(this)); 65 | }, 66 | 67 | onDeleteChannel: function(e) { 68 | e.preventDefault(); 69 | e.stopPropagation(); 70 | var channel = new Channel({ 71 | 'topic': $(e.target).data('topic'), 72 | 'name': $(e.target).data('channel') 73 | }); 74 | channel.destroy({'dataType': 'text'}) 75 | .done(function() { window.location.reload(true); }) 76 | .fail(this.handleAJAXError.bind(this)); 77 | } 78 | }); 79 | 80 | module.exports = LookupView; 81 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/node.js: -------------------------------------------------------------------------------- 1 | var Pubsub = require('../lib/pubsub'); 2 | var AppState = require('../app_state'); 3 | 4 | var BaseView = require('./base'); 5 | 6 | var NodeView = BaseView.extend({ 7 | className: 'node container-fluid', 8 | 9 | template: require('./spinner.hbs'), 10 | 11 | initialize: function() { 12 | BaseView.prototype.initialize.apply(this, arguments); 13 | this.listenTo(AppState, 'change:graph_interval', this.render); 14 | this.model.fetch() 15 | .done(function(data) { 16 | this.template = require('./node.hbs'); 17 | this.render({'message': data['message']}); 18 | }.bind(this)) 19 | .fail(this.handleViewError.bind(this)) 20 | .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); 21 | } 22 | }); 23 | 24 | module.exports = NodeView; 25 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/nodes.hbs: -------------------------------------------------------------------------------- 1 | {{> warning}} 2 | {{> error}} 3 | 4 |
5 |
6 |

NSQd Nodes ({{collection.length}})

7 |
8 |
9 | 10 |
11 |
12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | {{#if nsqlookupd.length}} 22 | 23 | {{/if}} 24 | 25 | 26 | {{#each collection}} 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | {{#if ../nsqlookupd.length}} 36 | 42 | {{/if}} 43 | 51 | 52 | {{/each}} 53 |
HostnameBroadcast AddressTCP PortHTTP PortVersionRegionZoneLookupd Conns.Topics
{{hostname}}{{broadcast_address}}{{tcp_port}}{{http_port}}{{version}}{{topology_region}}{{topology_zone}} 37 | {{remote_addresses.length}} 38 |
39 | {{#each remote_addresses}}{{this}}
{{/each}} 40 |
41 | 		44 | {{#if topics.length}} 45 | {{topics.length}} 46 | {{#each topics}} 47 | {{topic}} 48 | {{/each}} 49 | {{/if}} 50 |
54 |
55 |
56 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/nodes.js: -------------------------------------------------------------------------------- 1 | var $ = require('jquery'); 2 | 3 | var Pubsub = require('../lib/pubsub'); 4 | var AppState = require('../app_state'); 5 | 6 | var BaseView = require('./base'); 7 | 8 | var Nodes = require('../collections/nodes'); 9 | 10 | var NodesView = BaseView.extend({ 11 | className: 'nodes container-fluid', 12 | 13 | template: require('./spinner.hbs'), 14 | 15 | events: { 16 | 'click .conn-count': 'onClickConnCount' 17 | }, 18 | 19 | initialize: function() { 20 | BaseView.prototype.initialize.apply(this, arguments); 21 | this.listenTo(AppState, 'change:graph_interval', this.render); 22 | this.collection = new Nodes(); 23 | this.collection.fetch() 24 | .done(function(data) { 25 | this.template = require('./nodes.hbs'); 26 | this.render({'message': data['message']}); 27 | }.bind(this)) 28 | .fail(this.handleViewError.bind(this)) 29 | .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); 30 | }, 31 | 32 | onClickConnCount: function(e) { 33 | e.preventDefault(); 34 | $(e.target).next().toggle(); 35 | } 36 | }); 37 | 38 | module.exports = NodesView; 39 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/spinner.hbs: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 |
6 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/topic.js: -------------------------------------------------------------------------------- 1 | var $ = require('jquery'); 2 | 3 | window.jQuery = $; 4 | var bootstrap = require('bootstrap'); //eslint-disable-line no-unused-vars 5 | var bootbox = require('bootbox'); 6 | 7 | var Pubsub = require('../lib/pubsub'); 8 | var AppState = require('../app_state'); 9 | 10 | var BaseView = require('./base'); 11 | 12 | var TopicView = BaseView.extend({ 13 | className: 'topic container-fluid', 14 | 15 | template: require('./spinner.hbs'), 16 | 17 | events: { 18 | 'click .topic-actions button': 'topicAction' 19 | }, 20 | 21 | initialize: function() { 22 | BaseView.prototype.initialize.apply(this, arguments); 23 | this.listenTo(AppState, 'change:graph_interval', this.render); 24 | var isAdmin = this.model.get('isAdmin'); 25 | this.model.fetch() 26 | .done(function(data) { 27 | this.template = require('./topic.hbs'); 28 | this.render({'message': data['message'], 'isAdmin': isAdmin}); 29 | }.bind(this)) 30 | .fail(this.handleViewError.bind(this)) 31 | .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); 32 | }, 33 | 34 | topicAction: function(e) { 35 | e.preventDefault(); 36 | e.stopPropagation(); 37 | var action = $(e.currentTarget).data('action'); 38 | var txt = 'Are you sure you want to ' + 39 | action + ' ' + this.model.get('name') + '?'; 40 | bootbox.confirm(txt, function(result) { 41 | if (result !== true) { 42 | return; 43 | } 44 | if (action === 'delete') { 45 | $.ajax(this.model.url(), {'method': 'DELETE'}) 46 | .done(function() { window.location = AppState.basePath('/'); }); 47 | } else { 48 | $.post(this.model.url(), JSON.stringify({'action': action})) 49 | .done(function() { window.location.reload(true); }) 50 | .fail(this.handleAJAXError.bind(this)); 51 | } 52 | }.bind(this)); 53 | } 54 | }); 55 | 56 | module.exports = TopicView; 57 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/topics.hbs: -------------------------------------------------------------------------------- 1 | {{> warning}} 2 | {{> error}} 3 | 4 |
5 |
6 |

Topics

7 |
8 |
9 | 10 |
11 |
12 | {{#if collection.length}} 13 | 14 | 15 | 16 | {{#if graph_active}}{{/if}} 17 | {{#if graph_active}}{{/if}} 18 | {{#if graph_active}}{{/if}} 19 | 20 | {{#each collection}} 21 | 22 | 23 | {{#if ../graph_active}}{{/if}} 24 | {{#if ../graph_active}}{{/if}} 25 | {{#if ../graph_active}}{{/if}} 26 | 27 | {{/each}} 28 |
TopicDepthMessagesRate
{{name}}
29 | {{else}} 30 |

Notice

No Topics Found
31 | {{/if}} 32 |
33 |
34 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/topics.js: -------------------------------------------------------------------------------- 1 | var Pubsub = require('../lib/pubsub'); 2 | var AppState = require('../app_state'); 3 | 4 | var BaseView = require('./base'); 5 | var Topics = require('../collections/topics'); 6 | 7 | var TopicsView = BaseView.extend({ 8 | className: 'topics container-fluid', 9 | 10 | template: require('./spinner.hbs'), 11 | 12 | initialize: function() { 13 | BaseView.prototype.initialize.apply(this, arguments); 14 | this.listenTo(AppState, 'change:graph_interval', this.render); 15 | this.collection = new Topics(); 16 | this.collection.fetch() 17 | .done(function(data) { 18 | this.template = require('./topics.hbs'); 19 | this.render({'message': data['message']}); 20 | }.bind(this)) 21 | .fail(this.handleViewError.bind(this)) 22 | .always(Pubsub.trigger.bind(Pubsub, 'view:ready')); 23 | } 24 | }); 25 | 26 | module.exports = TopicsView; 27 | -------------------------------------------------------------------------------- /nsqadmin/static/js/views/warning.hbs: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | {{message}} 5 |
6 |
7 |
8 | -------------------------------------------------------------------------------- /nsqadmin/test/ca.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | Proc-Type: 4,ENCRYPTED 3 | DEK-Info: DES-EDE3-CBC,D6208487450BDF1C 4 | 5 | tucU/4j6agGQlW60D8V3Zr/QHcLhhyFagres1gGdWfGIqluNZb7omki/XidHXJSG 6 | eB9vV2Xb12/8umc31e7Mnmn9hd34230v/KlAnJ4ukDpJpbmjnEx3F9uiqYFi/yxQ 7 | avSsfF6Tsh3XOh3Oe27I/xfYx37g6Agd+EQEJ1hvWvygMIJMTDMP5ZaFoZANtFLy 8 | hDEZ6woJSn9avF/L+1GW8jl2aI1QbdKkK0jDHgFAwUI4sjWeXvEQNNYY3trTIoMo 9 | wab3vi+4XziFONbS4OZrZUYfZPB5YOFbtT2whzggp2HdSTiu48/Ld3N8SjuMrKfm 10 | uR+nd+ovQ5kVWHInzWAIXSyPhgR9ZY8eyXaHNJJfzNu3HY72lfzD/NtZfacMRBr6 11 | M3Wg/OKPS7ZrtqCWkY9P3KK9Cul8Jzy229fSqHo8Rg4= 12 | -----END RSA PRIVATE KEY----- 13 | -------------------------------------------------------------------------------- /nsqadmin/test/ca.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBVDCB/6ADAgECAgkAvHG4Z/7nX/gwDQYJKoZIhvcNAQEFBQAwDTELMAkGA1UE 3 | AwwCY2EwHhcNMTcwOTE2MTc0NDE0WhcNMjcwOTE0MTc0NDE0WjANMQswCQYDVQQD 4 | DAJjYTBcMA0GCSqGSIb3DQEBAQUAA0sAMEgCQQDJAM3Tr1BoxlLJtTy0oRcp93dT 5 | 9hhHwms8P1V3k2FpXYRS4deUo+uwcAM9KGDt9VMXVBEchtI4VYTvLgatBPUBAgMB 6 | AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMyLx7rjKBe/xZQLnVzI 7 | uqNNVzxRMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAANBAJiN1XgPPlQ2 8 | z7PhtzXStaz/BJVqhD7g9fsZsmoPX4ifDsTfzUsRB56Aq/NTsKiIYQkFPHH0donG 9 | ++a5ZVWjgYk= 10 | -----END CERTIFICATE----- 11 | -------------------------------------------------------------------------------- /nsqadmin/test/ca.srl: -------------------------------------------------------------------------------- 1 | 91418D04995922E7 2 | -------------------------------------------------------------------------------- /nsqadmin/test/cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEbjCCA1agAwIBAgIJAK6x7y6AwBmLMA0GCSqGSIb3DQEBBQUAMIGAMQswCQYD 3 | VQQGEwJVUzERMA8GA1UECBMITmV3IFlvcmsxFjAUBgNVBAcTDU5ldyBZb3JrIENp 4 | dHkxDDAKBgNVBAoTA05TUTETMBEGA1UEAxMKdGVzdC5sb2NhbDEjMCEGCSqGSIb3 5 | DQEJARYUbXJlaWZlcnNvbkBnbWFpbC5jb20wHhcNMTMwNjI4MDA0MzQ4WhcNMTYw 6 | NDE3MDA0MzQ4WjCBgDELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMRYw 7 | FAYDVQQHEw1OZXcgWW9yayBDaXR5MQwwCgYDVQQKEwNOU1ExEzARBgNVBAMTCnRl 8 | c3QubG9jYWwxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMIIB 9 | IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnX0KB+svwy+yHU2qggz/EaGg 10 | craKShagKo+9M9y5HLM852ngk5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS 11 | 25tVP1UIEnT5pBt2TeetLkl199Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQY 12 | shuEJHKeUNDbQKz5X+GjEdkTPO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECz 13 | ydZBgTPThy3uDtHIuCpxCwXd/vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF 14 | 7m3/0KbtUcXfy0QHueeuMr11E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QID 15 | AQABo4HoMIHlMB0GA1UdDgQWBBR3HMBws4lmYYSIgwoZsfW+bbgaMjCBtQYDVR0j 16 | BIGtMIGqgBR3HMBws4lmYYSIgwoZsfW+bbgaMqGBhqSBgzCBgDELMAkGA1UEBhMC 17 | VVMxETAPBgNVBAgTCE5ldyBZb3JrMRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MQww 18 | CgYDVQQKEwNOU1ExEzARBgNVBAMTCnRlc3QubG9jYWwxIzAhBgkqhkiG9w0BCQEW 19 | FG1yZWlmZXJzb25AZ21haWwuY29tggkArrHvLoDAGYswDAYDVR0TBAUwAwEB/zAN 20 | BgkqhkiG9w0BAQUFAAOCAQEANOYTbanW2iyV1v4oYpcM/y3TWcQKzSME8D2SGFZb 21 | dbMYU81hH3TTlQdvyeh3FAcdjhKE8Xi/RfNNjEslTBscdKXePGpZg6eXRNJzPP5K 22 | KZPf5u6tcpAeUOKrMqbGwbE+h2QixxG1EoVQtE421szsU2P7nHRTdHzKFRnOerfl 23 | Phm3NocR0P40Rv7WKdxpOvqc+XKf0onTruoVYoPWGpwcLixCG0zu4ZQ23/L/Dy18 24 | 4u70Hbq6O/6kq9FBFaDNp3IhiEdu2Cq6ZplU6bL9XDF27KIEErHwtuqBHVlMG+zB 25 | oH/k9vZvwH7OwAjHdKp+1yeZFLYC8K5hjFIHqcdwpZCNIg== 26 | -----END CERTIFICATE----- 27 | -------------------------------------------------------------------------------- /nsqadmin/test/client.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAuMz5IJ01Wqaj2qJAU7o2aoc2iP1BRhTba+cWrsjbZA+6kXAk 3 | 19Dlz/nNK8gxsBMl3rzTGWXKUm/rKBDp680zJkj8nzcGApGlzbDoqXym8ixOniOC 4 | aH/RODPC8OtD2UozeXlUOQWK+ufkMvmVBc/IeJyFkcm7jhX+LHVFn+3iWLRNbzGm 5 | hbf8oNHvybS7HSuV3OYiHDJxA0d6IgozqVOYVSqdDabQ2FN/8nKzpMHjwMpfT008 6 | 5hUhr72E5XuJ3biPUwoOrRETSa1/EQcAEH0wSwHU1AGdnGh3/5ynVupfef0qkyqw 7 | NseYs9U++IUFXqA1IuEmAlJn5QczAHjPQe8xBQIDAQABAoIBAGx4CyZEgCOUOgrD 8 | P2SloPkIIk9n7x82cNA11I+E35kszkI9g7KVL77SDcZL/DYwFwNU68c1gvq+LFXZ 9 | D6RTTlmDb5v4TPPHD33a/8UzoD33GbIif5HcrC4D28FTJgDtV6dOOsw5X6kD4WK2 10 | Me02V6HLpW677PVqHUV1FAfaNgf/1SIhJCechYSO+3pKPtli8DnsmOd336wNtWZR 11 | Q3Ctm63Eq7AiLmGLOkLYtll9Kr6WfCSFHBmqJg9MsQEfINXMCOB54Z+xkoijjJLO 12 | zNSk/3XvppB1JpmAusbo9Mq03Ci9JdinYSKxwVOG6e+5cosrOHpDQyZIWa87ZTpk 13 | kjLCgEECgYEA33JCGTEwaqfAuYOspCZvi2Mqz7t6QiwvwxgDf4JFLgDQArJfzKlC 14 | bAzaSU8bT+SN96CF9LU4XekN5/paGZAFjWxraPkK0oL28kQrtzfEQv2XL6GFoNAh 15 | WsQtoaGddL9VPWlCiHeueYERIDNrZCaVeAx6OV6W3CEwGOkRiQyntlUCgYEA07lf 16 | LpxF8aAgMToq3++D5RGAXaT2PYxhZZFb5Y2tS7a4T0ulQwXQi4H2dObGJ32zCjFG 17 | ls1H91mNKGBbyKnF1jT/dt/acqymvipASm1xq6uysIllA4xp56sDnJgG9bnXjYFY 18 | m2yVNIoIQGZ7jp7e6t1KQ9San6V8asWlvBLXX/ECgYEAuhF9dVj+xnH3DQTXSMIw 19 | 9NOZnO6zelMtWrqufwnN7ecDUJuVJupzw2JYi99yEO90QRbNNd+KlrkxuVFCojLK 20 | TOBR+VIZbv9cAJZACQxJRLfDpAhPLIDkpZ7jmMrqQYPqyX7TxqxTAB84UaY/8WAn 21 | 65YIWamo2ppQYQ4Eaim9pxkCgYAMwfW/TEFWruxhqvycY8VRzz0p51/DE6tmwFyG 22 | N4RCtK7kcE1z/Wy0i087ehBknslkCtYTDimQ+P9teGjvbXNzVdwy4Ig8MrUVblxT 23 | X8birkTlKFJC5XoYMJDWJb79nYYki6+4JdHTyaF3p/U4AdCy3ES2U6BBkGov0NsM 24 | uyHpMQKBgGc9wIGHwAGLg3iVHGXbXELdpzVztLRC7D12TPfdtGBLTD5NH5jAx8K/ 25 | w61+l4tzFcK3jfWzwdme277cBgfq/aw2DDt07vkOvPzhUwVZiE39bfddfIvyed99 26 | XzEMf8THh5wdm6mLsOVgdfOfFcDi5ReIv1/7+eOd/MQi8TxIVfAu 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /nsqadmin/test/client.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICyzCCAnWgAwIBAgIJAJFBjQSZWSLnMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV 3 | BAMMAmNhMB4XDTE3MDkxNjE3NDQxNFoXDTI3MDkxNDE3NDQxNFowgYcxIzAhBgkq 4 | hkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMQswCQYDVQQGEwJERTEMMAoG 5 | A1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEXMBUGA1UEChMOUmFuZG9tIENvbXBh 6 | bnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZuc3EuaW8wggEiMA0GCSqGSIb3DQEB 7 | AQUAA4IBDwAwggEKAoIBAQC4zPkgnTVapqPaokBTujZqhzaI/UFGFNtr5xauyNtk 8 | D7qRcCTX0OXP+c0ryDGwEyXevNMZZcpSb+soEOnrzTMmSPyfNwYCkaXNsOipfKby 9 | LE6eI4Jof9E4M8Lw60PZSjN5eVQ5BYr65+Qy+ZUFz8h4nIWRybuOFf4sdUWf7eJY 10 | tE1vMaaFt/yg0e/JtLsdK5Xc5iIcMnEDR3oiCjOpU5hVKp0NptDYU3/ycrOkwePA 11 | yl9PTTzmFSGvvYTle4nduI9TCg6tERNJrX8RBwAQfTBLAdTUAZ2caHf/nKdW6l95 12 | /SqTKrA2x5iz1T74hQVeoDUi4SYCUmflBzMAeM9B7zEFAgMBAAGjdTBzMAwGA1Ud 13 | EwEB/wQCMAAwHQYDVR0OBBYEFMzYXiTD7moi0oXqSQ5D2LsO51GXMB8GA1UdIwQY 14 | MBaAFMyLx7rjKBe/xZQLnVzIuqNNVzxRMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUE 15 | DDAKBggrBgEFBQcDAjANBgkqhkiG9w0BAQsFAANBAAujxas6toRhMl/+kZrly0G/ 16 | AvjbA3WY5cLLIdffGdQ5bsS3aOP23nj98ut7unNUNsCo+eUwpJgvabnFnL+NFZA= 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /nsqadmin/test/client.req: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu 3 | Y29tMQswCQYDVQQGEwJERTEMMAoGA1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEX 4 | MBUGA1UEChMOUmFuZG9tIENvbXBhbnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZu 5 | c3EuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4zPkgnTVapqPa 6 | okBTujZqhzaI/UFGFNtr5xauyNtkD7qRcCTX0OXP+c0ryDGwEyXevNMZZcpSb+so 7 | EOnrzTMmSPyfNwYCkaXNsOipfKbyLE6eI4Jof9E4M8Lw60PZSjN5eVQ5BYr65+Qy 8 | +ZUFz8h4nIWRybuOFf4sdUWf7eJYtE1vMaaFt/yg0e/JtLsdK5Xc5iIcMnEDR3oi 9 | CjOpU5hVKp0NptDYU3/ycrOkwePAyl9PTTzmFSGvvYTle4nduI9TCg6tERNJrX8R 10 | BwAQfTBLAdTUAZ2caHf/nKdW6l95/SqTKrA2x5iz1T74hQVeoDUi4SYCUmflBzMA 11 | eM9B7zEFAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEALD1VjbBjaeX7l7JR4IuL 12 | wrHVrFDiRYsgvqyw39j2MC95VwrGwzf4cXCE+RuqE/DhbV9UI7sWKaJfFs9Usq+g 13 | VSoKnHSEylt34y6ABSc5eAik+GnheJZbJ6UDjxcvNd0UpFGMrHbsXVyQd0Y1XAu7 14 | nxCYIa82kNA+Opb+ra03hkLC7wvRLbXTOB2g6JLkyhYR6S5GkOFTNnz2AJerN7zt 15 | NEL7owlRcjyIsL5tjDpDH1944NNtzhgmrUeIjB08reayuot9RKznMVGwBfY6DIHM 16 | Q4uNN3CMOOoAHr1UzvBf/qfvb6ltPTMKSV1OncLlC3C59NoO8vhKIHCN18Ya2OMu 17 | rw== 18 | -----END CERTIFICATE REQUEST----- 19 | -------------------------------------------------------------------------------- /nsqadmin/test/key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEogIBAAKCAQEAnX0KB+svwy+yHU2qggz/EaGgcraKShagKo+9M9y5HLM852ng 3 | k5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS25tVP1UIEnT5pBt2TeetLkl1 4 | 99Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQYshuEJHKeUNDbQKz5X+GjEdkT 5 | PO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECzydZBgTPThy3uDtHIuCpxCwXd 6 | /vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF7m3/0KbtUcXfy0QHueeuMr11 7 | E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QIDAQABAoIBACvtfKbIywG+hAf4 8 | ad7skRjx5DcbA2e29+XnQfb9UgTXWd2SgrmoLi5OypBkCTzkKN3mfTo70yZfV8dC 9 | Sxwz+9tfnTz0DssjhKThS+CiaFVCkeOfSfBfKSlCQUVHrSrh18CDhP+yvDlJwQTZ 10 | zSQMfPcsh9bmJe2kqtQP7ZgUp1o+vaB8Sju8YYrO6FllxbdLRGm4pfvvrHIRRmXa 11 | oVHn0ei0JpwoTY9kHYht4LNeJnbP/MCWdmcuv3Gnel7jAlhaKab5aNIGr0Xe7aIQ 12 | iX6mpZ0/Rnt8o/XcTOg8l3ruIdVuySX6SYn08JMnfFkXdNYRVhoV1tC5ElWkaZLf 13 | hPmj2yECgYEAyts0R0b8cZ6HTAyuLm3ilw0s0v0/MM9ZtaqMRilr2WEtAhF0GpHG 14 | TzmGnii0WcTNXD7NTsNcECR/0ZpXPRleMczsL2Juwd4FkQ37h7hdKPseJNrfyHRg 15 | VolOFBX9H14C3wMB9cwdsG4Egw7fE27WCoreEquHgwFxl1zBrXKH088CgYEAxr8w 16 | BKZs0bF7LRrFT5pH8hpMLYHMYk8ZIOfgmEGVBKDQCOERPR9a9kqUss7wl/98LVNK 17 | RnFlyWD6Z0/QcQsLL4LjBeZJ25qEMc6JXm9VGAzhXA1ZkUofVoYCnG+f6KUn8CuJ 18 | /AcV2ZDFsEP10IiQG0hKsceXiwFEvEr8306tMrcCgYBLgnscSR0xAeyk71dq6vZc 19 | ecgEpcX+2kAvclOSzlpZ6WVCjtKkDT0/Qk+M0eQIQkybGLl9pxS+4Yc+s2/jy2yX 20 | pwsHvGE0AvwZeZX2eDcdSRR4bYy9ZixyKdwJeAHnyivRbaIuJ5Opl9pQGpoI9snv 21 | 1K9DTdw8dK4exKVHdgl/WwKBgDkmLsuXg4EEtPOyV/xc08VVNIR9Z2T5c7NXmeiO 22 | KyiKiWeUOF3ID2L07S9BfENozq9F3PzGjMtMXJSqibiHwW6nB1rh7mj8VHjx9+Q0 23 | xVZGFeNfX1r84mgB3uxW2LeQDhzsmB/lda37CC14TU3qhu2hawEV8IijE73FHlOk 24 | Dv+fAoGAI4/XO5o5tNn5Djo8gHmGMCbinUE9+VySxl7wd7PK8w2VSofO88ofixDk 25 | NX94yBYhg5WZcLdPm45RyUnq+WVQYz9IKUrdxLFTH+wxyzUqZCW7jgXCvWV+071q 26 | vqm9C+kndq+18/1VKuCSGWnF7Ay4lbsgPXY2s4VKRxcb3QpZSPU= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /nsqadmin/test/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAuYs7NerYpXWq/IPaBGriHobhIs/i4AJHra7QhZ5N/EfBCb0N 3 | 6K3tI4GRHMZ4lB0HwINWmxaPs6D2yDCxpGgnru6qCu0/91+Lq+GEZj+Fz1E2yfCq 4 | tecTySvWf9en+SdTKFZTGGIAu3zFcozuX2fdhkjJ6ibZhnrXKlPU4IN0STkkRfFX 5 | +yb/6P88mGh7PZKBb67Qw1bppzXE/SWgHVqDqu++whT1quwxnRhh/SHgwugN+6iK 6 | uTDcJ1jQ1QWe+xna7Te6nNBUvL6dV49jpoDOjHsv+UaAvzNeHCHqypTC0wwYHXyg 7 | TTAU+E20c2NOCC7P0PAid1liNeuuEWtM2bRrFQIDAQABAoIBABPzc7d1fDw2bd9f 8 | Mic9cvkDWdwLbILX2+tCG+vyPMJ+2LP6Xy+A3DnwKbFlafvLL1U1Ci/8+hC/oymd 9 | isx54qJ9yU0Je9JWtMcTpc/0zqefPPvz4/dRVKBSFWuDve0dnGR++8poZ1nBrd2G 10 | Z+9cVMamtwd1i/hY5yAHCaHmoK9qxXuSjtbPkqDqruloGO5iT+tcICjAeWyJIx1P 11 | 5VJjuKDx4AhU4B8DpBSTtA02BKltvr2K6D4zhJ255oI4iLUkYk4IwxB7yZH+6Y++ 12 | mDa6e8iM3ut+qMkCxto/CfMCqYfoRzmykiazZ5qFXfGwv3OkWUEOBXU//Vpklvnh 13 | OQWYL/UCgYEA7mkKXbCrCNJEdly4LVLPKPFjVUPhMdXcqdnoKZ6BiVYFMqTuG4+w 14 | o/Rf6ZpqZ3Rf656/ypd1atVvK2g4zfq3j1W6lDnpLjKaLhc8a+ZH4NWXBORIzlUW 15 | aR2xVcPpAokor37FRAWbduiwVSCHlDtg48i+rvukyqmeRBKjrIgX3lsCgYEAxzut 16 | 5GiMZ9TkjqYXdlUDIDKWMHMfLfRY7+Q/cBdr7AfiMVw4PIWJG/IuxCGwGp77gRg6 17 | BeBsi8Htj5/EGbRiK4kjFyf8LqYGOeYLZ3I8+olr+tNmF4/yKwxku3VylpmrpCAV 18 | /8emso1rAWo6Y98MTNSGgvLqhDU6OD7tTSv3908CgYA4+aVeio/1RbrSxonFWxri 19 | 3/0rLVOuAzv+43KWL6kpVwNa/QtiTs6aABbDzwFKxAcAWinfkp6e727n4rpgj2A6 20 | wvQZ5FUTk0hBZ5ArAReAZcr3gk7b8H2wlUYCBxWyY3Dzr8oY3XYvzqAFWAbOp/oZ 21 | tanMS5swS6TlA8dVvhhmLQKBgGZwcBPOEctdcntKOSwVv/qxJ/oXZ0O4rHYENP4M 22 | fOgqkYnxsdSkkH/3AUbFT4gQkJ6q90KIRyeA+gXsDudskUFzTMCeRZMyuGbSurBg 23 | 06u6NvQL+CVLVSf/QlgEpnt63f8QpF8Up8iM4CUlGoq5Z9ilOdhg0GZT+/BpopgY 24 | cHIPAoGBAKyLumgeG+gMFK5x/Fi6zjBT5MK8Tw0VMkahW01Jx2laYibS/a8AEFmn 25 | ySdrmLPkOmmWgXCk3m2m5PkvM5qH/KNugOA0+WX2CTvwt4ZgwWUalQ5R43wSIeDC 26 | MXVfwC8uE66PmpYgmsu4H0vnCGfacOCQhfdq01SLobgBiQSrm/6D 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /nsqadmin/test/server.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIC3jCCAoigAwIBAgIJAJFBjQSZWSLmMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV 3 | BAMMAmNhMB4XDTE3MDkxNjE3NDQxNFoXDTI3MDkxNDE3NDQxNFowgYcxIzAhBgkq 4 | hkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMQswCQYDVQQGEwJERTEMMAoG 5 | A1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEXMBUGA1UEChMOUmFuZG9tIENvbXBh 6 | bnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZuc3EuaW8wggEiMA0GCSqGSIb3DQEB 7 | AQUAA4IBDwAwggEKAoIBAQC5izs16tildar8g9oEauIehuEiz+LgAketrtCFnk38 8 | R8EJvQ3ore0jgZEcxniUHQfAg1abFo+zoPbIMLGkaCeu7qoK7T/3X4ur4YRmP4XP 9 | UTbJ8Kq15xPJK9Z/16f5J1MoVlMYYgC7fMVyjO5fZ92GSMnqJtmGetcqU9Tgg3RJ 10 | OSRF8Vf7Jv/o/zyYaHs9koFvrtDDVumnNcT9JaAdWoOq777CFPWq7DGdGGH9IeDC 11 | 6A37qIq5MNwnWNDVBZ77GdrtN7qc0FS8vp1Xj2OmgM6Mey/5RoC/M14cIerKlMLT 12 | DBgdfKBNMBT4TbRzY04ILs/Q8CJ3WWI1664Ra0zZtGsVAgMBAAGjgYcwgYQwDAYD 13 | VR0TAQH/BAIwADAdBgNVHQ4EFgQUt6UXJ8BQB++I/cQG5ZgaiUyqOE8wHwYDVR0j 14 | BBgwFoAUzIvHuuMoF7/FlAudXMi6o01XPFEwDgYDVR0PAQH/BAQDAgWgMBMGA1Ud 15 | JQQMMAoGCCsGAQUFBwMBMA8GA1UdEQQIMAaHBH8AAAEwDQYJKoZIhvcNAQELBQAD 16 | QQCCwm0F2eTmtIcKDxrXGZ7q9y9mfsROfYaCnH+vKUDw2vmqQkInzhLeEElDGQcR 17 | ww0IKCnDHEruNb2tKyQM/70L 18 | -----END CERTIFICATE----- 19 | -------------------------------------------------------------------------------- /nsqadmin/test/server.req: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu 3 | Y29tMQswCQYDVQQGEwJERTEMMAoGA1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEX 4 | MBUGA1UEChMOUmFuZG9tIENvbXBhbnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZu 5 | c3EuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5izs16tildar8 6 | g9oEauIehuEiz+LgAketrtCFnk38R8EJvQ3ore0jgZEcxniUHQfAg1abFo+zoPbI 7 | MLGkaCeu7qoK7T/3X4ur4YRmP4XPUTbJ8Kq15xPJK9Z/16f5J1MoVlMYYgC7fMVy 8 | jO5fZ92GSMnqJtmGetcqU9Tgg3RJOSRF8Vf7Jv/o/zyYaHs9koFvrtDDVumnNcT9 9 | JaAdWoOq777CFPWq7DGdGGH9IeDC6A37qIq5MNwnWNDVBZ77GdrtN7qc0FS8vp1X 10 | j2OmgM6Mey/5RoC/M14cIerKlMLTDBgdfKBNMBT4TbRzY04ILs/Q8CJ3WWI1664R 11 | a0zZtGsVAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEASNjcoZbyNcwMQjMmNoil 12 | S/7pCRn4aYzZIVjrVtOHQ9GHC23MSep5um2gIcMFPiuYyu9Byl8CSVtc1op2fAKS 13 | vrugoZaCrp/A76hqOfNxgh7VmgTux8bG5Qcjaija1BNWpbyaWARdBxN/WgS5CpCj 14 | u2yzv8mrzzFNrDMlsmiEMvtkMzdhiZ4YY8zm6CdrbIR5z1eqf4e+rs4oJtTKNNAD 15 | hewk8CGiUW1hOx2jpjcIVMRy+ofVHRX2xQ6Sw8qxCNsiv8IPAAivgAbFJO76ZSbH 16 | eQ7uKWszmBEroyFvZ0rfmFLXuopU125pyBDl5FUKYAZzCBx9tr5dROCbw/rXDhke 17 | ig== 18 | -----END CERTIFICATE REQUEST----- 19 | -------------------------------------------------------------------------------- /nsqd/README.md: -------------------------------------------------------------------------------- 1 | ## nsqd 2 | 3 | `nsqd` is the daemon that receives, queues, and delivers messages to clients. 4 | 5 | Read the [docs](https://nsq.io/components/nsqd.html) 6 | -------------------------------------------------------------------------------- /nsqd/backend_queue.go: -------------------------------------------------------------------------------- 1 | package nsqd 2 | 3 | // BackendQueue represents the behavior for the secondary message 4 | // storage system 5 | type BackendQueue interface { 6 | Put([]byte) error 7 | ReadChan() <-chan []byte // this is expected to be an *unbuffered* channel 8 | Close() error 9 | Delete() error 10 | Depth() int64 11 | Empty() error 12 | } 13 | -------------------------------------------------------------------------------- /nsqd/buffer_pool.go: -------------------------------------------------------------------------------- 1 | package nsqd 2 | 3 | import ( 4 | "bytes" 5 | "sync" 6 | ) 7 | 8 | var bp sync.Pool 9 | 10 | func init() { 11 | bp.New = func() interface{} { 12 | return &bytes.Buffer{} 13 | } 14 | } 15 | 16 | func bufferPoolGet() *bytes.Buffer { 17 | return bp.Get().(*bytes.Buffer) 18 | } 19 | 20 | func bufferPoolPut(b *bytes.Buffer) { 21 | b.Reset() 22 | bp.Put(b) 23 | } 24 | -------------------------------------------------------------------------------- /nsqd/dqname.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | // +build !windows 3 | 4 | package nsqd 5 | 6 | func getBackendName(topicName, channelName string) string { 7 | // backend names, for uniqueness, automatically include the topic... : 8 | backendName := topicName + ":" + channelName 9 | return backendName 10 | } 11 | -------------------------------------------------------------------------------- /nsqd/dqname_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | package nsqd 5 | 6 | // On Windows, file names cannot contain colons. 7 | func getBackendName(topicName, channelName string) string { 8 | // backend names, for uniqueness, automatically include the topic... ; 9 | backendName := topicName + ";" + channelName 10 | return backendName 11 | } 12 | -------------------------------------------------------------------------------- /nsqd/dummy_backend_queue.go: -------------------------------------------------------------------------------- 1 | package nsqd 2 | 3 | type dummyBackendQueue struct { 4 | readChan chan []byte 5 | } 6 | 7 | func newDummyBackendQueue() BackendQueue { 8 | return &dummyBackendQueue{readChan: make(chan []byte)} 9 | } 10 | 11 | func (d *dummyBackendQueue) Put([]byte) error { 12 | return nil 13 | } 14 | 15 | func (d *dummyBackendQueue) ReadChan() <-chan []byte { 16 | return d.readChan 17 | } 18 | 19 | func (d *dummyBackendQueue) Close() error { 20 | return nil 21 | } 22 | 23 | func (d *dummyBackendQueue) Delete() error { 24 | return nil 25 | } 26 | 27 | func (d *dummyBackendQueue) Depth() int64 { 28 | return int64(0) 29 | } 30 | 31 | func (d *dummyBackendQueue) Empty() error { 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /nsqd/guid.go: -------------------------------------------------------------------------------- 1 | package nsqd 2 | 3 | // the core algorithm here was borrowed from: 4 | // Blake Mizerany's `noeqd` https://github.com/bmizerany/noeqd 5 | // and indirectly: 6 | // Twitter's `snowflake` https://github.com/twitter/snowflake 7 | 8 | // only minor cleanup and changes to introduce a type, combine the concept 9 | // of workerID + datacenterId into a single identifier, and modify the 10 | // behavior when sequences rollover for our specific implementation needs 11 | 12 | import ( 13 | "encoding/hex" 14 | "errors" 15 | "sync" 16 | "time" 17 | ) 18 | 19 | const ( 20 | nodeIDBits = uint64(10) 21 | sequenceBits = uint64(12) 22 | nodeIDShift = sequenceBits 23 | timestampShift = sequenceBits + nodeIDBits 24 | sequenceMask = int64(-1) ^ (int64(-1) << sequenceBits) 25 | 26 | // ( 2012-10-28 16:23:42 UTC ).UnixNano() >> 20 27 | twepoch = int64(1288834974288) 28 | ) 29 | 30 | var ErrTimeBackwards = errors.New("time has gone backwards") 31 | var ErrSequenceExpired = errors.New("sequence expired") 32 | var ErrIDBackwards = errors.New("ID went backward") 33 | 34 | type guid int64 35 | 36 | type guidFactory struct { 37 | sync.Mutex 38 | 39 | nodeID int64 40 | sequence int64 41 | lastTimestamp int64 42 | lastID guid 43 | } 44 | 45 | func NewGUIDFactory(nodeID int64) *guidFactory { 46 | return &guidFactory{ 47 | nodeID: nodeID, 48 | } 49 | } 50 | 51 | func (f *guidFactory) NewGUID() (guid, error) { 52 | f.Lock() 53 | 54 | // divide by 1048576, giving pseudo-milliseconds 55 | ts := time.Now().UnixNano() >> 20 56 | 57 | if ts < f.lastTimestamp { 58 | f.Unlock() 59 | return 0, ErrTimeBackwards 60 | } 61 | 62 | if f.lastTimestamp == ts { 63 | f.sequence = (f.sequence + 1) & sequenceMask 64 | if f.sequence == 0 { 65 | f.Unlock() 66 | return 0, ErrSequenceExpired 67 | } 68 | } else { 69 | f.sequence = 0 70 | } 71 | 72 | f.lastTimestamp = ts 73 | 74 | id := guid(((ts - twepoch) << timestampShift) | 75 | (f.nodeID << nodeIDShift) | 76 | f.sequence) 77 | 78 | if id <= f.lastID { 79 | f.Unlock() 80 | return 0, ErrIDBackwards 81 | } 82 | 83 | f.lastID = id 84 | 85 | f.Unlock() 86 | 87 | return id, nil 88 | } 89 | 90 | func (g guid) Hex() MessageID { 91 | var h MessageID 92 | var b [8]byte 93 | 94 | b[0] = byte(g >> 56) 95 | b[1] = byte(g >> 48) 96 | b[2] = byte(g >> 40) 97 | b[3] = byte(g >> 32) 98 | b[4] = byte(g >> 24) 99 | b[5] = byte(g >> 16) 100 | b[6] = byte(g >> 8) 101 | b[7] = byte(g) 102 | 103 | hex.Encode(h[:], b[:]) 104 | return h 105 | } 106 | -------------------------------------------------------------------------------- /nsqd/guid_test.go: -------------------------------------------------------------------------------- 1 | package nsqd 2 | 3 | import ( 4 | "testing" 5 | "unsafe" 6 | ) 7 | 8 | func BenchmarkGUIDCopy(b *testing.B) { 9 | source := make([]byte, 16) 10 | var dest MessageID 11 | for i := 0; i < b.N; i++ { 12 | copy(dest[:], source) 13 | } 14 | } 15 | 16 | func BenchmarkGUIDUnsafe(b *testing.B) { 17 | source := make([]byte, 16) 18 | var dest MessageID 19 | for i := 0; i < b.N; i++ { 20 | dest = *(*MessageID)(unsafe.Pointer(&source[0])) 21 | } 22 | _ = dest 23 | } 24 | 25 | func BenchmarkGUID(b *testing.B) { 26 | var okays, errors, fails int64 27 | var previd guid 28 | factory := &guidFactory{} 29 | for i := 0; i < b.N; i++ { 30 | id, err := factory.NewGUID() 31 | if err != nil { 32 | errors++ 33 | } else if id == previd { 34 | fails++ 35 | b.Fail() 36 | } else { 37 | okays++ 38 | } 39 | id.Hex() 40 | } 41 | b.Logf("okays=%d errors=%d bads=%d", okays, errors, fails) 42 | } 43 | -------------------------------------------------------------------------------- /nsqd/in_flight_pqueue.go: -------------------------------------------------------------------------------- 1 | package nsqd 2 | 3 | type inFlightPqueue []*Message 4 | 5 | func newInFlightPqueue(capacity int) inFlightPqueue { 6 | return make(inFlightPqueue, 0, capacity) 7 | } 8 | 9 | func (pq inFlightPqueue) Swap(i, j int) { 10 | pq[i], pq[j] = pq[j], pq[i] 11 | pq[i].index = i 12 | pq[j].index = j 13 | } 14 | 15 | func (pq *inFlightPqueue) Push(x *Message) { 16 | n := len(*pq) 17 | c := cap(*pq) 18 | if n+1 > c { 19 | npq := make(inFlightPqueue, n, c*2) 20 | copy(npq, *pq) 21 | *pq = npq 22 | } 23 | *pq = (*pq)[0 : n+1] 24 | x.index = n 25 | (*pq)[n] = x 26 | pq.up(n) 27 | } 28 | 29 | func (pq *inFlightPqueue) Pop() *Message { 30 | n := len(*pq) 31 | c := cap(*pq) 32 | pq.Swap(0, n-1) 33 | pq.down(0, n-1) 34 | if n < (c/2) && c > 25 { 35 | npq := make(inFlightPqueue, n, c/2) 36 | copy(npq, *pq) 37 | *pq = npq 38 | } 39 | x := (*pq)[n-1] 40 | x.index = -1 41 | *pq = (*pq)[0 : n-1] 42 | return x 43 | } 44 | 45 | func (pq *inFlightPqueue) Remove(i int) *Message { 46 | n := len(*pq) 47 | if n-1 != i { 48 | pq.Swap(i, n-1) 49 | pq.down(i, n-1) 50 | pq.up(i) 51 | } 52 | x := (*pq)[n-1] 53 | x.index = -1 54 | *pq = (*pq)[0 : n-1] 55 | return x 56 | } 57 | 58 | func (pq *inFlightPqueue) PeekAndShift(max int64) (*Message, int64) { 59 | if len(*pq) == 0 { 60 | return nil, 0 61 | } 62 | 63 | x := (*pq)[0] 64 | if x.pri > max { 65 | return nil, x.pri - max 66 | } 67 | pq.Pop() 68 | 69 | return x, 0 70 | } 71 | 72 | func (pq *inFlightPqueue) up(j int) { 73 | for { 74 | i := (j - 1) / 2 // parent 75 | if i == j || (*pq)[j].pri >= (*pq)[i].pri { 76 | break 77 | } 78 | pq.Swap(i, j) 79 | j = i 80 | } 81 | } 82 | 83 | func (pq *inFlightPqueue) down(i, n int) { 84 | for { 85 | j1 := 2*i + 1 86 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow 87 | break 88 | } 89 | j := j1 // left child 90 | if j2 := j1 + 1; j2 < n && (*pq)[j1].pri >= (*pq)[j2].pri { 91 | j = j2 // = 2*i + 2 // right child 92 | } 93 | if (*pq)[j].pri >= (*pq)[i].pri { 94 | break 95 | } 96 | pq.Swap(i, j) 97 | i = j 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /nsqd/in_flight_pqueue_test.go: -------------------------------------------------------------------------------- 1 | package nsqd 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "sort" 7 | "testing" 8 | 9 | "github.com/nsqio/nsq/internal/test" 10 | ) 11 | 12 | func TestPriorityQueue(t *testing.T) { 13 | c := 100 14 | pq := newInFlightPqueue(c) 15 | 16 | for i := 0; i < c+1; i++ { 17 | pq.Push(&Message{clientID: int64(i), pri: int64(i)}) 18 | } 19 | test.Equal(t, c+1, len(pq)) 20 | test.Equal(t, c*2, cap(pq)) 21 | 22 | for i := 0; i < c+1; i++ { 23 | msg := pq.Pop() 24 | test.Equal(t, int64(i), msg.clientID) 25 | } 26 | test.Equal(t, c/4, cap(pq)) 27 | } 28 | 29 | func TestUnsortedInsert(t *testing.T) { 30 | c := 100 31 | pq := newInFlightPqueue(c) 32 | ints := make([]int, 0, c) 33 | 34 | for i := 0; i < c; i++ { 35 | v := rand.Int() 36 | ints = append(ints, v) 37 | pq.Push(&Message{pri: int64(v)}) 38 | } 39 | test.Equal(t, c, len(pq)) 40 | test.Equal(t, c, cap(pq)) 41 | 42 | sort.Ints(ints) 43 | 44 | for i := 0; i < c; i++ { 45 | msg, _ := pq.PeekAndShift(int64(ints[len(ints)-1])) 46 | test.Equal(t, int64(ints[i]), msg.pri) 47 | } 48 | } 49 | 50 | func TestRemove(t *testing.T) { 51 | c := 100 52 | pq := newInFlightPqueue(c) 53 | 54 | msgs := make(map[MessageID]*Message) 55 | for i := 0; i < c; i++ { 56 | m := &Message{pri: int64(rand.Intn(100000000))} 57 | copy(m.ID[:], fmt.Sprintf("%016d", m.pri)) 58 | msgs[m.ID] = m 59 | pq.Push(m) 60 | } 61 | 62 | for i := 0; i < 10; i++ { 63 | idx := rand.Intn((c - 1) - i) 64 | var fm *Message 65 | for _, m := range msgs { 66 | if m.index == idx { 67 | fm = m 68 | break 69 | } 70 | } 71 | rm := pq.Remove(idx) 72 | test.Equal(t, fmt.Sprintf("%s", fm.ID), fmt.Sprintf("%s", rm.ID)) 73 | } 74 | 75 | lastPriority := pq.Pop().pri 76 | for i := 0; i < (c - 10 - 1); i++ { 77 | msg := pq.Pop() 78 | test.Equal(t, true, lastPriority <= msg.pri) 79 | lastPriority = msg.pri 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /nsqd/logger.go: -------------------------------------------------------------------------------- 1 | package nsqd 2 | 3 | import ( 4 | "github.com/nsqio/nsq/internal/lg" 5 | ) 6 | 7 | type Logger lg.Logger 8 | 9 | const ( 10 | LOG_DEBUG = lg.DEBUG 11 | LOG_INFO = lg.INFO 12 | LOG_WARN = lg.WARN 13 | LOG_ERROR = lg.ERROR 14 | LOG_FATAL = lg.FATAL 15 | ) 16 | 17 | func (n *NSQD) logf(level lg.LogLevel, f string, args ...interface{}) { 18 | opts := n.getOpts() 19 | lg.Logf(opts.Logger, opts.LogLevel, level, f, args...) 20 | } 21 | -------------------------------------------------------------------------------- /nsqd/message.go: -------------------------------------------------------------------------------- 1 | package nsqd 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "io" 7 | "time" 8 | ) 9 | 10 | const ( 11 | MsgIDLength = 16 12 | minValidMsgLength = MsgIDLength + 8 + 2 // Timestamp + Attempts 13 | ) 14 | 15 | type MessageID [MsgIDLength]byte 16 | 17 | type Message struct { 18 | ID MessageID 19 | Body []byte 20 | Timestamp int64 21 | Attempts uint16 22 | 23 | // for in-flight handling 24 | deliveryTS time.Time 25 | clientID int64 26 | pri int64 27 | index int 28 | deferred time.Duration 29 | } 30 | 31 | func NewMessage(id MessageID, body []byte) *Message { 32 | return &Message{ 33 | ID: id, 34 | Body: body, 35 | Timestamp: time.Now().UnixNano(), 36 | } 37 | } 38 | 39 | func (m *Message) WriteTo(w io.Writer) (int64, error) { 40 | var buf [10]byte 41 | var total int64 42 | 43 | binary.BigEndian.PutUint64(buf[:8], uint64(m.Timestamp)) 44 | binary.BigEndian.PutUint16(buf[8:10], uint16(m.Attempts)) 45 | 46 | n, err := w.Write(buf[:]) 47 | total += int64(n) 48 | if err != nil { 49 | return total, err 50 | } 51 | 52 | n, err = w.Write(m.ID[:]) 53 | total += int64(n) 54 | if err != nil { 55 | return total, err 56 | } 57 | 58 | n, err = w.Write(m.Body) 59 | total += int64(n) 60 | if err != nil { 61 | return total, err 62 | } 63 | 64 | return total, nil 65 | } 66 | 67 | // decodeMessage deserializes data (as []byte) and creates a new Message 68 | // 69 | // [x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x][x]... 70 | // | (int64) || || (hex string encoded in ASCII) || (binary) 71 | // | 8-byte || || 16-byte || N-byte 72 | // ------------------------------------------------------------------------------------------... 73 | // nanosecond timestamp ^^ message ID message body 74 | // (uint16) 75 | // 2-byte 76 | // attempts 77 | func decodeMessage(b []byte) (*Message, error) { 78 | var msg Message 79 | 80 | if len(b) < minValidMsgLength { 81 | return nil, fmt.Errorf("invalid message buffer size (%d)", len(b)) 82 | } 83 | 84 | msg.Timestamp = int64(binary.BigEndian.Uint64(b[:8])) 85 | msg.Attempts = binary.BigEndian.Uint16(b[8:10]) 86 | copy(msg.ID[:], b[10:10+MsgIDLength]) 87 | msg.Body = b[10+MsgIDLength:] 88 | 89 | return &msg, nil 90 | } 91 | 92 | func writeMessageToBackend(msg *Message, bq BackendQueue) error { 93 | buf := bufferPoolGet() 94 | defer bufferPoolPut(buf) 95 | _, err := msg.WriteTo(buf) 96 | if err != nil { 97 | return err 98 | } 99 | return bq.Put(buf.Bytes()) 100 | } 101 | -------------------------------------------------------------------------------- /nsqd/tcp.go: -------------------------------------------------------------------------------- 1 | package nsqd 2 | 3 | import ( 4 | "io" 5 | "net" 6 | "sync" 7 | 8 | "github.com/nsqio/nsq/internal/protocol" 9 | ) 10 | 11 | const ( 12 | typeConsumer = iota 13 | typeProducer 14 | ) 15 | 16 | type Client interface { 17 | Type() int 18 | Stats(string) ClientStats 19 | } 20 | 21 | type tcpServer struct { 22 | nsqd *NSQD 23 | conns sync.Map 24 | } 25 | 26 | func (p *tcpServer) Handle(conn net.Conn) { 27 | p.nsqd.logf(LOG_INFO, "TCP: new client(%s)", conn.RemoteAddr()) 28 | 29 | // The client should initialize itself by sending a 4 byte sequence indicating 30 | // the version of the protocol that it intends to communicate, this will allow us 31 | // to gracefully upgrade the protocol away from text/line oriented to whatever... 32 | buf := make([]byte, 4) 33 | _, err := io.ReadFull(conn, buf) 34 | if err != nil { 35 | p.nsqd.logf(LOG_ERROR, "failed to read protocol version - %s", err) 36 | conn.Close() 37 | return 38 | } 39 | protocolMagic := string(buf) 40 | 41 | p.nsqd.logf(LOG_INFO, "CLIENT(%s): desired protocol magic '%s'", 42 | conn.RemoteAddr(), protocolMagic) 43 | 44 | var prot protocol.Protocol 45 | switch protocolMagic { 46 | case " V2": 47 | prot = &protocolV2{nsqd: p.nsqd} 48 | default: 49 | protocol.SendFramedResponse(conn, frameTypeError, []byte("E_BAD_PROTOCOL")) 50 | conn.Close() 51 | p.nsqd.logf(LOG_ERROR, "client(%s) bad protocol magic '%s'", 52 | conn.RemoteAddr(), protocolMagic) 53 | return 54 | } 55 | 56 | client := prot.NewClient(conn) 57 | p.conns.Store(conn.RemoteAddr(), client) 58 | 59 | err = prot.IOLoop(client) 60 | if err != nil { 61 | p.nsqd.logf(LOG_ERROR, "client(%s) - %s", conn.RemoteAddr(), err) 62 | } 63 | 64 | p.conns.Delete(conn.RemoteAddr()) 65 | client.Close() 66 | } 67 | 68 | func (p *tcpServer) Close() { 69 | p.conns.Range(func(k, v interface{}) bool { 70 | v.(protocol.Client).Close() 71 | return true 72 | }) 73 | } 74 | -------------------------------------------------------------------------------- /nsqd/test/cert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # ./cert.sh foo@foo.com 127.0.0.1 3 | # Found: https://gist.github.com/ncw/9253562#file-makecert-sh 4 | 5 | if [ "$1" == "" ]; then 6 | echo "Need email as argument" 7 | exit 1 8 | fi 9 | 10 | if [ "$2" == "" ]; then 11 | echo "Need CN as argument" 12 | exit 1 13 | fi 14 | 15 | PRIVKEY="test" 16 | EMAIL=$1 17 | CN=$2 18 | 19 | rm -rf tmp 20 | mkdir tmp 21 | cd tmp 22 | 23 | echo "make CA" 24 | openssl req -new -x509 -days 3650 -keyout ca.key -out ca.pem \ 25 | -config ../openssl.conf -extensions ca \ 26 | -subj "/CN=ca" \ 27 | -passout pass:$PRIVKEY 28 | 29 | echo "make server cert" 30 | openssl genrsa -out server.key 2048 31 | openssl req -new -sha256 -key server.key -out server.req \ 32 | -subj "/emailAddress=${EMAIL}/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=${CN}" 33 | openssl x509 -req -days 3650 -sha256 -in server.req -CA ca.pem -CAkey ca.key -CAcreateserial -passin pass:$PRIVKEY -out server.pem \ 34 | -extfile ../openssl.conf -extensions server 35 | 36 | 37 | echo "make client cert" 38 | openssl genrsa -out client.key 2048 39 | openssl req -new -sha256 -key client.key -out client.req \ 40 | -subj "/emailAddress=${EMAIL}/C=DE/ST=NRW/L=Earth/O=Random Company/OU=IT/CN=${CN}" 41 | openssl x509 -req -days 3650 -sha256 -in client.req -CA ca.pem -CAkey ca.key -CAserial ca.srl -passin pass:$PRIVKEY -out client.pem \ 42 | -extfile ../openssl.conf -extensions client 43 | 44 | cd .. 45 | mv tmp/* certs 46 | rm -rf tmp 47 | -------------------------------------------------------------------------------- /nsqd/test/certs/ca.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | Proc-Type: 4,ENCRYPTED 3 | DEK-Info: DES-EDE3-CBC,D6208487450BDF1C 4 | 5 | tucU/4j6agGQlW60D8V3Zr/QHcLhhyFagres1gGdWfGIqluNZb7omki/XidHXJSG 6 | eB9vV2Xb12/8umc31e7Mnmn9hd34230v/KlAnJ4ukDpJpbmjnEx3F9uiqYFi/yxQ 7 | avSsfF6Tsh3XOh3Oe27I/xfYx37g6Agd+EQEJ1hvWvygMIJMTDMP5ZaFoZANtFLy 8 | hDEZ6woJSn9avF/L+1GW8jl2aI1QbdKkK0jDHgFAwUI4sjWeXvEQNNYY3trTIoMo 9 | wab3vi+4XziFONbS4OZrZUYfZPB5YOFbtT2whzggp2HdSTiu48/Ld3N8SjuMrKfm 10 | uR+nd+ovQ5kVWHInzWAIXSyPhgR9ZY8eyXaHNJJfzNu3HY72lfzD/NtZfacMRBr6 11 | M3Wg/OKPS7ZrtqCWkY9P3KK9Cul8Jzy229fSqHo8Rg4= 12 | -----END RSA PRIVATE KEY----- 13 | -------------------------------------------------------------------------------- /nsqd/test/certs/ca.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBVDCB/6ADAgECAgkAvHG4Z/7nX/gwDQYJKoZIhvcNAQEFBQAwDTELMAkGA1UE 3 | AwwCY2EwHhcNMTcwOTE2MTc0NDE0WhcNMjcwOTE0MTc0NDE0WjANMQswCQYDVQQD 4 | DAJjYTBcMA0GCSqGSIb3DQEBAQUAA0sAMEgCQQDJAM3Tr1BoxlLJtTy0oRcp93dT 5 | 9hhHwms8P1V3k2FpXYRS4deUo+uwcAM9KGDt9VMXVBEchtI4VYTvLgatBPUBAgMB 6 | AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMyLx7rjKBe/xZQLnVzI 7 | uqNNVzxRMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAANBAJiN1XgPPlQ2 8 | z7PhtzXStaz/BJVqhD7g9fsZsmoPX4ifDsTfzUsRB56Aq/NTsKiIYQkFPHH0donG 9 | ++a5ZVWjgYk= 10 | -----END CERTIFICATE----- 11 | -------------------------------------------------------------------------------- /nsqd/test/certs/ca.srl: -------------------------------------------------------------------------------- 1 | 91418D04995922E7 2 | -------------------------------------------------------------------------------- /nsqd/test/certs/cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEbjCCA1agAwIBAgIJAK6x7y6AwBmLMA0GCSqGSIb3DQEBBQUAMIGAMQswCQYD 3 | VQQGEwJVUzERMA8GA1UECBMITmV3IFlvcmsxFjAUBgNVBAcTDU5ldyBZb3JrIENp 4 | dHkxDDAKBgNVBAoTA05TUTETMBEGA1UEAxMKdGVzdC5sb2NhbDEjMCEGCSqGSIb3 5 | DQEJARYUbXJlaWZlcnNvbkBnbWFpbC5jb20wHhcNMTMwNjI4MDA0MzQ4WhcNMTYw 6 | NDE3MDA0MzQ4WjCBgDELMAkGA1UEBhMCVVMxETAPBgNVBAgTCE5ldyBZb3JrMRYw 7 | FAYDVQQHEw1OZXcgWW9yayBDaXR5MQwwCgYDVQQKEwNOU1ExEzARBgNVBAMTCnRl 8 | c3QubG9jYWwxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMIIB 9 | IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnX0KB+svwy+yHU2qggz/EaGg 10 | craKShagKo+9M9y5HLM852ngk5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS 11 | 25tVP1UIEnT5pBt2TeetLkl199Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQY 12 | shuEJHKeUNDbQKz5X+GjEdkTPO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECz 13 | ydZBgTPThy3uDtHIuCpxCwXd/vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF 14 | 7m3/0KbtUcXfy0QHueeuMr11E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QID 15 | AQABo4HoMIHlMB0GA1UdDgQWBBR3HMBws4lmYYSIgwoZsfW+bbgaMjCBtQYDVR0j 16 | BIGtMIGqgBR3HMBws4lmYYSIgwoZsfW+bbgaMqGBhqSBgzCBgDELMAkGA1UEBhMC 17 | VVMxETAPBgNVBAgTCE5ldyBZb3JrMRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MQww 18 | CgYDVQQKEwNOU1ExEzARBgNVBAMTCnRlc3QubG9jYWwxIzAhBgkqhkiG9w0BCQEW 19 | FG1yZWlmZXJzb25AZ21haWwuY29tggkArrHvLoDAGYswDAYDVR0TBAUwAwEB/zAN 20 | BgkqhkiG9w0BAQUFAAOCAQEANOYTbanW2iyV1v4oYpcM/y3TWcQKzSME8D2SGFZb 21 | dbMYU81hH3TTlQdvyeh3FAcdjhKE8Xi/RfNNjEslTBscdKXePGpZg6eXRNJzPP5K 22 | KZPf5u6tcpAeUOKrMqbGwbE+h2QixxG1EoVQtE421szsU2P7nHRTdHzKFRnOerfl 23 | Phm3NocR0P40Rv7WKdxpOvqc+XKf0onTruoVYoPWGpwcLixCG0zu4ZQ23/L/Dy18 24 | 4u70Hbq6O/6kq9FBFaDNp3IhiEdu2Cq6ZplU6bL9XDF27KIEErHwtuqBHVlMG+zB 25 | oH/k9vZvwH7OwAjHdKp+1yeZFLYC8K5hjFIHqcdwpZCNIg== 26 | -----END CERTIFICATE----- 27 | -------------------------------------------------------------------------------- /nsqd/test/certs/client.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAuMz5IJ01Wqaj2qJAU7o2aoc2iP1BRhTba+cWrsjbZA+6kXAk 3 | 19Dlz/nNK8gxsBMl3rzTGWXKUm/rKBDp680zJkj8nzcGApGlzbDoqXym8ixOniOC 4 | aH/RODPC8OtD2UozeXlUOQWK+ufkMvmVBc/IeJyFkcm7jhX+LHVFn+3iWLRNbzGm 5 | hbf8oNHvybS7HSuV3OYiHDJxA0d6IgozqVOYVSqdDabQ2FN/8nKzpMHjwMpfT008 6 | 5hUhr72E5XuJ3biPUwoOrRETSa1/EQcAEH0wSwHU1AGdnGh3/5ynVupfef0qkyqw 7 | NseYs9U++IUFXqA1IuEmAlJn5QczAHjPQe8xBQIDAQABAoIBAGx4CyZEgCOUOgrD 8 | P2SloPkIIk9n7x82cNA11I+E35kszkI9g7KVL77SDcZL/DYwFwNU68c1gvq+LFXZ 9 | D6RTTlmDb5v4TPPHD33a/8UzoD33GbIif5HcrC4D28FTJgDtV6dOOsw5X6kD4WK2 10 | Me02V6HLpW677PVqHUV1FAfaNgf/1SIhJCechYSO+3pKPtli8DnsmOd336wNtWZR 11 | Q3Ctm63Eq7AiLmGLOkLYtll9Kr6WfCSFHBmqJg9MsQEfINXMCOB54Z+xkoijjJLO 12 | zNSk/3XvppB1JpmAusbo9Mq03Ci9JdinYSKxwVOG6e+5cosrOHpDQyZIWa87ZTpk 13 | kjLCgEECgYEA33JCGTEwaqfAuYOspCZvi2Mqz7t6QiwvwxgDf4JFLgDQArJfzKlC 14 | bAzaSU8bT+SN96CF9LU4XekN5/paGZAFjWxraPkK0oL28kQrtzfEQv2XL6GFoNAh 15 | WsQtoaGddL9VPWlCiHeueYERIDNrZCaVeAx6OV6W3CEwGOkRiQyntlUCgYEA07lf 16 | LpxF8aAgMToq3++D5RGAXaT2PYxhZZFb5Y2tS7a4T0ulQwXQi4H2dObGJ32zCjFG 17 | ls1H91mNKGBbyKnF1jT/dt/acqymvipASm1xq6uysIllA4xp56sDnJgG9bnXjYFY 18 | m2yVNIoIQGZ7jp7e6t1KQ9San6V8asWlvBLXX/ECgYEAuhF9dVj+xnH3DQTXSMIw 19 | 9NOZnO6zelMtWrqufwnN7ecDUJuVJupzw2JYi99yEO90QRbNNd+KlrkxuVFCojLK 20 | TOBR+VIZbv9cAJZACQxJRLfDpAhPLIDkpZ7jmMrqQYPqyX7TxqxTAB84UaY/8WAn 21 | 65YIWamo2ppQYQ4Eaim9pxkCgYAMwfW/TEFWruxhqvycY8VRzz0p51/DE6tmwFyG 22 | N4RCtK7kcE1z/Wy0i087ehBknslkCtYTDimQ+P9teGjvbXNzVdwy4Ig8MrUVblxT 23 | X8birkTlKFJC5XoYMJDWJb79nYYki6+4JdHTyaF3p/U4AdCy3ES2U6BBkGov0NsM 24 | uyHpMQKBgGc9wIGHwAGLg3iVHGXbXELdpzVztLRC7D12TPfdtGBLTD5NH5jAx8K/ 25 | w61+l4tzFcK3jfWzwdme277cBgfq/aw2DDt07vkOvPzhUwVZiE39bfddfIvyed99 26 | XzEMf8THh5wdm6mLsOVgdfOfFcDi5ReIv1/7+eOd/MQi8TxIVfAu 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /nsqd/test/certs/client.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICyzCCAnWgAwIBAgIJAJFBjQSZWSLnMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV 3 | BAMMAmNhMB4XDTE3MDkxNjE3NDQxNFoXDTI3MDkxNDE3NDQxNFowgYcxIzAhBgkq 4 | hkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMQswCQYDVQQGEwJERTEMMAoG 5 | A1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEXMBUGA1UEChMOUmFuZG9tIENvbXBh 6 | bnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZuc3EuaW8wggEiMA0GCSqGSIb3DQEB 7 | AQUAA4IBDwAwggEKAoIBAQC4zPkgnTVapqPaokBTujZqhzaI/UFGFNtr5xauyNtk 8 | D7qRcCTX0OXP+c0ryDGwEyXevNMZZcpSb+soEOnrzTMmSPyfNwYCkaXNsOipfKby 9 | LE6eI4Jof9E4M8Lw60PZSjN5eVQ5BYr65+Qy+ZUFz8h4nIWRybuOFf4sdUWf7eJY 10 | tE1vMaaFt/yg0e/JtLsdK5Xc5iIcMnEDR3oiCjOpU5hVKp0NptDYU3/ycrOkwePA 11 | yl9PTTzmFSGvvYTle4nduI9TCg6tERNJrX8RBwAQfTBLAdTUAZ2caHf/nKdW6l95 12 | /SqTKrA2x5iz1T74hQVeoDUi4SYCUmflBzMAeM9B7zEFAgMBAAGjdTBzMAwGA1Ud 13 | EwEB/wQCMAAwHQYDVR0OBBYEFMzYXiTD7moi0oXqSQ5D2LsO51GXMB8GA1UdIwQY 14 | MBaAFMyLx7rjKBe/xZQLnVzIuqNNVzxRMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUE 15 | DDAKBggrBgEFBQcDAjANBgkqhkiG9w0BAQsFAANBAAujxas6toRhMl/+kZrly0G/ 16 | AvjbA3WY5cLLIdffGdQ5bsS3aOP23nj98ut7unNUNsCo+eUwpJgvabnFnL+NFZA= 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /nsqd/test/certs/client.req: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu 3 | Y29tMQswCQYDVQQGEwJERTEMMAoGA1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEX 4 | MBUGA1UEChMOUmFuZG9tIENvbXBhbnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZu 5 | c3EuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC4zPkgnTVapqPa 6 | okBTujZqhzaI/UFGFNtr5xauyNtkD7qRcCTX0OXP+c0ryDGwEyXevNMZZcpSb+so 7 | EOnrzTMmSPyfNwYCkaXNsOipfKbyLE6eI4Jof9E4M8Lw60PZSjN5eVQ5BYr65+Qy 8 | +ZUFz8h4nIWRybuOFf4sdUWf7eJYtE1vMaaFt/yg0e/JtLsdK5Xc5iIcMnEDR3oi 9 | CjOpU5hVKp0NptDYU3/ycrOkwePAyl9PTTzmFSGvvYTle4nduI9TCg6tERNJrX8R 10 | BwAQfTBLAdTUAZ2caHf/nKdW6l95/SqTKrA2x5iz1T74hQVeoDUi4SYCUmflBzMA 11 | eM9B7zEFAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEALD1VjbBjaeX7l7JR4IuL 12 | wrHVrFDiRYsgvqyw39j2MC95VwrGwzf4cXCE+RuqE/DhbV9UI7sWKaJfFs9Usq+g 13 | VSoKnHSEylt34y6ABSc5eAik+GnheJZbJ6UDjxcvNd0UpFGMrHbsXVyQd0Y1XAu7 14 | nxCYIa82kNA+Opb+ra03hkLC7wvRLbXTOB2g6JLkyhYR6S5GkOFTNnz2AJerN7zt 15 | NEL7owlRcjyIsL5tjDpDH1944NNtzhgmrUeIjB08reayuot9RKznMVGwBfY6DIHM 16 | Q4uNN3CMOOoAHr1UzvBf/qfvb6ltPTMKSV1OncLlC3C59NoO8vhKIHCN18Ya2OMu 17 | rw== 18 | -----END CERTIFICATE REQUEST----- 19 | -------------------------------------------------------------------------------- /nsqd/test/certs/key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEogIBAAKCAQEAnX0KB+svwy+yHU2qggz/EaGgcraKShagKo+9M9y5HLM852ng 3 | k5c+t+tJJbx3N954Wr1FXBuGIv1ltU05rU4zhvBS25tVP1UIEnT5pBt2TeetLkl1 4 | 99Y7fxh1hKmnwJMG3fy3VZdNXEndBombXMmtXpQYshuEJHKeUNDbQKz5X+GjEdkT 5 | PO/HY/VMHsxS23pbSimQozMg3hvLIdgv0aS3QECzydZBgTPThy3uDtHIuCpxCwXd 6 | /vDF68ATlYgo3h3lh2vxNwM/pjklIUhzMh4XaKQF7m3/0KbtUcXfy0QHueeuMr11 7 | E9MAFNyRN4xf9Fk1yB97KJ3PJBTC5WD/m1nW+QIDAQABAoIBACvtfKbIywG+hAf4 8 | ad7skRjx5DcbA2e29+XnQfb9UgTXWd2SgrmoLi5OypBkCTzkKN3mfTo70yZfV8dC 9 | Sxwz+9tfnTz0DssjhKThS+CiaFVCkeOfSfBfKSlCQUVHrSrh18CDhP+yvDlJwQTZ 10 | zSQMfPcsh9bmJe2kqtQP7ZgUp1o+vaB8Sju8YYrO6FllxbdLRGm4pfvvrHIRRmXa 11 | oVHn0ei0JpwoTY9kHYht4LNeJnbP/MCWdmcuv3Gnel7jAlhaKab5aNIGr0Xe7aIQ 12 | iX6mpZ0/Rnt8o/XcTOg8l3ruIdVuySX6SYn08JMnfFkXdNYRVhoV1tC5ElWkaZLf 13 | hPmj2yECgYEAyts0R0b8cZ6HTAyuLm3ilw0s0v0/MM9ZtaqMRilr2WEtAhF0GpHG 14 | TzmGnii0WcTNXD7NTsNcECR/0ZpXPRleMczsL2Juwd4FkQ37h7hdKPseJNrfyHRg 15 | VolOFBX9H14C3wMB9cwdsG4Egw7fE27WCoreEquHgwFxl1zBrXKH088CgYEAxr8w 16 | BKZs0bF7LRrFT5pH8hpMLYHMYk8ZIOfgmEGVBKDQCOERPR9a9kqUss7wl/98LVNK 17 | RnFlyWD6Z0/QcQsLL4LjBeZJ25qEMc6JXm9VGAzhXA1ZkUofVoYCnG+f6KUn8CuJ 18 | /AcV2ZDFsEP10IiQG0hKsceXiwFEvEr8306tMrcCgYBLgnscSR0xAeyk71dq6vZc 19 | ecgEpcX+2kAvclOSzlpZ6WVCjtKkDT0/Qk+M0eQIQkybGLl9pxS+4Yc+s2/jy2yX 20 | pwsHvGE0AvwZeZX2eDcdSRR4bYy9ZixyKdwJeAHnyivRbaIuJ5Opl9pQGpoI9snv 21 | 1K9DTdw8dK4exKVHdgl/WwKBgDkmLsuXg4EEtPOyV/xc08VVNIR9Z2T5c7NXmeiO 22 | KyiKiWeUOF3ID2L07S9BfENozq9F3PzGjMtMXJSqibiHwW6nB1rh7mj8VHjx9+Q0 23 | xVZGFeNfX1r84mgB3uxW2LeQDhzsmB/lda37CC14TU3qhu2hawEV8IijE73FHlOk 24 | Dv+fAoGAI4/XO5o5tNn5Djo8gHmGMCbinUE9+VySxl7wd7PK8w2VSofO88ofixDk 25 | NX94yBYhg5WZcLdPm45RyUnq+WVQYz9IKUrdxLFTH+wxyzUqZCW7jgXCvWV+071q 26 | vqm9C+kndq+18/1VKuCSGWnF7Ay4lbsgPXY2s4VKRxcb3QpZSPU= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /nsqd/test/certs/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEowIBAAKCAQEAuYs7NerYpXWq/IPaBGriHobhIs/i4AJHra7QhZ5N/EfBCb0N 3 | 6K3tI4GRHMZ4lB0HwINWmxaPs6D2yDCxpGgnru6qCu0/91+Lq+GEZj+Fz1E2yfCq 4 | tecTySvWf9en+SdTKFZTGGIAu3zFcozuX2fdhkjJ6ibZhnrXKlPU4IN0STkkRfFX 5 | +yb/6P88mGh7PZKBb67Qw1bppzXE/SWgHVqDqu++whT1quwxnRhh/SHgwugN+6iK 6 | uTDcJ1jQ1QWe+xna7Te6nNBUvL6dV49jpoDOjHsv+UaAvzNeHCHqypTC0wwYHXyg 7 | TTAU+E20c2NOCC7P0PAid1liNeuuEWtM2bRrFQIDAQABAoIBABPzc7d1fDw2bd9f 8 | Mic9cvkDWdwLbILX2+tCG+vyPMJ+2LP6Xy+A3DnwKbFlafvLL1U1Ci/8+hC/oymd 9 | isx54qJ9yU0Je9JWtMcTpc/0zqefPPvz4/dRVKBSFWuDve0dnGR++8poZ1nBrd2G 10 | Z+9cVMamtwd1i/hY5yAHCaHmoK9qxXuSjtbPkqDqruloGO5iT+tcICjAeWyJIx1P 11 | 5VJjuKDx4AhU4B8DpBSTtA02BKltvr2K6D4zhJ255oI4iLUkYk4IwxB7yZH+6Y++ 12 | mDa6e8iM3ut+qMkCxto/CfMCqYfoRzmykiazZ5qFXfGwv3OkWUEOBXU//Vpklvnh 13 | OQWYL/UCgYEA7mkKXbCrCNJEdly4LVLPKPFjVUPhMdXcqdnoKZ6BiVYFMqTuG4+w 14 | o/Rf6ZpqZ3Rf656/ypd1atVvK2g4zfq3j1W6lDnpLjKaLhc8a+ZH4NWXBORIzlUW 15 | aR2xVcPpAokor37FRAWbduiwVSCHlDtg48i+rvukyqmeRBKjrIgX3lsCgYEAxzut 16 | 5GiMZ9TkjqYXdlUDIDKWMHMfLfRY7+Q/cBdr7AfiMVw4PIWJG/IuxCGwGp77gRg6 17 | BeBsi8Htj5/EGbRiK4kjFyf8LqYGOeYLZ3I8+olr+tNmF4/yKwxku3VylpmrpCAV 18 | /8emso1rAWo6Y98MTNSGgvLqhDU6OD7tTSv3908CgYA4+aVeio/1RbrSxonFWxri 19 | 3/0rLVOuAzv+43KWL6kpVwNa/QtiTs6aABbDzwFKxAcAWinfkp6e727n4rpgj2A6 20 | wvQZ5FUTk0hBZ5ArAReAZcr3gk7b8H2wlUYCBxWyY3Dzr8oY3XYvzqAFWAbOp/oZ 21 | tanMS5swS6TlA8dVvhhmLQKBgGZwcBPOEctdcntKOSwVv/qxJ/oXZ0O4rHYENP4M 22 | fOgqkYnxsdSkkH/3AUbFT4gQkJ6q90KIRyeA+gXsDudskUFzTMCeRZMyuGbSurBg 23 | 06u6NvQL+CVLVSf/QlgEpnt63f8QpF8Up8iM4CUlGoq5Z9ilOdhg0GZT+/BpopgY 24 | cHIPAoGBAKyLumgeG+gMFK5x/Fi6zjBT5MK8Tw0VMkahW01Jx2laYibS/a8AEFmn 25 | ySdrmLPkOmmWgXCk3m2m5PkvM5qH/KNugOA0+WX2CTvwt4ZgwWUalQ5R43wSIeDC 26 | MXVfwC8uE66PmpYgmsu4H0vnCGfacOCQhfdq01SLobgBiQSrm/6D 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /nsqd/test/certs/server.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIC3jCCAoigAwIBAgIJAJFBjQSZWSLmMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV 3 | BAMMAmNhMB4XDTE3MDkxNjE3NDQxNFoXDTI3MDkxNDE3NDQxNFowgYcxIzAhBgkq 4 | hkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwuY29tMQswCQYDVQQGEwJERTEMMAoG 5 | A1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEXMBUGA1UEChMOUmFuZG9tIENvbXBh 6 | bnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZuc3EuaW8wggEiMA0GCSqGSIb3DQEB 7 | AQUAA4IBDwAwggEKAoIBAQC5izs16tildar8g9oEauIehuEiz+LgAketrtCFnk38 8 | R8EJvQ3ore0jgZEcxniUHQfAg1abFo+zoPbIMLGkaCeu7qoK7T/3X4ur4YRmP4XP 9 | UTbJ8Kq15xPJK9Z/16f5J1MoVlMYYgC7fMVyjO5fZ92GSMnqJtmGetcqU9Tgg3RJ 10 | OSRF8Vf7Jv/o/zyYaHs9koFvrtDDVumnNcT9JaAdWoOq777CFPWq7DGdGGH9IeDC 11 | 6A37qIq5MNwnWNDVBZ77GdrtN7qc0FS8vp1Xj2OmgM6Mey/5RoC/M14cIerKlMLT 12 | DBgdfKBNMBT4TbRzY04ILs/Q8CJ3WWI1664Ra0zZtGsVAgMBAAGjgYcwgYQwDAYD 13 | VR0TAQH/BAIwADAdBgNVHQ4EFgQUt6UXJ8BQB++I/cQG5ZgaiUyqOE8wHwYDVR0j 14 | BBgwFoAUzIvHuuMoF7/FlAudXMi6o01XPFEwDgYDVR0PAQH/BAQDAgWgMBMGA1Ud 15 | JQQMMAoGCCsGAQUFBwMBMA8GA1UdEQQIMAaHBH8AAAEwDQYJKoZIhvcNAQELBQAD 16 | QQCCwm0F2eTmtIcKDxrXGZ7q9y9mfsROfYaCnH+vKUDw2vmqQkInzhLeEElDGQcR 17 | ww0IKCnDHEruNb2tKyQM/70L 18 | -----END CERTIFICATE----- 19 | -------------------------------------------------------------------------------- /nsqd/test/certs/server.req: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIICzTCCAbUCAQAwgYcxIzAhBgkqhkiG9w0BCQEWFG1yZWlmZXJzb25AZ21haWwu 3 | Y29tMQswCQYDVQQGEwJERTEMMAoGA1UECBMDTlJXMQ4wDAYDVQQHEwVFYXJ0aDEX 4 | MBUGA1UEChMOUmFuZG9tIENvbXBhbnkxCzAJBgNVBAsTAklUMQ8wDQYDVQQDEwZu 5 | c3EuaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC5izs16tildar8 6 | g9oEauIehuEiz+LgAketrtCFnk38R8EJvQ3ore0jgZEcxniUHQfAg1abFo+zoPbI 7 | MLGkaCeu7qoK7T/3X4ur4YRmP4XPUTbJ8Kq15xPJK9Z/16f5J1MoVlMYYgC7fMVy 8 | jO5fZ92GSMnqJtmGetcqU9Tgg3RJOSRF8Vf7Jv/o/zyYaHs9koFvrtDDVumnNcT9 9 | JaAdWoOq777CFPWq7DGdGGH9IeDC6A37qIq5MNwnWNDVBZ77GdrtN7qc0FS8vp1X 10 | j2OmgM6Mey/5RoC/M14cIerKlMLTDBgdfKBNMBT4TbRzY04ILs/Q8CJ3WWI1664R 11 | a0zZtGsVAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEASNjcoZbyNcwMQjMmNoil 12 | S/7pCRn4aYzZIVjrVtOHQ9GHC23MSep5um2gIcMFPiuYyu9Byl8CSVtc1op2fAKS 13 | vrugoZaCrp/A76hqOfNxgh7VmgTux8bG5Qcjaija1BNWpbyaWARdBxN/WgS5CpCj 14 | u2yzv8mrzzFNrDMlsmiEMvtkMzdhiZ4YY8zm6CdrbIR5z1eqf4e+rs4oJtTKNNAD 15 | hewk8CGiUW1hOx2jpjcIVMRy+ofVHRX2xQ6Sw8qxCNsiv8IPAAivgAbFJO76ZSbH 16 | eQ7uKWszmBEroyFvZ0rfmFLXuopU125pyBDl5FUKYAZzCBx9tr5dROCbw/rXDhke 17 | ig== 18 | -----END CERTIFICATE REQUEST----- 19 | -------------------------------------------------------------------------------- /nsqd/test/openssl.conf: -------------------------------------------------------------------------------- 1 | [req] 2 | distinguished_name = req_distinguished_name 3 | 4 | [req_distinguished_name] 5 | 6 | [ca] 7 | basicConstraints = critical, CA:true 8 | subjectKeyIdentifier = hash 9 | keyUsage = critical, cRLSign, keyCertSign 10 | 11 | [client] 12 | basicConstraints = critical, CA:FALSE 13 | subjectKeyIdentifier = hash 14 | authorityKeyIdentifier = keyid,issuer 15 | keyUsage = critical, digitalSignature, keyEncipherment 16 | extendedKeyUsage = clientAuth 17 | 18 | [server] 19 | basicConstraints = critical, CA:FALSE 20 | subjectKeyIdentifier = hash 21 | authorityKeyIdentifier = keyid,issuer 22 | keyUsage = critical, digitalSignature, keyEncipherment 23 | extendedKeyUsage = serverAuth 24 | subjectAltName = IP:127.0.0.1 25 | -------------------------------------------------------------------------------- /nsqlookupd/README.md: -------------------------------------------------------------------------------- 1 | ## nsqlookupd 2 | 3 | `nsqlookupd` is the daemon that manages topology metadata and serves client requests to 4 | discover the location of topics at runtime. 5 | 6 | Read the [docs](https://nsq.io/components/nsqlookupd.html) 7 | -------------------------------------------------------------------------------- /nsqlookupd/client_v1.go: -------------------------------------------------------------------------------- 1 | package nsqlookupd 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | type ClientV1 struct { 8 | net.Conn 9 | peerInfo *PeerInfo 10 | } 11 | 12 | func NewClientV1(conn net.Conn) *ClientV1 { 13 | return &ClientV1{ 14 | Conn: conn, 15 | } 16 | } 17 | 18 | func (c *ClientV1) String() string { 19 | return c.RemoteAddr().String() 20 | } 21 | -------------------------------------------------------------------------------- /nsqlookupd/logger.go: -------------------------------------------------------------------------------- 1 | package nsqlookupd 2 | 3 | import ( 4 | "github.com/nsqio/nsq/internal/lg" 5 | ) 6 | 7 | type Logger lg.Logger 8 | 9 | const ( 10 | LOG_DEBUG = lg.DEBUG 11 | LOG_INFO = lg.INFO 12 | LOG_WARN = lg.WARN 13 | LOG_ERROR = lg.ERROR 14 | LOG_FATAL = lg.FATAL 15 | ) 16 | 17 | func (n *NSQLookupd) logf(level lg.LogLevel, f string, args ...interface{}) { 18 | lg.Logf(n.opts.Logger, n.opts.LogLevel, level, f, args...) 19 | } 20 | -------------------------------------------------------------------------------- /nsqlookupd/lookup_protocol_v1_test.go: -------------------------------------------------------------------------------- 1 | package nsqlookupd 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | 8 | "github.com/nsqio/nsq/internal/protocol" 9 | "github.com/nsqio/nsq/internal/test" 10 | ) 11 | 12 | func TestIOLoopReturnsClientErrWhenSendFails(t *testing.T) { 13 | fakeConn := test.NewFakeNetConn() 14 | fakeConn.WriteFunc = func(b []byte) (int, error) { 15 | return 0, errors.New("write error") 16 | } 17 | 18 | testIOLoopReturnsClientErr(t, fakeConn) 19 | } 20 | 21 | func TestIOLoopReturnsClientErrWhenSendSucceeds(t *testing.T) { 22 | fakeConn := test.NewFakeNetConn() 23 | fakeConn.WriteFunc = func(b []byte) (int, error) { 24 | return len(b), nil 25 | } 26 | 27 | testIOLoopReturnsClientErr(t, fakeConn) 28 | } 29 | 30 | func testIOLoopReturnsClientErr(t *testing.T, fakeConn test.FakeNetConn) { 31 | fakeConn.ReadFunc = func(b []byte) (int, error) { 32 | return copy(b, []byte("INVALID_COMMAND\n")), nil 33 | } 34 | 35 | opts := NewOptions() 36 | opts.Logger = test.NewTestLogger(t) 37 | opts.LogLevel = LOG_DEBUG 38 | opts.TCPAddress = "127.0.0.1:0" 39 | opts.HTTPAddress = "127.0.0.1:0" 40 | 41 | nsqlookupd, err := New(opts) 42 | test.Nil(t, err) 43 | prot := &LookupProtocolV1{nsqlookupd: nsqlookupd} 44 | 45 | nsqlookupd.tcpServer = &tcpServer{nsqlookupd: prot.nsqlookupd} 46 | 47 | errChan := make(chan error) 48 | testIOLoop := func() { 49 | client := prot.NewClient(fakeConn) 50 | errChan <- prot.IOLoop(client) 51 | defer prot.nsqlookupd.Exit() 52 | } 53 | go testIOLoop() 54 | 55 | var timeout bool 56 | 57 | select { 58 | case err = <-errChan: 59 | case <-time.After(2 * time.Second): 60 | timeout = true 61 | } 62 | 63 | test.Equal(t, false, timeout) 64 | 65 | test.NotNil(t, err) 66 | test.Equal(t, "E_INVALID invalid command INVALID_COMMAND", err.Error()) 67 | test.NotNil(t, err.(*protocol.FatalClientErr)) 68 | } 69 | -------------------------------------------------------------------------------- /nsqlookupd/nsqlookupd.go: -------------------------------------------------------------------------------- 1 | package nsqlookupd 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | "os" 8 | "sync" 9 | 10 | "github.com/nsqio/nsq/internal/http_api" 11 | "github.com/nsqio/nsq/internal/protocol" 12 | "github.com/nsqio/nsq/internal/util" 13 | "github.com/nsqio/nsq/internal/version" 14 | ) 15 | 16 | type NSQLookupd struct { 17 | sync.RWMutex 18 | opts *Options 19 | tcpListener net.Listener 20 | httpListener net.Listener 21 | tcpServer *tcpServer 22 | waitGroup util.WaitGroupWrapper 23 | DB *RegistrationDB 24 | } 25 | 26 | func New(opts *Options) (*NSQLookupd, error) { 27 | var err error 28 | 29 | if opts.Logger == nil { 30 | opts.Logger = log.New(os.Stderr, opts.LogPrefix, log.Ldate|log.Ltime|log.Lmicroseconds) 31 | } 32 | l := &NSQLookupd{ 33 | opts: opts, 34 | DB: NewRegistrationDB(), 35 | } 36 | 37 | l.logf(LOG_INFO, version.String("nsqlookupd")) 38 | 39 | l.tcpServer = &tcpServer{nsqlookupd: l} 40 | l.tcpListener, err = net.Listen("tcp", opts.TCPAddress) 41 | if err != nil { 42 | return nil, fmt.Errorf("listen (%s) failed - %s", opts.TCPAddress, err) 43 | } 44 | l.httpListener, err = net.Listen("tcp", opts.HTTPAddress) 45 | if err != nil { 46 | return nil, fmt.Errorf("listen (%s) failed - %s", opts.HTTPAddress, err) 47 | } 48 | 49 | return l, nil 50 | } 51 | 52 | // Main starts an instance of nsqlookupd and returns an 53 | // error if there was a problem starting up. 54 | func (l *NSQLookupd) Main() error { 55 | exitCh := make(chan error) 56 | var once sync.Once 57 | exitFunc := func(err error) { 58 | once.Do(func() { 59 | if err != nil { 60 | l.logf(LOG_FATAL, "%s", err) 61 | } 62 | exitCh <- err 63 | }) 64 | } 65 | 66 | l.waitGroup.Wrap(func() { 67 | exitFunc(protocol.TCPServer(l.tcpListener, l.tcpServer, l.logf)) 68 | }) 69 | httpServer := newHTTPServer(l) 70 | l.waitGroup.Wrap(func() { 71 | exitFunc(http_api.Serve(l.httpListener, httpServer, "HTTP", l.logf)) 72 | }) 73 | 74 | err := <-exitCh 75 | return err 76 | } 77 | 78 | func (l *NSQLookupd) RealTCPAddr() *net.TCPAddr { 79 | return l.tcpListener.Addr().(*net.TCPAddr) 80 | } 81 | 82 | func (l *NSQLookupd) RealHTTPAddr() *net.TCPAddr { 83 | return l.httpListener.Addr().(*net.TCPAddr) 84 | } 85 | 86 | func (l *NSQLookupd) Exit() { 87 | if l.tcpListener != nil { 88 | l.tcpListener.Close() 89 | } 90 | 91 | if l.tcpServer != nil { 92 | l.tcpServer.Close() 93 | } 94 | 95 | if l.httpListener != nil { 96 | l.httpListener.Close() 97 | } 98 | l.waitGroup.Wait() 99 | } 100 | -------------------------------------------------------------------------------- /nsqlookupd/options.go: -------------------------------------------------------------------------------- 1 | package nsqlookupd 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "time" 7 | 8 | "github.com/nsqio/nsq/internal/lg" 9 | ) 10 | 11 | type Options struct { 12 | LogLevel lg.LogLevel `flag:"log-level"` 13 | LogPrefix string `flag:"log-prefix"` 14 | Logger Logger 15 | 16 | TCPAddress string `flag:"tcp-address"` 17 | HTTPAddress string `flag:"http-address"` 18 | BroadcastAddress string `flag:"broadcast-address"` 19 | 20 | InactiveProducerTimeout time.Duration `flag:"inactive-producer-timeout"` 21 | TombstoneLifetime time.Duration `flag:"tombstone-lifetime"` 22 | } 23 | 24 | func NewOptions() *Options { 25 | hostname, err := os.Hostname() 26 | if err != nil { 27 | log.Fatal(err) 28 | } 29 | 30 | return &Options{ 31 | LogPrefix: "[nsqlookupd] ", 32 | LogLevel: lg.INFO, 33 | TCPAddress: "0.0.0.0:4160", 34 | HTTPAddress: "0.0.0.0:4161", 35 | BroadcastAddress: hostname, 36 | 37 | InactiveProducerTimeout: 300 * time.Second, 38 | TombstoneLifetime: 45 * time.Second, 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /nsqlookupd/tcp.go: -------------------------------------------------------------------------------- 1 | package nsqlookupd 2 | 3 | import ( 4 | "io" 5 | "net" 6 | "sync" 7 | 8 | "github.com/nsqio/nsq/internal/protocol" 9 | ) 10 | 11 | type tcpServer struct { 12 | nsqlookupd *NSQLookupd 13 | conns sync.Map 14 | } 15 | 16 | func (p *tcpServer) Handle(conn net.Conn) { 17 | p.nsqlookupd.logf(LOG_INFO, "TCP: new client(%s)", conn.RemoteAddr()) 18 | 19 | // The client should initialize itself by sending a 4 byte sequence indicating 20 | // the version of the protocol that it intends to communicate, this will allow us 21 | // to gracefully upgrade the protocol away from text/line oriented to whatever... 22 | buf := make([]byte, 4) 23 | _, err := io.ReadFull(conn, buf) 24 | if err != nil { 25 | p.nsqlookupd.logf(LOG_ERROR, "failed to read protocol version - %s", err) 26 | conn.Close() 27 | return 28 | } 29 | protocolMagic := string(buf) 30 | 31 | p.nsqlookupd.logf(LOG_INFO, "CLIENT(%s): desired protocol magic '%s'", 32 | conn.RemoteAddr(), protocolMagic) 33 | 34 | var prot protocol.Protocol 35 | switch protocolMagic { 36 | case " V1": 37 | prot = &LookupProtocolV1{nsqlookupd: p.nsqlookupd} 38 | default: 39 | protocol.SendResponse(conn, []byte("E_BAD_PROTOCOL")) 40 | conn.Close() 41 | p.nsqlookupd.logf(LOG_ERROR, "client(%s) bad protocol magic '%s'", 42 | conn.RemoteAddr(), protocolMagic) 43 | return 44 | } 45 | 46 | client := prot.NewClient(conn) 47 | p.conns.Store(conn.RemoteAddr(), client) 48 | 49 | err = prot.IOLoop(client) 50 | if err != nil { 51 | p.nsqlookupd.logf(LOG_ERROR, "client(%s) - %s", conn.RemoteAddr(), err) 52 | } 53 | 54 | p.conns.Delete(conn.RemoteAddr()) 55 | client.Close() 56 | } 57 | 58 | func (p *tcpServer) Close() { 59 | p.conns.Range(func(k, v interface{}) bool { 60 | v.(protocol.Client).Close() 61 | return true 62 | }) 63 | } 64 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | GOMAXPROCS=1 go test -timeout 90s ./... 5 | 6 | if [ "$GOARCH" = "amd64" ] || [ "$GOARCH" = "arm64" ]; then 7 | # go test: -race is only supported on linux/amd64, linux/ppc64le, 8 | # linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64 9 | GOMAXPROCS=4 go test -timeout 90s -race ./... 10 | fi 11 | 12 | # no tests, but a build is something 13 | for dir in apps/*/ bench/*/; do 14 | dir=${dir%/} 15 | if grep -q '^package main$' $dir/*.go 2>/dev/null; then 16 | echo "building $dir" 17 | go build -o $dir/$(basename $dir) ./$dir 18 | else 19 | echo "(skipped $dir)" 20 | fi 21 | done 22 | 23 | # disable "composite literal uses unkeyed fields" 24 | go vet -composites=false ./... 25 | 26 | FMTDIFF="$(find apps internal nsqd nsqlookupd -name '*.go' -exec gofmt -d '{}' ';')" 27 | if [ -n "$FMTDIFF" ]; then 28 | printf '%s\n' "$FMTDIFF" 29 | exit 1 30 | fi 31 | --------------------------------------------------------------------------------