7 |
9 |
3 |
13 |
14 |
15 |
16 |
32 |
33 |
45 |
--------------------------------------------------------------------------------
/document/api/admin/api/consumeconfig/create.md:
--------------------------------------------------------------------------------
1 | ### 创建消费者
2 |
3 | - 接口功能
4 |
5 | > 创建一个新的消费者
6 |
7 | ```
8 | GET /admin/api/consumeConfig/create
9 | ```
10 |
11 | - 请求参数
12 |
13 | |参数|必选|类型|说明|
14 | |:----- |:-------|:-----|----- |
15 | |topic |true |string |Topic名 |
16 | |channel |true |string |Channel名 |
17 | |description |false |string |描述，默认空 |
18 | |owner |false |string |责任人，默认空 |
19 | |monitorThreshold |false |int |积压报警阈值，默认50000 |
20 | |handleNum |false |int |该队列的并发量，默认2 |
21 | |maxInFlight |false |int |NSQD最多同时推送多少个消息，默认2 |
22 | |isRequeue |false |bool |失败，超时等情况是否重新入队，默认false |
23 | |timeoutDial |false |int |超时时间，默认3590秒 |
24 | |timeoutRead |false |int |读超时时间，默认3590秒 |
25 | |timeoutWrite |false |int |写超时时间，默认3590秒 |
26 | |invalid |false |int |是否有效，默认0有效，1无效 |
27 |
28 | - 返回结果
29 |
30 | ```
31 | {
32 | code: 200,
33 | msg: "ok",
34 | result: {
35 | id: 1,
36 | topic: "test_topic_1",
37 | channel: "one",
38 | description: "描述",
39 | owner: "责任人",
40 | monitorThreshold: 0,
41 | handleNum: 6,
42 | maxInFlight: 6,
43 | isRequeue: false,
44 | timeoutDial: 3590,
45 | timeoutRead: 3590,
46 | timeoutWrite: 3590,
47 | invalid: 0,
48 | createdAt: "2020-08-20T15:08:39+08:00",
49 | updatedAt: "0001-01-01T00:00:00Z",
50 | serverList: null,
51 | }
52 | }
53 | ```
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
2 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
3 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
4 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
5 | github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
6 | github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
7 | github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E=
8 | github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
9 | github.com/nsqio/go-nsq v1.0.8 h1:3L2F8tNLlwXXlp2slDUrUWSBn2O3nMh8R1/KEDFTHPk=
10 | github.com/nsqio/go-nsq v1.0.8/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
11 | github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
12 | github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
13 | gorm.io/driver/mysql v1.0.1 h1:omJoilUzyrAp0xNoio88lGJCroGdIOen9hq2A/+3ifw=
14 | gorm.io/driver/mysql v1.0.1/go.mod h1:KtqSthtg55lFp3S5kUXqlGaelnWpKitn4k1xZTnoiPw=
15 | gorm.io/gorm v1.9.19/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
16 | gorm.io/gorm v1.20.1 h1:+hOwlHDqvqmBIMflemMVPLJH7tZYK4RxFDBHEfJTup0=
17 | gorm.io/gorm v1.20.1/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
18 |
--------------------------------------------------------------------------------
/web/vue-admin/src/layout/mixin/ResizeHandler.js:
--------------------------------------------------------------------------------
1 | import store from '@/store'
2 |
3 | const { body } = document
4 | const WIDTH = 992 // refer to Bootstrap's responsive design
5 |
6 | export default {
7 | watch: {
8 | $route(route) {
9 | if (this.device === 'mobile' && this.sidebar.opened) {
10 | store.dispatch('app/closeSideBar', { withoutAnimation: false })
11 | }
12 | }
13 | },
14 | beforeMount() {
15 | window.addEventListener('resize', this.$_resizeHandler)
16 | },
17 | beforeDestroy() {
18 | window.removeEventListener('resize', this.$_resizeHandler)
19 | },
20 | mounted() {
21 | const isMobile = this.$_isMobile()
22 | if (isMobile) {
23 | store.dispatch('app/toggleDevice', 'mobile')
24 | store.dispatch('app/closeSideBar', { withoutAnimation: true })
25 | }
26 | },
27 | methods: {
28 | // use $_ for mixins properties
29 | // https://vuejs.org/v2/style-guide/index.html#Private-property-names-essential
30 | $_isMobile() {
31 | const rect = body.getBoundingClientRect()
32 | return rect.width - 1 < WIDTH
33 | },
34 | $_resizeHandler() {
35 | if (!document.hidden) {
36 | const isMobile = this.$_isMobile()
37 | store.dispatch('app/toggleDevice', isMobile ? 'mobile' : 'desktop')
38 |
39 | if (isMobile) {
40 | store.dispatch('app/closeSideBar', { withoutAnimation: true })
41 | }
42 | }
43 | }
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/internal/worker/fastcgi.go:
--------------------------------------------------------------------------------
1 | package worker
2 |
3 | import (
4 | "bytes"
5 | "github.com/changba/nsqproxy/config"
6 | "github.com/changba/nsqproxy/internal/module/fastcgi"
7 | "github.com/nsqio/go-nsq"
8 | "io/ioutil"
9 | )
10 |
11 | type FastCGIWorker struct {
12 | workerConfig workerConfig
13 | }
14 |
15 | func (w *FastCGIWorker) new(workerConfig workerConfig) {
16 | w.workerConfig = workerConfig
17 | }
18 |
19 | func (w *FastCGIWorker) Send(message *nsq.Message) ([]byte, error) {
20 | //连接到work
21 | fc, err := fastcgi.DialTimeout("tcp", w.workerConfig.addr, w.workerConfig.timeoutDial, w.workerConfig.timeoutWrite, w.workerConfig.timeoutRead)
22 | if err != nil {
23 | return nil, newWorkerErrorConnect(err)
24 | }
25 | defer fc.Close()
26 | //fpm参数，php可以用$_SERVER获取
27 | server := w.getServer(w.workerConfig, message)
28 | //给work发送数据
29 | rd := bytes.NewReader(message.Body)
30 | resp, err := fc.Post(server, "", rd, rd.Len())
31 | if err != nil {
32 | return nil, newWorkerErrorWrite(err)
33 | }
34 | content, err := ioutil.ReadAll(resp.Body)
35 | if err != nil {
36 | return nil, newWorkerErrorRead(err)
37 | }
38 | return content, nil
39 | }
40 |
41 | func (w *FastCGIWorker) getServer(wc workerConfig, message *nsq.Message) map[string]string {
42 | //fpm参数，php可以用$_SERVER获取
43 | server := make(map[string]string)
44 | server["SCRIPT_FILENAME"] = wc.extra
45 | server["REMOTE_ADDR"] = config.SystemConfig.InternalIP
46 | server["MESSAGE_ID"] = string(message.ID[:])
47 | return server
48 | }
49 |
--------------------------------------------------------------------------------
/internal/module/tool/httpsyncpool.go:
--------------------------------------------------------------------------------
1 | package tool
2 |
3 | import (
4 | "errors"
5 | "net"
6 | "net/http"
7 | "sync"
8 | "time"
9 | )
10 |
11 | //用sync.pool管理http client
12 | //用sync.Pool和不用，QPS提升10%，CPU下降50%
13 | type HttpClientPool struct {
14 | pool sync.Pool
15 | }
16 |
17 | func NewHttpClientPool() *HttpClientPool {
18 | return &HttpClientPool{
19 | pool: sync.Pool{
20 | New: func() interface{} {
21 | return NewHttpClient()
22 | },
23 | },
24 | }
25 | }
26 |
27 | func (h *HttpClientPool) GetClient() *http.Client {
28 | return h.pool.Get().(*http.Client)
29 | }
30 |
31 | func (h *HttpClientPool) PutClient(client *http.Client) {
32 | h.pool.Put(client)
33 | }
34 |
35 | func (h *HttpClientPool) Dial(req *http.Request) (*http.Response, error) {
36 | client := h.GetClient()
37 | if client == nil {
38 | return nil, errors.New("HttpClientPool.GetClient is nil")
39 | }
40 | defer h.PutClient(client)
41 | return client.Do(req)
42 | }
43 |
44 | //创建一个http client
45 | func NewHttpClient() *http.Client {
46 | //参数是复制的DefaultClient，只是改了MaxIdleConns和MaxIdleConnsPerHost
47 | return &http.Client{
48 | Transport: &http.Transport{
49 | Proxy: http.ProxyFromEnvironment,
50 | DialContext: (&net.Dialer{
51 | Timeout: 30 * time.Second,
52 | KeepAlive: 30 * time.Second,
53 | DualStack: true,
54 | }).DialContext,
55 | MaxIdleConns: 500,
56 | IdleConnTimeout: 90 * time.Second,
57 | TLSHandshakeTimeout: 10 * time.Second,
58 | ExpectContinueTimeout: 1 * time.Second,
59 | MaxIdleConnsPerHost: 500,
60 | },
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/internal/worker/http.go:
--------------------------------------------------------------------------------
1 | package worker
2 |
3 | import (
4 | "errors"
5 | "github.com/changba/nsqproxy/internal/module/tool"
6 | "github.com/nsqio/go-nsq"
7 | "io/ioutil"
8 | "net/http"
9 | "strings"
10 | )
11 |
12 | type HTTPWorker struct {
13 | workerConfig workerConfig
14 | clientPool *tool.HttpClientPool
15 | }
16 |
17 | func (w *HTTPWorker) new(wc workerConfig) {
18 | w.workerConfig = wc
19 | w.clientPool = tool.NewHttpClientPool()
20 | }
21 |
22 | //给HTTP发消息
23 | func (w *HTTPWorker) Send(message *nsq.Message) ([]byte, error) {
24 | //构造HTTP请求
25 | //values := url.Values{}
26 | //values.Set("param", string(message.Body))
27 | req, err := http.NewRequest("POST", "http://"+w.workerConfig.addr+"/"+w.workerConfig.extra, strings.NewReader(string(message.Body)))
28 | if err != nil {
29 | return nil, err
30 | }
31 | //含下划线会被nginx抛弃，横线会被转为下划线。
32 | req.Header.Set("MESSAGE_ID", string(message.ID[:]))
33 | req.Header.Set("MESSAGE-ID", string(message.ID[:]))
34 | req.Header.Set("CONTENT-TYPE", "application/x-www-form-urlencoded")
35 | //获取http.Client
36 | client := w.clientPool.GetClient()
37 | if client == nil {
38 | return nil, errors.New("HttpClientPool.GetClient is nil")
39 | }
40 | defer w.clientPool.PutClient(client)
41 | client.Timeout = w.workerConfig.timeoutDial
42 | //发送请求
43 | resp, err := client.Do(req)
44 | if err != nil {
45 | return nil, newWorkerErrorWrite(err)
46 | }
47 | defer resp.Body.Close()
48 | content, err := ioutil.ReadAll(resp.Body)
49 | if err != nil {
50 | return nil, newWorkerErrorRead(err)
51 | }
52 | return content, nil
53 | }
54 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | GO111MODULE=on
2 |
3 | filename = nsqproxy
4 | nowDate = $(shell date +"%Y%m%d%H%M%S")
5 |
6 | .PHONY: build
7 | build:
8 | @echo "Build..."
9 | mkdir -p bin
10 | CGO_ENABLED=0 go build -o bin/$(filename) cmd/nsqproxy.go
11 |
12 | .PHONY: build-linux
13 | build-linux:
14 | @echo "Build for linux..."
15 | mkdir -p bin
16 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/$(filename)-linux-$(nowDate) cmd/nsqproxy.go
17 |
18 | .PHONY: build-all
19 | build-all:
20 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/$(filename)-linux-amd64-$(nowDate) cmd/nsqproxy.go
21 | CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o bin/$(filename)-darwin-amd64-$(nowDate) cmd/nsqproxy.go
22 | CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o bin/$(filename)-windows-amd64-$(nowDate) cmd/nsqproxy.go
23 |
24 | .PHONY: clean
25 | clean:
26 | rm -rf bin/nsqproxy*
27 |
28 | .PHONY: kill
29 | kill:
30 | -killall nsqproxy
31 |
32 | .PHONY: test
33 | test:
34 | go test ./...
35 |
36 | .PHONY: run
37 | run: build kill
38 | nohup ./bin/nsqproxy &
39 |
40 | .PHONY: statik
41 | statik:
42 | go get github.com/rakyll/statik
43 | go generate ./...
44 |
45 | .PHONY: vue-install
46 | vue-install:
47 | cd web/vue-admin && npm install
48 |
49 | .PHONY: vue-install-taobao
50 | vue-install-taobao:
51 | cd web/vue-admin && npm install --registry=https://registry.npm.taobao.org
52 |
53 | .PHONY: vue-build
54 | vue-build:
55 | cd web/vue-admin && npm run build:prod
56 | mkdir -p web/public && cp -r web/vue-admin/dist/* web/public/
57 |
58 | .PHONY: vue-dev
59 | vue-dev:
60 | cd web/vue-admin && yarn run dev
--------------------------------------------------------------------------------
/internal/worker/error.go:
--------------------------------------------------------------------------------
1 | package worker
2 |
3 | //连接错误号
4 | const errorConnect = 1
5 |
6 | //写入错误号
7 | const errorWrite = 2
8 |
9 | //读取
10 | const errorRead = 3
11 |
12 | //其他
13 | const errorOther = 0
14 |
15 | //连接错误
16 | type workerErrorConnect struct {
17 | err error
18 | }
19 |
20 | func newWorkerErrorConnect(err error) error {
21 | return workerErrorConnect{
22 | err: err,
23 | }
24 | }
25 |
26 | func (e workerErrorConnect) Error() string {
27 | return e.err.Error()
28 | }
29 |
30 | //写错误
31 | type workerErrorWrite struct {
32 | err error
33 | }
34 |
35 | func newWorkerErrorWrite(err error) error {
36 | return workerErrorWrite{
37 | err: err,
38 | }
39 | }
40 |
41 | func (e workerErrorWrite) Error() string {
42 | return e.err.Error()
43 | }
44 |
45 | //读错误
46 | type workerErrorRead struct {
47 | Err error
48 | }
49 |
50 | func newWorkerErrorRead(err error) error {
51 | return workerErrorRead{
52 | Err: err,
53 | }
54 | }
55 |
56 | func (e workerErrorRead) Error() string {
57 | return e.Err.Error()
58 | }
59 |
60 | func getWorkerType(err error) int {
61 | switch err.(type) {
62 | case workerErrorConnect:
63 | return errorConnect
64 | case workerErrorWrite:
65 | return errorWrite
66 | case workerErrorRead:
67 | return errorRead
68 | }
69 | return errorOther
70 | }
71 |
72 | func IsErrorConnect(err error) bool {
73 | return getWorkerType(err) == errorConnect
74 | }
75 |
76 | func IsErrorWrite(err error) bool {
77 | return getWorkerType(err) == errorWrite
78 | }
79 |
80 | func IsErrorRead(err error) bool {
81 | return getWorkerType(err) == errorRead
82 | }
83 |
--------------------------------------------------------------------------------
/internal/worker/worker.go:
--------------------------------------------------------------------------------
1 | package worker
2 |
3 | import (
4 | "errors"
5 | "github.com/nsqio/go-nsq"
6 | "strings"
7 | "time"
8 | )
9 |
10 | //和Worker的通信协议
11 | //HTTP协议
12 | const protocolHttp = "http"
13 |
14 | //FastCGI协议
15 | const protocolFastCGI = "fastcgi"
16 |
17 | //自定义CBNSQ协议（唱吧NSQ）
18 | const protocolCBNSQ = "cbnsq"
19 |
20 | type workerConfig struct {
21 | addr string
22 | protocol string //小写
23 | extra string
24 | timeoutDial time.Duration
25 | timeoutWrite time.Duration
26 | timeoutRead time.Duration
27 | }
28 |
29 | func newWorkerConfig(addr, protocol, extra string, timeoutDial, timeoutWrite, timeoutRead time.Duration) workerConfig {
30 | return workerConfig{
31 | addr: addr,
32 | protocol: strings.ToLower(protocol),
33 | extra: extra,
34 | timeoutDial: timeoutDial,
35 | timeoutWrite: timeoutWrite,
36 | timeoutRead: timeoutRead,
37 | }
38 | }
39 |
40 | //worker接口
41 | type Worker interface {
42 | new(workerConfig)
43 | Send(*nsq.Message) ([]byte, error)
44 | }
45 |
46 | func NewWorker(addr, protocol, extra string, timeoutDial, timeoutWrite, timeoutRead time.Duration) (Worker, error) {
47 | wc := newWorkerConfig(addr, protocol, extra, timeoutDial, timeoutWrite, timeoutRead)
48 | var handler Worker
49 | switch wc.protocol {
50 | case protocolHttp:
51 | handler = &HTTPWorker{}
52 | case protocolFastCGI:
53 | handler = &FastCGIWorker{}
54 | case protocolCBNSQ:
55 | handler = &CBNSQWorker{}
56 | default:
57 | return nil, errors.New("worker invalid protocol")
58 | }
59 | handler.new(wc)
60 | return handler, nil
61 | }
62 |
--------------------------------------------------------------------------------
/web/vue-admin/src/components/SvgIcon/index.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
6 |
7 |
8 |
47 |
48 |
63 |
--------------------------------------------------------------------------------
/document/api/README.md:
--------------------------------------------------------------------------------
1 | # API
2 |
3 | ## 通用响应值
4 |
5 | |返回字段|字段类型|说明|
6 | |:----- |:------|:-----------------------------|
7 | |code |int |成功为200，其他为失败|
8 | |msg |string |成功为ok，失败时为原因|
9 | |result |string |响应正文|
10 |
11 | ## 服务相关
12 |
13 | * 查看状态：[service/status](service/status.md)
14 | * 查看角色：[service/getrole](service/getrole.md)
15 | * pprof：[debug/pprof](debug/pprof.md)
16 |
17 | ## 后台相关
18 |
19 | ### 服务器管理
20 |
21 | * 新增：[admin/api/workServer/create](admin/api/workserver/create.md)
22 | * 修改：[admin/api/workServer/update](admin/api/workserver/update.md)
23 | * 删除一个：[admin/api/workServer/delete](admin/api/workserver/delete.md)
24 | * 查询一个：[admin/api/workServer/get](admin/api/workserver/get.md)
25 | * 查询所有：[admin/api/workServer/page](admin/api/workserver/page.md)
26 |
27 | ### 消费者管理
28 |
29 | * 新增：[admin/api/consumeConfig/create](admin/api/consumeconfig/create.md)
30 | * 修改：[admin/api/consumeConfig/update](admin/api/consumeconfig/update.md)
31 | * 删除一个：[admin/api/consumeConfig/delete](admin/api/consumeconfig/delete.md)
32 | * 查询一个：[admin/api/consumeConfig/get](admin/api/consumeconfig/get.md)
33 | * 查询所有：[admin/api/consumeConfig/page](admin/api/consumeconfig/page.md)
34 | * 查询一个消费者关联服务器：[admin/api/consumeConfig/workList](admin/api/consumeConfig/worklist.md)
35 |
36 | ### 消费者和服务器关联关系管理
37 |
38 | * 新增：[admin/api/consumeServerMap/create](admin/api/consumeservermap/create.md)
39 | * 修改：[admin/api/consumeServerMap/update](admin/api/consumeservermap/update.md)
40 | * 删除一个：[admin/api/consumeServerMap/delete](admin/api/consumeservermap/delete.md)
41 | * 查询一个（含关联服务器信息）：[admin/api/consumeServerMap/getWork](admin/api/consumeservermap/getWork.md)
--------------------------------------------------------------------------------
/document/api/admin/api/consumeconfig/page.md:
--------------------------------------------------------------------------------
1 | ### 批量查询消费者
2 |
3 | - 接口功能
4 |
5 | > 批量查询消费者
6 |
7 | ```
8 | GET /admin/api/consumeConfig/page
9 | ```
10 |
11 | - 请求参数
12 |
13 | |参数|必选|类型|说明|
14 | |:----- |:-------|:-----|----- |
15 | |topic |false |string |Topic名，模糊查询，%topic%的方式查询 |
16 | |page |false |int |分页，默认1，小于等于零时表示第一页，一页20条 |
17 |
18 | - 返回结果
19 |
20 | ```
21 | {
22 | code: 200,
23 | msg: "ok",
24 | result: [
25 | {
26 | id: 1,
27 | topic: "test_topic_1",
28 | channel: "one",
29 | description: "描述",
30 | owner: "责任人",
31 | monitorThreshold: 0,
32 | handleNum: 6,
33 | maxInFlight: 6,
34 | isRequeue: false,
35 | timeoutDial: 3590,
36 | timeoutRead: 3590,
37 | timeoutWrite: 3590,
38 | invalid: 0,
39 | createdAt: "2020-08-20T15:08:39+08:00",
40 | updatedAt: "0001-01-01T00:00:00Z",
41 | serverList: null
42 | },
43 | {
44 | id: 2,
45 | topic: "test_topic_2",
46 | channel: "one",
47 | description: "描述",
48 | owner: "责任人",
49 | monitorThreshold: 0,
50 | handleNum: 2,
51 | maxInFlight: 2,
52 | isRequeue: false,
53 | timeoutDial: 3590,
54 | timeoutRead: 3590,
55 | timeoutWrite: 3590,
56 | invalid: 0,
57 | createdAt: "2020-08-20T15:08:39+08:00",
58 | updatedAt: "0001-01-01T00:00:00Z",
59 | serverList: null
60 | }
61 | ]
62 | }
63 | ```
--------------------------------------------------------------------------------
/web/vue-admin/src/utils/request.js:
--------------------------------------------------------------------------------
1 | import axios from 'axios'
2 | import { Message } from 'element-ui'
3 | import qs from 'qs'
4 |
5 | // 创建axios实例
6 | const service = axios.create({
7 | timeout: 30000,
8 | headers: {
9 | 'X-Requested-With': 'XMLHttpRequest'
10 | },
11 | transformRequest: [function(data, headers) {
12 | if (data instanceof FormData) {
13 | return data
14 | }
15 |
16 | if (data?.__json__ === true) {
17 | headers['Content-Type'] = 'application/json; charset=utf-8'
18 | data.__json__ = undefined
19 |
20 | return JSON.stringify(data)
21 | }
22 | return qs.stringify(data)
23 | }],
24 | data: {
25 | _timestamp: new Date()
26 | }
27 | })
28 |
29 | service.interceptors.request.use(config => {
30 | if (config.method === 'get') {
31 | config.params = config.data
32 | }
33 |
34 | return config
35 | }, error => {
36 | // Do something with request error
37 | console.log(error) // for debug
38 | Promise.reject(error)
39 | })
40 |
41 | service.interceptors.response.use(
42 | response => {
43 | const res = response.data
44 | if (res.code !== 200) {
45 | Message({
46 | message: res.msg || 'Error',
47 | type: 'error',
48 | duration: 3 * 1000
49 | })
50 | return Promise.reject(new Error(res.msg || 'Error'))
51 | } else {
52 | return res
53 | }
54 | },
55 | error => {
56 | console.log('err' + error) // for debug
57 | Message({
58 | message: error.message,
59 | type: 'error',
60 | duration: 5 * 1000
61 | })
62 | return Promise.reject(error)
63 | }
64 | )
65 |
66 | export default service
67 |
--------------------------------------------------------------------------------
/web/vue-admin/src/layout/components/Sidebar/index.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
5 |
15 |
17 |
18 |
19 |
20 |
21 |
57 |
--------------------------------------------------------------------------------
/web/vue-admin/src/views/tree/index.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
15 |
16 |
17 |
78 |
79 |
--------------------------------------------------------------------------------
/internal/httper/router.go:
--------------------------------------------------------------------------------
1 | package httper
2 |
3 | import (
4 | "github.com/changba/nsqproxy/config"
5 | "net/http"
6 | )
7 |
8 | // 启动HTTP
9 | func (h *Httper) router() {
10 | //获取状态
11 | http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
12 | Success(w, "ok")
13 | })
14 | //获取角色
15 | http.HandleFunc("/getRole", func(w http.ResponseWriter, r *http.Request) {
16 | Success(w, config.SystemConfig.Role)
17 | })
18 |
19 | //后台
20 | http.Handle("/admin/", http.StripPrefix("/admin/", http.FileServer(h.statikFS)))
21 | //后台接口
22 | // 消费者管理
23 | consumeConfig := ConsumeConfig{}
24 | http.HandleFunc("/admin/api/consumeConfig/create", consumeConfig.Create)
25 | http.HandleFunc("/admin/api/consumeConfig/update", consumeConfig.Update)
26 | http.HandleFunc("/admin/api/consumeConfig/delete", consumeConfig.Delete)
27 | http.HandleFunc("/admin/api/consumeConfig/page", consumeConfig.Page)
28 | http.HandleFunc("/admin/api/consumeConfig/get", consumeConfig.Get)
29 | http.HandleFunc("/admin/api/consumeConfig/workList", consumeConfig.WorkList)
30 | //Worker机管理
31 | workServer := WorkServer{}
32 | http.HandleFunc("/admin/api/workServer/create", workServer.Create)
33 | http.HandleFunc("/admin/api/workServer/update", workServer.Update)
34 | http.HandleFunc("/admin/api/workServer/delete", workServer.Delete)
35 | http.HandleFunc("/admin/api/workServer/page", workServer.Page)
36 | http.HandleFunc("/admin/api/workServer/all", workServer.All)
37 | http.HandleFunc("/admin/api/workServer/get", workServer.Get)
38 | //消费者和Worker机关联关系管理
39 | consumeServerMap := ConsumeServerMap{}
40 | http.HandleFunc("/admin/api/consumeServerMap/create", consumeServerMap.Create)
41 | http.HandleFunc("/admin/api/consumeServerMap/update", consumeServerMap.Update)
42 | http.HandleFunc("/admin/api/consumeServerMap/delete", consumeServerMap.Delete)
43 | http.HandleFunc("/admin/api/consumeServerMap/getWork", consumeServerMap.GetWork)
44 | }
45 |
--------------------------------------------------------------------------------
/cmd/nsqproxy.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | //go:generate echo "statik -src=../web/public/ -dest=../internal -f"
4 | //go:generate statik -src=../web/public/ -dest=../internal -f
5 |
6 | import (
7 | "github.com/changba/nsqproxy/config"
8 | "github.com/changba/nsqproxy/internal/backup"
9 | "github.com/changba/nsqproxy/internal/httper"
10 | "github.com/changba/nsqproxy/internal/model"
11 | "github.com/changba/nsqproxy/internal/module/logger"
12 | "github.com/changba/nsqproxy/internal/module/tool"
13 | "github.com/changba/nsqproxy/internal/proxy"
14 | "os"
15 | "os/signal"
16 | "syscall"
17 | "time"
18 | )
19 |
20 | // 主函数
21 | func main() {
22 | //初始化系统配置
23 | config.NewSystemConfig()
24 | model.NewDB(config.SystemConfig.DbHost, config.SystemConfig.DbPort, config.SystemConfig.DbUsername, config.SystemConfig.DbPassword, config.SystemConfig.DbName)
25 | //创建一个proxy实例
26 | p := proxy.NewProxy()
27 | //异常捕获
28 | defer func() {
29 | tool.PanicHandlerForLog()
30 | logger.Fatalf("nsqproxy will exit")
31 | os.Exit(2)
32 | }()
33 | //开启HTTP
34 | httper.NewHttper(config.SystemConfig.HttpAddr).Run()
35 | //灾备
36 | backup.Backup(config.SystemConfig.MasterAddr)
37 | //启动一个proxy实例
38 | logger.Infof("nsqproxy is starting")
39 | p.Run()
40 | //监听信号
41 | listenSignal(p)
42 | logger.Infof("nsqproxy end success")
43 | }
44 |
45 | // 监听信号
46 | func listenSignal(p *proxy.Proxy) {
47 | sigChannel := make(chan os.Signal)
48 | signal.Notify(sigChannel, syscall.SIGINT, syscall.SIGTERM, syscall.SIGTRAP)
49 | for {
50 | sig := <-sigChannel
51 | logger.Infof("nsqproxy receive signal: %s", sig.String())
52 | if sig == syscall.SIGTRAP {
53 | continue
54 | }
55 | logger.Infof("nsqproxy is closing consumes...")
56 | p.SetExitFlag()
57 | p.Stop()
58 | //等待10秒
59 | logger.Infof("nsqproxy will be closed master process ten seconds later.")
60 |
61 | time.Sleep(10)
62 | //time.Sleep(10 * time.Second)
63 | break
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/web/vue-admin/src/utils/scroll-to.js:
--------------------------------------------------------------------------------
1 | Math.easeInOutQuad = function(t, b, c, d) {
2 | t /= d / 2
3 | if (t < 1) {
4 | return c / 2 * t * t + b
5 | }
6 | t--
7 | return -c / 2 * (t * (t - 2) - 1) + b
8 | }
9 |
10 | // requestAnimationFrame for Smart Animating http://goo.gl/sx5sts
11 | var requestAnimFrame = (function() {
12 | return window.requestAnimationFrame || window.webkitRequestAnimationFrame || window.mozRequestAnimationFrame || function(callback) { window.setTimeout(callback, 1000 / 60) }
13 | })()
14 |
15 | /**
16 | * Because it's so fucking difficult to detect the scrolling element, just move them all
17 | * @param {number} amount
18 | */
19 | function move(amount) {
20 | document.documentElement.scrollTop = amount
21 | document.body.parentNode.scrollTop = amount
22 | document.body.scrollTop = amount
23 | }
24 |
25 | function position() {
26 | return document.documentElement.scrollTop || document.body.parentNode.scrollTop || document.body.scrollTop
27 | }
28 |
29 | /**
30 | * @param {number} to
31 | * @param {number} duration
32 | * @param {Function} callback
33 | */
34 | export function scrollTo(to, duration, callback) {
35 | const start = position()
36 | const change = to - start
37 | const increment = 20
38 | let currentTime = 0
39 | duration = (typeof (duration) === 'undefined') ? 500 : duration
40 | var animateScroll = function() {
41 | // increment the time
42 | currentTime += increment
43 | // find the value with the quadratic in-out easing function
44 | var val = Math.easeInOutQuad(currentTime, start, change, duration)
45 | // move the document.body
46 | move(val)
47 | // do the animation unless its over
48 | if (currentTime < duration) {
49 | requestAnimFrame(animateScroll)
50 | } else {
51 | if (callback && typeof (callback) === 'function') {
52 | // the animation is done so lets callback
53 | callback()
54 | }
55 | }
56 | }
57 | animateScroll()
58 | }
59 |
--------------------------------------------------------------------------------
/web/vue-admin/src/router/index.js:
--------------------------------------------------------------------------------
1 | import Vue from 'vue'
2 | import Router from 'vue-router'
3 |
4 | Vue.use(Router)
5 |
6 | import Layout from '@/layout'
7 |
8 | export const constantRoutes = [
9 | {
10 | path: '/login',
11 | component: () => import('@/views/login/index'),
12 | hidden: true
13 | },
14 |
15 | {
16 | path: '/404',
17 | component: () => import('@/views/404'),
18 | hidden: true
19 | },
20 |
21 | {
22 | path: '/',
23 | component: Layout,
24 | redirect: '/consumeconfig',
25 | name: 'NSQ消费者配置',
26 | meta: { title: 'NSQ消费者配置', icon: 'form' },
27 | children: [
28 | {
29 | path: '/consumeconfig',
30 | name: 'NSQ消费者配置',
31 | component: () => import('@/views/consumeconfig/index'),
32 | meta: { title: 'NSQ消费者配置', icon: 'form' }
33 | },
34 | {
35 | path: '/consumeServerMap/:id',
36 | name: '消费者的work机权重',
37 | component: () => import('@/views/consumeconfig/weight'),
38 | hidden: true,
39 | meta: { title: '消费者的work机权重', icon: 'user' }
40 | }
41 | ]
42 | },
43 |
44 | {
45 | path: '/workerserver',
46 | component: Layout,
47 | children: [{
48 | path: '/workerserver',
49 | name: 'NSQ消费者服务器列表',
50 | component: () => import('@/views/workerserver/index'),
51 | meta: { title: 'NSQ消费者服务器列表', icon: 'table' }
52 | }]
53 | },
54 | // 404 page must be placed at the end !!!
55 | { path: '*', redirect: '/404', hidden: true }
56 | ]
57 |
58 | const createRouter = () => new Router({
59 | // mode: 'history', // require service support
60 | scrollBehavior: () => ({ y: 0 }),
61 | routes: constantRoutes
62 | })
63 |
64 | const router = createRouter()
65 |
66 | // Detail see: https://github.com/vuejs/vue-router/issues/1234#issuecomment-357941465
67 | export function resetRouter() {
68 | const newRouter = createRouter()
69 | router.matcher = newRouter.matcher // reset router
70 | }
71 |
72 | export default router
73 |
--------------------------------------------------------------------------------
/internal/worker/worker_test.go:
--------------------------------------------------------------------------------
1 | package worker
2 |
3 | import (
4 | "errors"
5 | "github.com/nsqio/go-nsq"
6 | "testing"
7 | "time"
8 | )
9 |
10 | func TestNewWorker_IsError(t *testing.T) {
11 | if !IsErrorConnect(newWorkerErrorConnect(errors.New("hello world"))) {
12 | t.Fatal("IsErrorConnect failed")
13 | }
14 | if !IsErrorWrite(newWorkerErrorWrite(errors.New("hello world"))) {
15 | t.Fatal("IsErrorWrite failed")
16 | }
17 | if !IsErrorRead(newWorkerErrorRead(errors.New("hello world"))) {
18 | t.Fatal("IsErrorRead failed")
19 | }
20 | }
21 |
22 | func TestNewWorker(t *testing.T) {
23 | //构造消息
24 | messageId := nsq.MessageID([16]byte{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p'})
25 | body := []byte("Hello world")
26 | message := nsq.NewMessage(messageId, body)
27 |
28 | //HTTP
29 | handler, err := NewWorker("127.0.0.1:80", "HtTp", "index.php", 1*time.Second, 1*time.Second, 1*time.Second)
30 | if err != nil {
31 | t.Fatalf("http NewWorker error: %s", err.Error())
32 | }
33 | res, err := handler.Send(message)
34 | if err != nil {
35 | t.Fatalf("http send error: %s", err.Error())
36 | }
37 | if string(res) != "200 ok" {
38 | t.Fatalf("http response body is not match")
39 | }
40 |
41 | //FastCGI
42 | handler, err = NewWorker("127.0.0.1:9000", "FaStCgI", "/var/www/index.php", 1*time.Second, 1*time.Second, 1*time.Second)
43 | if err != nil {
44 | t.Fatalf("fastcgi NewWorker error: %s", err.Error())
45 | }
46 | res, err = handler.Send(message)
47 | if err != nil {
48 | t.Fatalf("fastcgi send error: %s", err.Error())
49 | }
50 | if string(res) != "200 ok" {
51 | t.Fatalf("fastcgi response body is not match")
52 | }
53 |
54 | //CBNSQ
55 | handler, err = NewWorker("127.0.0.1:19910", "CbNsQ", "", 1*time.Second, 1*time.Second, 1*time.Second)
56 | if err != nil {
57 | t.Fatalf("cbnsq NewWorker error: %s", err.Error())
58 | }
59 | res, err = handler.Send(message)
60 | if err != nil {
61 | t.Fatalf("cbnsq send error: %s", err.Error())
62 | }
63 | if string(res) != "200 ok" {
64 | t.Fatalf("cbnsq response body is not match")
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/web/vue-admin/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "vue-admin-template",
3 | "version": "4.4.0",
4 | "description": "A vue admin template with Element UI & axios & iconfont & permission control & lint",
5 | "author": "Pan
3 |
4 |
5 | ![]()
6 |
8 |
9 | ![]()
10 |
12 |
13 |
14 |
15 |
16 |
33 |
34 |
83 |
--------------------------------------------------------------------------------
/document/protocol/cbnsq.md:
--------------------------------------------------------------------------------
1 | ## CBNSQ 协议
2 |
3 | - 协议描述
4 |
5 | > 自定义的基于TCP的文本协议。
6 |
7 | - CBNSQ协议是自定义的非常简单的一个基于文本的TCP的协议。
8 | - 就是消息长度 + 消息ID + 消息内容。 msg的长度(8字节，0填充） + messageId（16字节） + msg正文
9 | - 如：有个用户注册的消息是：{"topic":"test","classname":"UserService","methodname":"addUser","param":["userid", "username", "password"],"addtime":"2020-11-27 14:30:34"}
10 | - 第一部分：消息长度。消息主体是个json，长度140，补齐8位，那么这部分为00000140。8位的极限是99999999/1024/1024≈95M，足够了吧。
11 | - 第二部分：消息ID，这个是NSQ的消息唯一ID，16位。如qwertyuiopasdfgh。注意：这16位是不计入第一部分的消息长度的。
12 | - 第三部分：消息主体。即刚才提到的JSON串。
13 | - 完整的消息为：00000140qwertyuiopasdfgh{"topic":"test","classname":"UserService","methodname":"addUser","param":["userid", "username", "password"],"addtime":"2020-11-27 14:30:34"}
14 |
15 | ```
16 | 消息ID：消息体（从0开始）的第8位到第23位。
17 | 消息正文：消息体（从0开始）的第24位到[消息长度]，[消息长度]为消息体的第0位到第7位。
18 | ```
19 |
20 | ### PHP使用示例
21 |
22 | ```php
23 | childProcessCount = 10;
52 |
53 | //设置MeepoPS实例名称
54 | $cbNsq->instanceName = 'MeepoPS-CBSNQ';
55 |
56 | //设置回调函数 - 这是所有应用的业务代码入口
57 | $cbNsq->callbackNewData = 'callbackNewData';
58 |
59 | //启动MeepoPS
60 | \MeepoPS\runMeepoPS();
61 |
62 |
63 | //以下为回调函数, 业务相关.
64 | //回调 - 收到新消息
65 | function callbackNewData($connect, $data)
66 | {
67 | file_put_contents('../cbnsq.log', date('Y-m-d H:i:s') . ' ' . $_SERVER['MESSAGE_ID'] . ' ' . $data ."\n", FILE_APPEND);
68 | $connect->send("200 ok");
69 | }
70 | ```
--------------------------------------------------------------------------------
/web/vue-admin/src/permission.js:
--------------------------------------------------------------------------------
1 | import router from './router'
2 | // import store from './store'
3 | // import { Message } from 'element-ui'
4 | import NProgress from 'nprogress' // progress bar
5 | import 'nprogress/nprogress.css' // progress bar style
6 | // import { getToken } from '@/utils/auth' // get token from cookie
7 | // import getPageTitle from '@/utils/get-page-title'
8 |
9 | NProgress.configure({ showSpinner: false }) // NProgress Configuration
10 |
11 | // const whiteList = ['/login'] // no redirect whitelist
12 |
13 | // router.beforeEach(async(to, from, next) => {
14 | // // start progress bar
15 | // NProgress.start()
16 |
17 | // // set page title
18 | // document.title = getPageTitle(to.meta.title)
19 |
20 | // // determine whether the user has logged in
21 | // const hasToken = getToken()
22 |
23 | // if (hasToken) {
24 | // if (to.path === '/login') {
25 | // // if is logged in, redirect to the home page
26 | // next({ path: '/' })
27 | // NProgress.done()
28 | // } else {
29 | // const hasGetUserInfo = store.getters.name
30 | // if (hasGetUserInfo) {
31 | // next()
32 | // } else {
33 | // try {
34 | // // get user info
35 | // await store.dispatch('user/getInfo')
36 |
37 | // next()
38 | // } catch (error) {
39 | // // remove token and go to login page to re-login
40 | // await store.dispatch('user/resetToken')
41 | // Message.error(error || 'Has Error')
42 | // next(`/login?redirect=${to.path}`)
43 | // NProgress.done()
44 | // }
45 | // }
46 | // }
47 | // } else {
48 | // /* has no token*/
49 |
50 | // if (whiteList.indexOf(to.path) !== -1) {
51 | // // in the free login whitelist, go directly
52 | // next()
53 | // } else {
54 | // // other pages that do not have permission to access are redirected to the login page.
55 | // next(`/login?redirect=${to.path}`)
56 | // NProgress.done()
57 | // }
58 | // }
59 | // })
60 |
61 | router.afterEach(() => {
62 | // finish progress bar
63 | NProgress.done()
64 | })
65 |
--------------------------------------------------------------------------------
/web/vue-admin/src/views/table/index.vue:
--------------------------------------------------------------------------------
1 |
2 | {{ title }}
7 |{{ title }}
11 |
3 |
11 |
12 |
13 | {{ scope.row.Id }}
14 |
15 |
16 |
17 |
18 | {{ scope.row.CreatedAt }}
19 |
20 |
21 |
22 |
23 | {{ scope.row.author }}
24 |
25 |
26 |
27 |
28 | {{ scope.row.pageviews }}
29 |
30 |
31 |
32 |
33 | {{ scope.row.status }}
34 |
35 |
36 |
37 |
38 |
39 | {{ scope.row.display_time }}
40 |
41 |
42 |
43 |
44 |
45 |
46 |
72 |
--------------------------------------------------------------------------------
/internal/proxy/loadbalance_test.go:
--------------------------------------------------------------------------------
1 | package proxy
2 |
3 | import (
4 | "github.com/changba/nsqproxy/internal/model"
5 | "testing"
6 | )
7 |
8 | func init() {
9 | model.NewDB("0.0.0.0", "3306", "root", "", "nsqproxy")
10 | }
11 |
12 | func TestNewLoadBalance(t *testing.T) {
13 | p := NewProxy()
14 | if len(p.consumeConfigList) <= 0 || len(p.consumeConfigList[0].ServerList) <= 0 {
15 | t.Fatal("consume workerlist为空")
16 | }
17 | loadbalance := newLoadBalance(LoadBalanceMethodLoop, p.consumeConfigList[0].ServerList)
18 | if loadbalance.method != LoadBalanceMethodLoop || loadbalance.dispatcher == nil {
19 | t.Fatal("newLoadBalance failed")
20 | }
21 | }
22 |
23 | func TestLoadBalance_pickWorker(t *testing.T) {
24 | p := NewProxy()
25 | loadbalance := newLoadBalance(LoadBalanceMethodLoop, p.consumeConfigList[0].ServerList)
26 | work, err := loadbalance.pickWorker()
27 | if err != nil {
28 | t.Fatalf("pickWorker error: %s", err.Error())
29 | }
30 | if work.Id <= 0 || work.Weight <= 0 || len(work.WorkServer.Addr) <= 0 || len(work.WorkServer.Protocol) <= 0 {
31 | t.Fatalf("pickWorker error: no available work")
32 | }
33 | }
34 |
35 | func TestLoadBalanceLoop_new(t *testing.T) {
36 | p := NewProxy()
37 | loop := &loadBalanceLoop{}
38 | loop.new(p.consumeConfigList[0].ServerList)
39 | if len(loop.list) <= 0 {
40 | t.Fatalf("loop.list is empty")
41 | }
42 | //检验capacity是否相等
43 | capacity := 0
44 | //检验生成的list中，每台机器的占比是否相等
45 | list := make(map[int]int, 0)
46 | for _, v := range p.consumeConfigList[0].ServerList {
47 | capacity += v.Weight * 100
48 | list[v.Id] = v.Weight * 100
49 | }
50 | if loop.capacity != capacity {
51 | t.Fatalf("loop.capacity is not match. action:%d expect:%d", loop.capacity, capacity)
52 | }
53 | count := make(map[int]int, 0)
54 | for _, v := range loop.list {
55 | count[v.Id] = v.Weight * 100
56 | }
57 | if len(count) != len(list) {
58 | t.Fatalf("loop.list length does not match. action:%d expect:%d", len(count), len(list))
59 | }
60 | for k, v := range list {
61 | if _, ok := count[k]; !ok {
62 | t.Fatalf("the id %d of loop.list not exists", k)
63 | }
64 | if v != count[k] {
65 | t.Fatalf("the id %d total of loop.list does not match. action:%d expect:%d", k, count[k], v)
66 | }
67 | }
68 | }
69 |
--------------------------------------------------------------------------------
/internal/worker/cbnsq.go:
--------------------------------------------------------------------------------
1 | package worker
2 |
3 | import (
4 | "bytes"
5 | "errors"
6 | "fmt"
7 | "github.com/nsqio/go-nsq"
8 | "net"
9 | "strconv"
10 | "time"
11 | )
12 |
13 | // CBNSQ协议是自定义的非常简单的一个基于文本的TCP的协议。
14 | // 就是消息长度 + 消息内容。 msg的长度(8字节，0填充） + messageId（16字节） + msg正文
15 | // 如：有个用户注册的消息是：{"topic":"test","classname":"UserService","methodname":"addUser","param":["userid", "username", "password"],"addtime":"2020-11-27 14:30:34"}
16 | // 第一部分：消息长度。消息主体是个json，长度140，补齐8位，那么这部分为00000140。8位的极限是99999999/1024/1024≈95M，足够了吧。
17 | // 第二部分：消息ID，这个是NSQ的消息唯一ID，16位。如qwertyuiopasdfgh。注意：这16位是不计入第一部分的消息长度的。
18 | // 第三部分：消息主体。即刚才提到的JSON串。
19 | // 完整的消息为：00000140qwertyuiopasdfgh{"topic":"test","classname":"UserService","methodname":"addUser","param":["userid", "username", "password"],"addtime":"2020-11-27 14:30:34"}
20 |
21 | type CBNSQWorker struct {
22 | workerConfig workerConfig
23 | }
24 |
25 | func (w *CBNSQWorker) new(wc workerConfig) {
26 | w.workerConfig = wc
27 | }
28 |
29 | func (w *CBNSQWorker) Send(message *nsq.Message) ([]byte, error) {
30 | //连接到worker
31 | conn, err := net.DialTimeout("tcp", w.workerConfig.addr, w.workerConfig.timeoutDial)
32 | if err != nil {
33 | return nil, newWorkerErrorConnect(err)
34 | }
35 | //设置连接的读写超时时间
36 | _ = conn.SetWriteDeadline(time.Now().Add(w.workerConfig.timeoutWrite))
37 | _ = conn.SetReadDeadline(time.Now().Add(w.workerConfig.timeoutRead))
38 | defer conn.Close()
39 | //给worker发送数据
40 | data := w.encode(message)
41 | n, err := conn.Write(data)
42 | if n == 0 {
43 | return nil, newWorkerErrorWrite(errors.New("n of conn.Write is 0"))
44 | }
45 | if err != nil {
46 | return nil, newWorkerErrorWrite(errors.New("conn.Write" + err.Error()))
47 | }
48 | //从worker读取响应
49 | buf := make([]byte, 128)
50 | n, err = conn.Read(buf)
51 | if err != nil {
52 | return nil, newWorkerErrorRead(errors.New("conn.Read" + err.Error()))
53 | }
54 | if n == 0 {
55 | return nil, newWorkerErrorRead(errors.New("response length is 0"))
56 | }
57 | return buf[:n], nil
58 | }
59 |
60 | // CBNSQ协议打包数据
61 | // msg的长度(8字节，0填充） + messageId（16字节） + msg正文
62 | func (w *CBNSQWorker) encode(message *nsq.Message) []byte {
63 | header := fmt.Sprintf("%08s", strconv.Itoa(len(message.Body)))
64 | return bytes.Join([]([]byte){[]byte(header), message.ID[:], message.Body}, []byte(""))
65 | }
66 |
--------------------------------------------------------------------------------
/web/vue-admin/src/layout/index.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
12 |
13 |
14 |
52 |
53 |
94 |
--------------------------------------------------------------------------------
/web/vue-admin/src/components/Breadcrumb/index.vue:
--------------------------------------------------------------------------------
1 |
2 |
6 |
11 |
3 |
15 |
16 |
17 |
92 |
93 |
102 |
--------------------------------------------------------------------------------
/web/vue-admin/src/icons/svg/form.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/internal/module/logger/log.go:
--------------------------------------------------------------------------------
1 | package logger
2 |
3 | //log库
4 |
5 | import (
6 | "log"
7 | "os"
8 | "strings"
9 | )
10 |
11 | type LogLevel int
12 |
13 | const (
14 | LOG_DEBUG LogLevel = iota
15 | LOG_INFO
16 | LOG_WARNING
17 | LOG_ERROR
18 | LOG_FATAL
19 | )
20 |
21 | func (ll LogLevel) String() string {
22 | switch ll {
23 | case LOG_DEBUG:
24 | return "debug"
25 | case LOG_INFO:
26 | return "info"
27 | case LOG_WARNING:
28 | return "warning"
29 | case LOG_ERROR:
30 | return "error"
31 | case LOG_FATAL:
32 | return "fatal"
33 | default:
34 | return "info"
35 | }
36 | }
37 |
38 | func getLevelByString(levelString string) LogLevel {
39 | switch strings.ToLower(levelString) {
40 | case "debug":
41 | return LOG_DEBUG
42 | case "info":
43 | return LOG_INFO
44 | case "warning":
45 | return LOG_WARNING
46 | case "error":
47 | return LOG_ERROR
48 | case "fatal":
49 | return LOG_FATAL
50 | default:
51 | return LOG_INFO
52 | }
53 | }
54 |
55 | type Logger struct {
56 | *log.Logger
57 | Level LogLevel
58 | logFile *os.File
59 | }
60 |
61 | func NewLogger(fileName string, prefix string, level string) *Logger {
62 | //目录是否存在
63 | pathSliceList := strings.Split(fileName, "/")
64 | //不是当前目录
65 | if len(pathSliceList) > 1 {
66 | pathSliceList = pathSliceList[:len(pathSliceList)-1]
67 | path := strings.Join(pathSliceList, "/")
68 | _, err := os.Stat(path)
69 | //创建目录
70 | if err != nil && os.IsNotExist(err) {
71 | err := os.Mkdir(path, os.ModePerm)
72 | if err != nil {
73 | panic("create dir failed: " + err.Error())
74 | }
75 | }
76 | }
77 | logFile, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
78 | if err != nil {
79 | panic("open log file " + fileName + " error: " + err.Error())
80 | }
81 | logger := log.New(logFile, prefix, log.LstdFlags)
82 | logger.SetFlags(log.Ldate | log.Ltime)
83 | return &Logger{
84 | Logger: logger,
85 | Level: getLevelByString(level),
86 | logFile: logFile,
87 | }
88 | }
89 |
90 | func (l *Logger) WithLevelf(lev LogLevel, format string, v ...interface{}) {
91 | if l == nil || lev < l.Level {
92 | return
93 | }
94 | format = "[" + lev.String() + "]" + " " + format
95 | l.logf(format, v...)
96 | }
97 |
98 | func (l *Logger) logf(format string, v ...interface{}) {
99 | l.Printf(format, v...)
100 | }
101 |
102 | func (l *Logger) Close() bool {
103 | _ = l.logFile.Close()
104 | return true
105 | }
106 |
--------------------------------------------------------------------------------
/document/doc/quick_start.md:
--------------------------------------------------------------------------------
1 | # 快速开始
2 |
3 | ## 简介
4 | NSQProxy是 队列NSQ 和 消费者 之间的桥梁。负责将 队列中的任务 根据管理后台的配置 通过指定的协议 分发给指定的Worker机。在唱吧内部使用2年，高效稳定的处理着每日数十亿条消息。
5 |
6 | * 队列中的任务：生产者入队到NSQ的数据。
7 | * 管理后台的配置：每个Topic/Channel都可以配置在哪台机器上消费、同时消费的并发量是几个、是否重新入队、积压多少个开始报警等。
8 | * 通过指定的协议：支持HTTP、FastCGI、CBNSQ等协议将数据发送给Worker机。
9 | * Worker机：执行消费者程序的服务器。
10 | * 数据流转：生产者 -> NSQ -> NSQProxy -> 消费者
11 |
12 | > 本文档以HTTP作为通信协议，编写一个go代码作为消费者。
13 |
14 | ## 启动依赖
15 | NSQProxy是一个中间转发器，因此需要上下游依赖。
16 |
17 | #### 依赖NSQ
18 | NSQ是队列服务，因此NSQProxy的上游是NSQ。NSQ会将任务下发给NSQProxy，站在NSQ的视角中，NSQProxy是它的消费者。
19 |
20 | * 启动NSQLookupd `nsqlookupd -broadcast-address="0.0.0.0" -http-address="0.0.0.0:4161" -tcp-address="0.0.0.0:4160"`
21 | * 启动NSQD `nsqd -broadcast-address="0.0.0.0" -lookupd-tcp-address="0.0.0.0:4160" -tcp-address="0.0.0.0:4150" -http-address="0.0.0.0:4151"`
22 |
23 | #### 依赖MySQL
24 | MySQL中存储着各Topic/Channel的配置信息，因此NSQProxy依赖于MySQL。
25 |
26 | 启动MySQL的方式多种多样，如`mysqld` 和 `service mysql start`等等。
27 |
28 | ## 下载安装
29 | 下载并启动NSQProxy。
30 |
31 | * 下载最新版本的压缩包 https://github.com/changba/nsqproxy/releases
32 | * 解压
33 | * 启动（注意替换为自己的MySQL信息） `./nsqproxy -dbHost=127.0.0.1 -dbPort=3306 -dbUsername=root -dbPassword=rootpsd -dbName=nsqproxy -logLevel=debug -nsqlookupdHTTP=127.0.0.1:4161`
34 | * 命令行 `curl http://0.0.0.0:19421/status` 输出ok
35 | * 浏览器打开 http://0.0.0.0:19421/admin
36 |
37 | ## 部署消费者
38 | 本文编写一个go代码作为消费者，这个go代码使用HTTP协议监听8888端口。
39 | ```golang
40 | package main
41 |
42 | import (
43 | "fmt"
44 | "io/ioutil"
45 | "net/http"
46 | )
47 |
48 | // 启动HTTP
49 | func main() {
50 | http.HandleFunc("/nsqTask", func(w http.ResponseWriter, r *http.Request) {
51 | fmt.Println("MessageID：" + r.Header.Get("MESSAGE_ID"))
52 | data, _ := ioutil.ReadAll(r.Body)
53 | fmt.Println("MessageBody：" + string(data))
54 | _, _ = w.Write([]byte("200 ok"))
55 | })
56 | if err := http.ListenAndServe("0.0.0.0:8888", nil); err != nil {
57 | panic("ListenAndServe error: " + err.Error())
58 | }
59 | }
60 | ```
61 |
62 | 运行 `go run test.go`
63 |
64 | ## 后台操作
65 | 把Topic和消费者关联起来。浏览器打开 http://0.0.0.0:19421/admin
66 |
67 | 1、添加Worker机
68 |
69 | 
70 |
71 | 2、添加新消费者配置
72 |
73 | 
74 |
75 | 3、把消费者和Worker机关联起来
76 |
77 | 
78 |
79 | ## 具体示例
80 |
81 | 此时，我们给NSQ入队，NSQ就会把消息推给NSQProxy，NSQProxy根据刚才的配置，就会把消息推送给0.0.0.0:8888的Golang程序。
82 |
83 | 1、入队给NSQ `curl -d 'name=xiaoming&sex=male&age=18' 'http://0.0.0.0:4151/pub?topic=test_topic'`
84 |
85 | 2、查看刚才编写的Golang程序的输出，可以拿到消息的唯一ID和消息的内容
86 |
87 | 
--------------------------------------------------------------------------------
/web/vue-admin/src/layout/components/Navbar.vue:
--------------------------------------------------------------------------------
1 |
2 |
6 |
7 |
8 |
35 |
36 |
114 |
--------------------------------------------------------------------------------
/web/vue-admin/src/views/form/index.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
7 |
8 |
9 |
12 |
13 |
14 |
15 |
17 | -
18 |
19 |
21 |
22 |
23 |
25 |
26 |
27 |
32 |
33 |
34 |
35 |
38 |
39 |
40 |
42 |
43 | Create
44 | Cancel
45 |
46 |
47 |
48 |
49 |
50 |
79 |
80 |
85 |
86 |
--------------------------------------------------------------------------------
/internal/module/tool/guid.go:
--------------------------------------------------------------------------------
1 | package tool
2 |
3 | // the core algorithm here was borrowed from:
4 | // Blake Mizerany's `noeqd` https://github.com/bmizerany/noeqd
5 | // and indirectly:
6 | // Twitter's `snowflake` https://github.com/twitter/snowflake
7 |
8 | // only minor cleanup and changes to introduce a type, combine the concept
9 | // of workerID + datacenterId into a single identifier, and modify the
10 | // behavior when sequences rollover for our specific implementation needs
11 |
12 | import (
13 | "encoding/hex"
14 | "errors"
15 | "sync"
16 | "time"
17 | )
18 |
19 | func GenerateUniqueId(node int64) [16]byte {
20 | retry:
21 | id, err := NewGUIDFactory(node).NewGUID()
22 | if err != nil {
23 | time.Sleep(time.Millisecond)
24 | goto retry
25 | }
26 | return id.Hex()
27 | }
28 |
29 | const (
30 | nodeIDBits = uint64(10)
31 | sequenceBits = uint64(12)
32 | nodeIDShift = sequenceBits
33 | timestampShift = sequenceBits + nodeIDBits
34 | sequenceMask = int64(-1) ^ (int64(-1) << sequenceBits)
35 |
36 | // ( 2012-10-28 16:23:42 UTC ).UnixNano() >> 20
37 | twepoch = int64(1288834974288)
38 | )
39 |
40 | var ErrTimeBackwards = errors.New("time has gone backwards")
41 | var ErrSequenceExpired = errors.New("sequence expired")
42 | var ErrIDBackwards = errors.New("ID went backward")
43 |
44 | type guid int64
45 |
46 | type guidFactory struct {
47 | sync.Mutex
48 |
49 | nodeID int64
50 | sequence int64
51 | lastTimestamp int64
52 | lastID guid
53 | }
54 |
55 | func NewGUIDFactory(nodeID int64) *guidFactory {
56 | return &guidFactory{
57 | nodeID: nodeID,
58 | }
59 | }
60 |
61 | func (f *guidFactory) NewGUID() (guid, error) {
62 | f.Lock()
63 |
64 | // divide by 1048576, giving pseudo-milliseconds
65 | ts := time.Now().UnixNano() >> 20
66 |
67 | if ts < f.lastTimestamp {
68 | f.Unlock()
69 | return 0, ErrTimeBackwards
70 | }
71 |
72 | if f.lastTimestamp == ts {
73 | f.sequence = (f.sequence + 1) & sequenceMask
74 | if f.sequence == 0 {
75 | f.Unlock()
76 | return 0, ErrSequenceExpired
77 | }
78 | } else {
79 | f.sequence = 0
80 | }
81 |
82 | f.lastTimestamp = ts
83 |
84 | id := guid(((ts - twepoch) << timestampShift) |
85 | (f.nodeID << nodeIDShift) |
86 | f.sequence)
87 |
88 | if id <= f.lastID {
89 | f.Unlock()
90 | return 0, ErrIDBackwards
91 | }
92 |
93 | f.lastID = id
94 |
95 | f.Unlock()
96 |
97 | return id, nil
98 | }
99 |
100 | func (g guid) Hex() [16]byte {
101 | var h [16]byte
102 | var b [8]byte
103 |
104 | b[0] = byte(g >> 56)
105 | b[1] = byte(g >> 48)
106 | b[2] = byte(g >> 40)
107 | b[3] = byte(g >> 32)
108 | b[4] = byte(g >> 24)
109 | b[5] = byte(g >> 16)
110 | b[6] = byte(g >> 8)
111 | b[7] = byte(g)
112 |
113 | hex.Encode(h[:], b[:])
114 | return h
115 | }
116 |
--------------------------------------------------------------------------------
/web/vue-admin/src/layout/components/Sidebar/SidebarItem.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
8 |
9 |
10 |
11 |
12 |
13 |
24 |
25 |
26 |
27 |
96 |
--------------------------------------------------------------------------------
/web/vue-admin/tests/unit/components/Breadcrumb.spec.js:
--------------------------------------------------------------------------------
1 | import { mount, createLocalVue } from '@vue/test-utils'
2 | import VueRouter from 'vue-router'
3 | import ElementUI from 'element-ui'
4 | import Breadcrumb from '@/components/Breadcrumb/index.vue'
5 |
6 | const localVue = createLocalVue()
7 | localVue.use(VueRouter)
8 | localVue.use(ElementUI)
9 |
10 | const routes = [
11 | {
12 | path: '/',
13 | name: 'home',
14 | children: [{
15 | path: 'dashboard',
16 | name: 'dashboard'
17 | }]
18 | },
19 | {
20 | path: '/menu',
21 | name: 'menu',
22 | children: [{
23 | path: 'menu1',
24 | name: 'menu1',
25 | meta: { title: 'menu1' },
26 | children: [{
27 | path: 'menu1-1',
28 | name: 'menu1-1',
29 | meta: { title: 'menu1-1' }
30 | },
31 | {
32 | path: 'menu1-2',
33 | name: 'menu1-2',
34 | redirect: 'noredirect',
35 | meta: { title: 'menu1-2' },
36 | children: [{
37 | path: 'menu1-2-1',
38 | name: 'menu1-2-1',
39 | meta: { title: 'menu1-2-1' }
40 | },
41 | {
42 | path: 'menu1-2-2',
43 | name: 'menu1-2-2'
44 | }]
45 | }]
46 | }]
47 | }]
48 |
49 | const router = new VueRouter({
50 | routes
51 | })
52 |
53 | describe('Breadcrumb.vue', () => {
54 | const wrapper = mount(Breadcrumb, {
55 | localVue,
56 | router
57 | })
58 | it('dashboard', () => {
59 | router.push('/dashboard')
60 | const len = wrapper.findAll('.el-breadcrumb__inner').length
61 | expect(len).toBe(1)
62 | })
63 | it('normal route', () => {
64 | router.push('/menu/menu1')
65 | const len = wrapper.findAll('.el-breadcrumb__inner').length
66 | expect(len).toBe(2)
67 | })
68 | it('nested route', () => {
69 | router.push('/menu/menu1/menu1-2/menu1-2-1')
70 | const len = wrapper.findAll('.el-breadcrumb__inner').length
71 | expect(len).toBe(4)
72 | })
73 | it('no meta.title', () => {
74 | router.push('/menu/menu1/menu1-2/menu1-2-2')
75 | const len = wrapper.findAll('.el-breadcrumb__inner').length
76 | expect(len).toBe(3)
77 | })
78 | // it('click link', () => {
79 | // router.push('/menu/menu1/menu1-2/menu1-2-2')
80 | // const breadcrumbArray = wrapper.findAll('.el-breadcrumb__inner')
81 | // const second = breadcrumbArray.at(1)
82 | // console.log(breadcrumbArray)
83 | // const href = second.find('a').attributes().href
84 | // expect(href).toBe('#/menu/menu1')
85 | // })
86 | // it('noRedirect', () => {
87 | // router.push('/menu/menu1/menu1-2/menu1-2-1')
88 | // const breadcrumbArray = wrapper.findAll('.el-breadcrumb__inner')
89 | // const redirectBreadcrumb = breadcrumbArray.at(2)
90 | // expect(redirectBreadcrumb.contains('a')).toBe(false)
91 | // })
92 | it('last breadcrumb', () => {
93 | router.push('/menu/menu1/menu1-2/menu1-2-1')
94 | const breadcrumbArray = wrapper.findAll('.el-breadcrumb__inner')
95 | const redirectBreadcrumb = breadcrumbArray.at(3)
96 | expect(redirectBreadcrumb.contains('a')).toBe(false)
97 | })
98 | })
99 |
--------------------------------------------------------------------------------
/internal/model/consumeservermap.go:
--------------------------------------------------------------------------------
1 | package model
2 |
3 | import (
4 | "errors"
5 | "time"
6 | )
7 |
8 | type ConsumeServerMap struct {
9 | Id int `json:"id" gorm:"primaryKey"`
10 | Consumeid int `json:"consumeid"`
11 | Serverid int `json:"serverid"`
12 | Weight int `json:"weight"`
13 | Invalid int `json:"invalid"`
14 | //创建时间
15 | CreatedAt time.Time `json:"createdAt"`
16 | //更新时间
17 | UpdatedAt time.Time `json:"updatedAt"`
18 |
19 | WorkServer WorkServer `json:"workServer" gorm:"-"`
20 | }
21 |
22 | func (ConsumeServerMap) TableName() string {
23 | return "nsqproxy_consume_server_map"
24 | }
25 |
26 | func (ConsumeServerMap) CreateTable() error {
27 | sql := "CREATE TABLE IF NOT EXISTS `nsqproxy_consume_server_map` (" +
28 | "`id` int(11) unsigned NOT NULL AUTO_INCREMENT," +
29 | "`consumeid` int(11) NOT NULL COMMENT '消费者id'," +
30 | "`serverid` int(11) NOT NULL COMMENT '服务器id'," +
31 | "`weight` int(11) DEFAULT '0' COMMENT '权重'," +
32 | "`invalid` tinyint(4) DEFAULT '0' COMMENT '是否有效，0是有效'," +
33 | "`created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间'," +
34 | "`updated_at` timestamp NULL DEFAULT NULL COMMENT '更新时间'," +
35 | "PRIMARY KEY (`id`)," +
36 | "UNIQUE KEY `index_uq_cid_sid` (`consumeid`,`serverid`)" +
37 | ") ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='消费者和可消费的服务器之间的关联关系';"
38 | return db.Exec(sql).Error
39 | }
40 |
41 | //两份配置是否相等
42 | func (m ConsumeServerMap) IsEqual(newMap ConsumeServerMap) bool {
43 | if m.Id != newMap.Id || m.Consumeid != newMap.Consumeid || m.Serverid != newMap.Serverid || m.Weight != newMap.Weight || m.Invalid != newMap.Invalid {
44 | return false
45 | }
46 | if !m.WorkServer.IsEqual(newMap.WorkServer) {
47 | return false
48 | }
49 | return true
50 | }
51 |
52 | func (m *ConsumeServerMap) Create() (int, error) {
53 | result := db.Create(m)
54 | if result.Error != nil {
55 | return 0, result.Error
56 | } else if result.RowsAffected <= 0 {
57 | return 0, errors.New("RowsAffected is zero")
58 | } else if m.Id <= 0 {
59 | return 0, errors.New("primaryKey is zero")
60 | }
61 | return m.Id, nil
62 | }
63 |
64 | func (m *ConsumeServerMap) Delete() (int64, error) {
65 | if m.Id <= 0 {
66 | return 0, errors.New("primaryKey is zero")
67 | }
68 | result := db.Delete(m, m.Id)
69 | return result.RowsAffected, result.Error
70 | }
71 |
72 | func (m *ConsumeServerMap) Update() (int64, error) {
73 | if m.Id <= 0 {
74 | return 0, errors.New("primaryKey is zero")
75 | }
76 | result := db.Select("Id", "Consumeid", "Serverid", "Weight", "Invalid", "UpdatedAt").Updates(m)
77 | if result.Error != nil {
78 | return 0, result.Error
79 | }
80 | return result.RowsAffected, nil
81 | }
82 |
83 | func (m *ConsumeServerMap) Get() (int64, error) {
84 | if m.Id <= 0 {
85 | return 0, errors.New("primaryKey is zero")
86 | }
87 | result := db.First(m)
88 | return result.RowsAffected, result.Error
89 | }
90 |
91 | func (m *ConsumeServerMap) AllByConsumeid(consumeid int) ([]ConsumeServerMap, error) {
92 | mList := make([]ConsumeServerMap, 0)
93 | result := db.Where("consumeid = ?", consumeid).Find(&mList)
94 | return mList, result.Error
95 | }
96 |
--------------------------------------------------------------------------------
/web/vue-admin/src/utils/index.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Created by PanJiaChen on 16/11/18.
3 | */
4 |
5 | /**
6 | * Parse the time to string
7 | * @param {(Object|string|number)} time
8 | * @param {string} cFormat
9 | * @returns {string | null}
10 | */
11 | export function parseTime(time, cFormat) {
12 | if (arguments.length === 0 || !time) {
13 | return null
14 | }
15 | const format = cFormat || '{y}-{m}-{d} {h}:{i}:{s}'
16 | let date
17 | if (typeof time === 'object') {
18 | date = time
19 | } else {
20 | if ((typeof time === 'string')) {
21 | if ((/^[0-9]+$/.test(time))) {
22 | // support "1548221490638"
23 | time = parseInt(time)
24 | } else {
25 | // support safari
26 | // https://stackoverflow.com/questions/4310953/invalid-date-in-safari
27 | time = time.replace(new RegExp(/-/gm), '/')
28 | }
29 | }
30 |
31 | if ((typeof time === 'number') && (time.toString().length === 10)) {
32 | time = time * 1000
33 | }
34 | date = new Date(time)
35 | }
36 | const formatObj = {
37 | y: date.getFullYear(),
38 | m: date.getMonth() + 1,
39 | d: date.getDate(),
40 | h: date.getHours(),
41 | i: date.getMinutes(),
42 | s: date.getSeconds(),
43 | a: date.getDay()
44 | }
45 | const time_str = format.replace(/{([ymdhisa])+}/g, (result, key) => {
46 | const value = formatObj[key]
47 | // Note: getDay() returns 0 on Sunday
48 | if (key === 'a') { return ['日', '一', '二', '三', '四', '五', '六'][value ] }
49 | return value.toString().padStart(2, '0')
50 | })
51 | return time_str
52 | }
53 |
54 | /**
55 | * @param {number} time
56 | * @param {string} option
57 | * @returns {string}
58 | */
59 | export function formatTime(time, option) {
60 | if (('' + time).length === 10) {
61 | time = parseInt(time) * 1000
62 | } else {
63 | time = +time
64 | }
65 | const d = new Date(time)
66 | const now = Date.now()
67 |
68 | const diff = (now - d) / 1000
69 |
70 | if (diff < 30) {
71 | return '刚刚'
72 | } else if (diff < 3600) {
73 | // less 1 hour
74 | return Math.ceil(diff / 60) + '分钟前'
75 | } else if (diff < 3600 * 24) {
76 | return Math.ceil(diff / 3600) + '小时前'
77 | } else if (diff < 3600 * 24 * 2) {
78 | return '1天前'
79 | }
80 | if (option) {
81 | return parseTime(time, option)
82 | } else {
83 | return (
84 | d.getMonth() +
85 | 1 +
86 | '月' +
87 | d.getDate() +
88 | '日' +
89 | d.getHours() +
90 | '时' +
91 | d.getMinutes() +
92 | '分'
93 | )
94 | }
95 | }
96 |
97 | /**
98 | * @param {string} url
99 | * @returns {Object}
100 | */
101 | export function param2Obj(url) {
102 | const search = decodeURIComponent(url.split('?')[1]).replace(/\+/g, ' ')
103 | if (!search) {
104 | return {}
105 | }
106 | const obj = {}
107 | const searchArr = search.split('&')
108 | searchArr.forEach(v => {
109 | const index = v.indexOf('=')
110 | if (index !== -1) {
111 | const name = v.substring(0, index)
112 | const val = v.substring(index + 1, v.length)
113 | obj[name] = val
114 | }
115 | })
116 | return obj
117 | }
118 |
--------------------------------------------------------------------------------
/document/api/admin/api/consumeconfig/worklist.md:
--------------------------------------------------------------------------------
1 | ### 查询一个消费者关联服务器
2 |
3 | - 接口功能
4 |
5 | > 根据消费者ID，查询消费者所关联的所有服务器列表
6 |
7 | ```
8 | GET /admin/api/consumeConfig/workList
9 | ```
10 |
11 | - 请求参数
12 |
13 | |参数|必选|类型|说明|
14 | |:----- |:-------|:-----|----- |
15 | |id |true |int |消费者ID |
16 |
17 | - 返回结果
18 |
19 | ```
20 | {
21 | code: 200,
22 | msg: "ok",
23 | result: {
24 | id: 1,
25 | topic: "test_topic",
26 | channel: "one",
27 | description: "描述",
28 | owner: "责任人",
29 | monitorThreshold: 0,
30 | handleNum: 6,
31 | maxInFlight: 6,
32 | isRequeue: false,
33 | timeoutDial: 3590,
34 | timeoutRead: 3590,
35 | timeoutWrite: 3590,
36 | invalid: 0,
37 | createdAt: "2020-08-20T15:08:39+08:00",
38 | updatedAt: "0001-01-01T00:00:00Z",
39 | serverList: [
40 | {
41 | id: 1,
42 | consumeid: 1,
43 | serverid: 1,
44 | weight: 1,
45 | invalid: 0,
46 | createdAt: "2018-10-16T14:36:33+08:00",
47 | updatedAt: "0001-01-01T00:00:00Z",
48 | workServer: {
49 | id: 1,
50 | addr: "1.1.1.1:80,
51 | protocol: "HTTP",
52 | extra: "",
53 | description: "通用机器1",
54 | owner: "",
55 | invalid: 0,
56 | createdAt: "2018-10-16T14:30:58+08:00",
57 | updatedAt: "0001-01-01T00:00:00Z"
58 | }
59 | },
60 | {
61 | id: 2,
62 | consumeid: 1,
63 | serverid: 2,
64 | weight: 1,
65 | invalid: 0,
66 | createdAt: "2018-10-16T14:36:33+08:00",
67 | updatedAt: "0001-01-01T00:00:00Z",
68 | workServer: {
69 | id: 2,
70 | addr: "1.1.1.2:80,
71 | protocol: "HTTP",
72 | extra: "",
73 | description: "通用机器2",
74 | owner: "",
75 | invalid: 0,
76 | createdAt: "2018-10-16T14:30:58+08:00",
77 | updatedAt: "0001-01-01T00:00:00Z"
78 | }
79 | },
80 | {
81 | id: 3,
82 | consumeid: 1,
83 | serverid: 3,
84 | weight: 1,
85 | invalid: 0,
86 | createdAt: "2018-10-16T14:36:33+08:00",
87 | updatedAt: "0001-01-01T00:00:00Z",
88 | workServer: {
89 | id: 3,
90 | addr: "1.1.1.3:80,
91 | protocol: "HTTP",
92 | extra: "",
93 | description: "通用机器3",
94 | owner: "",
95 | invalid: 0,
96 | createdAt: "2018-10-16T14:30:58+08:00",
97 | updatedAt: "0001-01-01T00:00:00Z"
98 | }
99 | }
100 | ]
101 | }
102 | }
103 | ```
--------------------------------------------------------------------------------
/internal/model/consumeservermap_test.go:
--------------------------------------------------------------------------------
1 | package model
2 |
3 | import (
4 | "testing"
5 | )
6 |
7 | func TestConsumeServerMap_CreateTable(t *testing.T) {
8 | NewDB("0.0.0.0", "3306", "root", "", "nsqproxy")
9 | m := ConsumeServerMap{}
10 | err := m.CreateTable()
11 | if err != nil {
12 | t.Fatalf("create table error: %s", err.Error())
13 | }
14 | db, err := db.DB()
15 | if err != nil {
16 | t.Fatalf("get db error: %s", err.Error())
17 | }
18 | err = db.Close()
19 | if err != nil {
20 | t.Fatalf("close db error: %s", err.Error())
21 | }
22 | }
23 |
24 | func TestConsumeServerMap_CURD(t *testing.T) {
25 | NewDB("0.0.0.0", "3306", "root", "", "nsqproxy")
26 | m := ConsumeServerMap{
27 | Consumeid: 1,
28 | Serverid: 2,
29 | Weight: 1,
30 | }
31 | //新增
32 | result := db.Table("nsqproxy_consume_server_map").Create(&m)
33 | id := m.Id
34 | if result.Error != nil {
35 | t.Fatalf("[insert]error: %s", result.Error.Error())
36 | }
37 | if result.RowsAffected != 1 {
38 | t.Fatalf("[insert]RowsAffected is not 1. result: %d", result.RowsAffected)
39 | }
40 | if id <= 0 {
41 | t.Fatalf("[insert]id is 0")
42 | }
43 |
44 | //查询
45 | find := ConsumeServerMap{}
46 | result = db.Table("nsqproxy_consume_server_map").First(&find, id)
47 | if result.Error != nil {
48 | t.Fatalf("[select]error: %s", result.Error.Error())
49 | }
50 | if result.RowsAffected != 1 {
51 | t.Fatalf("[select]RowsAffected is not 1. result: %d", result.RowsAffected)
52 | }
53 |
54 | //修改
55 | m.Invalid = 10
56 | m.Weight = 11
57 | result = db.Save(&m)
58 | if result.Error != nil {
59 | t.Fatalf("[update]error: %s", result.Error.Error())
60 | }
61 | if result.RowsAffected != 1 {
62 | t.Fatalf("[update]RowsAffected is not 1. result: %d", result.RowsAffected)
63 | }
64 | if m.Id != id {
65 | t.Fatalf("[update]id has changed")
66 | }
67 |
68 | //查询
69 | find = ConsumeServerMap{}
70 | result = db.First(&find, id)
71 | if result.Error != nil {
72 | t.Fatalf("[select2]error: %s", result.Error.Error())
73 | }
74 | if find.Invalid != 10 || find.Weight != 11 {
75 | t.Fatalf("[select2]update failed")
76 | }
77 | if result.RowsAffected != 1 {
78 | t.Fatalf("[select2]RowsAffected is not 1. result: %d", result.RowsAffected)
79 | }
80 | if !m.IsEqual(find) {
81 | t.Fatalf("[select]Query result failed")
82 | }
83 |
84 | //删除
85 | result = db.Delete(&m)
86 | if result.Error != nil {
87 | t.Fatalf("[delete]error: %s", result.Error.Error())
88 | }
89 | if result.RowsAffected != 1 {
90 | t.Fatalf("[delete]RowsAffected is not 1. result: %d", result.RowsAffected)
91 | }
92 |
93 | //查询
94 | find = ConsumeServerMap{}
95 | result = db.First(&find, id)
96 | if find.Serverid != 0 {
97 | t.Fatalf("[select3]id is not 0")
98 | }
99 | if result.Error == nil {
100 | t.Fatalf("[select3]no error")
101 | }
102 | if result.RowsAffected != 0 {
103 | t.Fatalf("[select3]RowsAffected is not 1. result: %d", result.RowsAffected)
104 | }
105 |
106 | //关闭
107 | db, err := db.DB()
108 | if err != nil {
109 | t.Fatalf("get db error: %s", err.Error())
110 | }
111 | err = db.Close()
112 | if err != nil {
113 | t.Fatalf("close db error: %s", err.Error())
114 | }
115 | }
116 |
--------------------------------------------------------------------------------
/internal/model/workserver_test.go:
--------------------------------------------------------------------------------
1 | package model
2 |
3 | import (
4 | "testing"
5 | )
6 |
7 | func TestWorkServer_CreateTable(t *testing.T) {
8 | NewDB("0.0.0.0", "3306", "root", "", "nsqproxy")
9 | w := WorkServer{}
10 | err := w.CreateTable()
11 | if err != nil {
12 | t.Fatalf("create table error: %s", err.Error())
13 | }
14 | db, err := db.DB()
15 | if err != nil {
16 | t.Fatalf("get db error: %s", err.Error())
17 | }
18 | err = db.Close()
19 | if err != nil {
20 | t.Fatalf("close db error: %s", err.Error())
21 | }
22 | }
23 |
24 | func TestWorkServer_CURD(t *testing.T) {
25 | NewDB("0.0.0.0", "3306", "root", "", "nsqproxy")
26 | w := WorkServer{
27 | Addr: "0.0.0.0:80",
28 | Protocol: "HTTP",
29 | }
30 | //新增
31 | result := db.Table("nsqproxy_work_server").Create(&w)
32 | id := w.Id
33 | if result.Error != nil {
34 | t.Fatalf("[insert]error: %s", result.Error.Error())
35 | }
36 | if result.RowsAffected != 1 {
37 | t.Fatalf("[insert]RowsAffected is not 1. result: %d", result.RowsAffected)
38 | }
39 | if id <= 0 {
40 | t.Fatalf("[insert]serverid is 0")
41 | }
42 |
43 | //查询
44 | find := WorkServer{}
45 | result = db.Table("nsqproxy_work_server").First(&find, id)
46 | if result.Error != nil {
47 | t.Fatalf("[select]error: %s", result.Error.Error())
48 | }
49 | if result.RowsAffected != 1 {
50 | t.Fatalf("[select]RowsAffected is not 1. result: %d", result.RowsAffected)
51 | }
52 | if !w.IsEqual(find) {
53 | t.Fatalf("[select]Query result failed")
54 | }
55 |
56 | //修改
57 | w.Addr = "0.0.0.0:81"
58 | w.Protocol = "CBNSQ"
59 | result = db.Save(&w)
60 | if result.Error != nil {
61 | t.Fatalf("[update]error: %s", result.Error.Error())
62 | }
63 | if result.RowsAffected != 1 {
64 | t.Fatalf("[update]RowsAffected is not 1. result: %d", result.RowsAffected)
65 | }
66 | if w.Id != id {
67 | t.Fatalf("[update]id has changed")
68 | }
69 |
70 | //查询
71 | find = WorkServer{}
72 | result = db.First(&find, id)
73 | if result.Error != nil {
74 | t.Fatalf("[select2]error: %s", result.Error.Error())
75 | }
76 | if find.Addr != "0.0.0.0:81" || find.Protocol != "CBNSQ" {
77 | t.Fatalf("[select2]update failed")
78 | }
79 | if result.RowsAffected != 1 {
80 | t.Fatalf("[select2]RowsAffected is not 1. result: %d", result.RowsAffected)
81 | }
82 | if !w.IsEqual(find) {
83 | t.Fatalf("[select2]Query result failed")
84 | }
85 | //删除
86 | result = db.Delete(&w)
87 | if result.Error != nil {
88 | t.Fatalf("[delete]error: %s", result.Error.Error())
89 | }
90 | if result.RowsAffected != 1 {
91 | t.Fatalf("[delete]RowsAffected is not 1. result: %d", result.RowsAffected)
92 | }
93 |
94 | //查询
95 | find = WorkServer{}
96 | result = db.First(&find, id)
97 | if find.Id != 0 {
98 | t.Fatalf("[select3]id is not 0")
99 | }
100 | if result.Error == nil {
101 | t.Fatalf("[select3]no error")
102 | }
103 | if result.RowsAffected != 0 {
104 | t.Fatalf("[select3]RowsAffected is not 1. result: %d", result.RowsAffected)
105 | }
106 |
107 | //关闭
108 | db, err := db.DB()
109 | if err != nil {
110 | t.Fatalf("get db error: %s", err.Error())
111 | }
112 | err = db.Close()
113 | if err != nil {
114 | t.Fatalf("close db error: %s", err.Error())
115 | }
116 | }
117 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # NSQProxy
2 | NSQProxy是Golang开发的NSQ和Worker之间的中间件，根据数据库配置，负责消息转发。NSQProxy启动后，接受NSQD队列内容，然后通过HTTP/FastCGI/CBNSQ等协议转发给Worker机执行。在唱吧内部使用2年，高效稳定的处理着每日数十亿条消息。
3 |
4 | [](https://goreportcard.com/report/github.com/changba/nsqproxy)
5 | [](https://github.com/changba/nsqproxy/blob/master/LICENSE)
6 | [](https://github.com/changba/nsqproxy/releases)
7 | [](https://github.com/changba/nsqproxy/releases)
8 |
9 | ## 解决的问题
10 |
11 | * 各Topic执行机器可配
12 | * 各Topic消费速度可配
13 | * 各Worker机协议可配
14 | * HTTP：将消息发送给配好的URL。
15 | * FastCGI：将消息发送给配置的服务端，如PHP-FPM。
16 | * CBNSQ：自定义的基于TCP的文本协议。
17 | * 可视化界面管理
18 | * 队列积压超出阈值报警
19 | * 散乱在各处的消费者集中化管理
20 | * 通过网络分发，无需安装.so等扩展库，因此无需修改线上环境
21 |
22 |
23 | ## 有图有真相
24 |
25 | 
26 |
27 | 
28 |
29 | 
30 |
31 | ## 使用
32 | 请先部署好NSQLookupd、NSQd、MySQL
33 |
34 | > 启动NSQLookupd `nsqlookupd -broadcast-address="0.0.0.0" -http-address="0.0.0.0:4161" -tcp-address="0.0.0.0:4160"`
35 |
36 | > 启动NSQD `nsqd -broadcast-address="0.0.0.0" -lookupd-tcp-address="0.0.0.0:4160" -tcp-address="0.0.0.0:4150" -http-address="0.0.0.0:4151"`
37 |
38 | > 启动MySQL
39 |
40 | ### 安装
41 |
42 | #### 二进制安装
43 |
44 | * 下载最新版本的压缩包 https://github.com/changba/nsqproxy/releases
45 | * 解压
46 | * 启动（注意替换为自己的MySQL信息） `./nsqproxy -dbHost=127.0.0.1 -dbPort=3306 -dbUsername=root -dbPassword=rootpsd -dbName=nsqproxy -logLevel=debug -nsqlookupdHTTP=127.0.0.1:4161`
47 | * 命令行 `curl http://0.0.0.0:19421/status` 输出ok
48 | * 浏览器打开 http://0.0.0.0:19421/admin
49 |
50 | #### 源码安装
51 |
52 | * 要求Go1.13及以上
53 | * 下载本项目 `go get github.com/changba/nsqproxy`
54 | * `cd nsqproxy`
55 | * `export GO111MODULE=on`
56 | * 编译 `make build`
57 | * 启动（注意替换为自己的MySQL信息） `./bin/nsqproxy -dbHost=127.0.0.1 -dbPort=3306 -dbUsername=root -dbPassword=rootpsd -dbName=nsqproxy -logLevel=debug -nsqlookupdHTTP=127.0.0.1:4161`
58 | * 命令行 `curl http://0.0.0.0:19421/status` 输出ok
59 | * 浏览器打开 http://0.0.0.0:19421/admin
60 |
61 | ### 快速开始
62 |
63 | * [快速体验](document/doc/quick_start.md)
64 | * [启动参数](document/doc/flag.md)
65 | * [make命令](document/doc/make.md)
66 | * [文档](document/doc/README.md)
67 |
68 | ## 二次开发
69 |
70 | ### 前端
71 | 使用VUE开发，所有源码均在/web/vue-admin目录中，开发完成后需要编译，编译后的文件存放在/web/public/目录中。使用开源项目statik将静态文件/web/public/变成一个go文件internal/statik/statik.go，这样前端的静态文件也会被我们编译到同一个二进制文件中了。
72 |
73 | * 启动go服务 `make run`
74 | * 安装VUE `make vue-install`（如果国内被墙可以使用淘宝的源进行安装：make vue-install-taobao）
75 | * 开启VUE开发环境 `make vue-dev`
76 | * 浏览器打开 http://0.0.0.0:9528/admin
77 | * 开发前端相关功能
78 | * 编译VUE `make vue-build`
79 | * 前段文件转换为一个go文件 `make statik`
80 | * 编译go服务 `make build`
81 | * 浏览器打开 http://0.0.0.0:19421/admin
82 |
83 | ### 接口文档
84 | * 通过接口对数据库增删改查：[查看接口文档](document/api/README.md)
85 |
86 | ## TODO LIST
87 |
88 | * 协议增加protobuf
89 | * 后台增加用户权限管理
90 | * 报警HOOK
91 | * 日志按天分割
92 |
93 | ## License
94 |
95 | © [Changba.com](https://changba.com), 2020~time.Now
96 |
97 | Released under the [MIT License](https://github.com/changba/nsqproxy/blob/main/LICENSE)
--------------------------------------------------------------------------------
/internal/proxy/loadbalance.go:
--------------------------------------------------------------------------------
1 | package proxy
2 |
3 | import (
4 | "errors"
5 | "github.com/changba/nsqproxy/internal/model"
6 | )
7 |
8 | const LoadBalanceMethodLoop = "loop"
9 |
10 | //负载均衡分发器
11 | //目前只支持轮询
12 | type loadBalanceDispatcher interface {
13 | //初始化方法
14 | new([]model.ConsumeServerMap)
15 | //选择
16 | pick() (model.ConsumeServerMap, error)
17 | }
18 |
19 | type loadBalance struct {
20 | method string //负载均衡的方法
21 | dispatcher loadBalanceDispatcher //负载均衡分发器，具体的算法执行 者
22 | }
23 |
24 | func newLoadBalance(method string, consumeServerMapList []model.ConsumeServerMap) *loadBalance {
25 | //过滤一下，选出可用的机器
26 | list := make([]model.ConsumeServerMap, 0)
27 | for _, v := range consumeServerMapList {
28 | if v.Id <= 0 || v.Weight <= 0 || len(v.WorkServer.Addr) <= 0 || len(v.WorkServer.Protocol) <= 0 {
29 | continue
30 | }
31 | list = append(list, v)
32 | }
33 |
34 | //构造分发器，目前只支持轮询
35 | //轮询
36 | var dispatcher loadBalanceDispatcher
37 | if method == LoadBalanceMethodLoop {
38 | dispatcher = &loadBalanceLoop{}
39 | } else {
40 | method = LoadBalanceMethodLoop
41 | dispatcher = &loadBalanceLoop{}
42 | }
43 | dispatcher.new(list)
44 |
45 | return &loadBalance{
46 | method: method,
47 | dispatcher: dispatcher,
48 | }
49 | }
50 |
51 | //选择一个work机器 - 轮询的方式根据权重选择
52 | func (l *loadBalance) pickWorker() (model.ConsumeServerMap, error) {
53 | work, err := l.dispatcher.pick()
54 | //失败了就再取一次
55 | if err != nil || work.Id <= 0 || work.Weight <= 0 || len(work.WorkServer.Addr) <= 0 || len(work.WorkServer.Protocol) <= 0 {
56 | work, err = l.dispatcher.pick()
57 | if err != nil || work.Id <= 0 || work.Weight <= 0 || len(work.WorkServer.Addr) <= 0 || len(work.WorkServer.Protocol) <= 0 {
58 | return work, errors.New("no available work")
59 | }
60 | }
61 | return work, nil
62 | }
63 |
64 | type loadBalanceLoop struct {
65 | list []model.ConsumeServerMap //机器列表
66 | capacity int //权重总分
67 | seek int
68 | }
69 |
70 | func (l *loadBalanceLoop) new(consumeServerMapList []model.ConsumeServerMap) {
71 | //所有机器的权重和
72 | capacity := 0
73 | for _, v := range consumeServerMapList {
74 | capacity += v.Weight
75 | }
76 | //生成Worker列表，按顺序使用，比rand节省CPU
77 | multiple := 100
78 | capacity *= multiple
79 | wl := make([]model.ConsumeServerMap, capacity)
80 | //填充
81 | index := 0
82 | for i := 0; i < multiple; i++ {
83 | for _, consumeServerMap := range consumeServerMapList {
84 | for j := 0; j < consumeServerMap.Weight; j++ {
85 | wl[index] = consumeServerMap
86 | index++
87 | }
88 | }
89 | }
90 | l.list = wl
91 | l.capacity = capacity
92 | l.seek = 0
93 | }
94 |
95 | //选择一个work机器 - 轮询的方式根据权重选择
96 | func (l *loadBalanceLoop) pick() (model.ConsumeServerMap, error) {
97 | //这里无需加锁，并发问题最多导致分布不是绝对的均衡。你多执行一个我少执行一个无伤大雅。
98 | seek := l.seek
99 | if seek >= l.capacity {
100 | l.seek = 0
101 | seek = 0
102 | }
103 | work := l.list[seek]
104 | //失败了就取索引为的0，补取一次
105 | if work.Id <= 0 || work.Weight <= 0 || len(work.WorkServer.Addr) <= 0 || len(work.WorkServer.Protocol) <= 0 {
106 | l.seek = 0
107 | seek = 0
108 | work = l.list[seek]
109 | if work.Id <= 0 || work.Weight <= 0 || len(work.WorkServer.Addr) <= 0 || len(work.WorkServer.Protocol) <= 0 {
110 | return work, errors.New("no available work")
111 | }
112 | }
113 | l.seek++
114 | return work, nil
115 | }
116 |
--------------------------------------------------------------------------------
/internal/model/consumeconfig_test.go:
--------------------------------------------------------------------------------
1 | package model
2 |
3 | import (
4 | "testing"
5 | )
6 |
7 | func TestConsumeConfig_CreateTable(t *testing.T) {
8 | NewDB("0.0.0.0", "3306", "root", "", "nsqproxy")
9 | c := ConsumeConfig{}
10 | err := c.CreateTable()
11 | if err != nil {
12 | t.Fatalf("create table error: %s", err.Error())
13 | }
14 | db, err := db.DB()
15 | if err != nil {
16 | t.Fatalf("get db error: %s", err.Error())
17 | }
18 | err = db.Close()
19 | if err != nil {
20 | t.Fatalf("close db error: %s", err.Error())
21 | }
22 | }
23 |
24 | func TestConsumeConfig_CURD(t *testing.T) {
25 | NewDB("0.0.0.0", "3306", "root", "", "nsqproxy")
26 | c := ConsumeConfig{
27 | Topic: "golang_test_topic",
28 | Channel: "golang_test_channel",
29 | Description: "go test",
30 | Owner: "go",
31 | }
32 | //新增
33 | result := db.Table("nsqproxy_consume_config").Create(&c)
34 | id := c.Id
35 | if result.Error != nil {
36 | t.Fatalf("[insert]error: %s", result.Error.Error())
37 | }
38 | if result.RowsAffected != 1 {
39 | t.Fatalf("[insert]RowsAffected is not 1. result: %d", result.RowsAffected)
40 | }
41 | if id <= 0 {
42 | t.Fatalf("[insert]consumeid is 0")
43 | }
44 |
45 | //查询
46 | find := ConsumeConfig{}
47 | result = db.Table("nsqproxy_consume_config").First(&find, id)
48 | if result.Error != nil {
49 | t.Fatalf("[select]error: %s", result.Error.Error())
50 | }
51 | if result.RowsAffected != 1 {
52 | t.Fatalf("[select]RowsAffected is not 1. result: %d", result.RowsAffected)
53 | }
54 | if !c.IsEqual(find) {
55 | t.Fatalf("[select]Query result failed")
56 | }
57 |
58 | //修改
59 | c.MonitorThreshold = 100
60 | c.HandleNum = 10
61 | c.MaxInFlight = 20
62 | result = db.Save(&c)
63 | if result.Error != nil {
64 | t.Fatalf("[update]error: %s", result.Error.Error())
65 | }
66 | if result.RowsAffected != 1 {
67 | t.Fatalf("[update]RowsAffected is not 1. result: %d", result.RowsAffected)
68 | }
69 | if c.Id != id {
70 | t.Fatalf("[update]id has changed")
71 | }
72 |
73 | //查询
74 | find = ConsumeConfig{}
75 | result = db.First(&find, id)
76 | if result.Error != nil {
77 | t.Fatalf("[select2]error: %s", result.Error.Error())
78 | }
79 | if find.MonitorThreshold != 100 || find.HandleNum != 10 || find.MaxInFlight != 20 {
80 | t.Fatalf("[select2]update failed")
81 | }
82 | if result.RowsAffected != 1 {
83 | t.Fatalf("[select2]RowsAffected is not 1. result: %d", result.RowsAffected)
84 | }
85 | if !c.IsEqual(find) {
86 | t.Fatalf("[select2]Query result failed")
87 | }
88 |
89 | //删除
90 | result = db.Delete(&c)
91 | if result.Error != nil {
92 | t.Fatalf("[delete]error: %s", result.Error.Error())
93 | }
94 | if result.RowsAffected != 1 {
95 | t.Fatalf("[delete]RowsAffected is not 1. result: %d", result.RowsAffected)
96 | }
97 |
98 | //查询
99 | find = ConsumeConfig{}
100 | result = db.First(&find, id)
101 | if find.Id != 0 {
102 | t.Fatalf("[select3]id is not 0")
103 | }
104 | if result.Error == nil {
105 | t.Fatalf("[select3]no error")
106 | }
107 | if result.RowsAffected != 0 {
108 | t.Fatalf("[select3]RowsAffected is not 1. result: %d", result.RowsAffected)
109 | }
110 |
111 | //关闭
112 | db, err := db.DB()
113 | if err != nil {
114 | t.Fatalf("get db error: %s", err.Error())
115 | }
116 | err = db.Close()
117 | if err != nil {
118 | t.Fatalf("close db error: %s", err.Error())
119 | }
120 | }
121 |
--------------------------------------------------------------------------------
/config/systemconfig.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | "flag"
5 | "github.com/changba/nsqproxy/internal/module/logger"
6 | "github.com/changba/nsqproxy/internal/module/tool"
7 | "strings"
8 | "time"
9 | )
10 |
11 | const NsqproxyVersion = "1.0.0"
12 |
13 | const RoleMaster = "master"
14 | const RoleBackup = "backup"
15 |
16 | var SystemConfig = &systemConfig{}
17 |
18 | type systemConfig struct {
19 | //-----本项目相关-----
20 | //本机监听的端口
21 | HttpAddr string
22 | //主库IP端口，本机为主库时请留空。
23 | MasterAddr string
24 |
25 | //-----NSQ相关-----
26 | NsqlookupdHttpAddrList []string
27 |
28 | //-----日志相关-----
29 | //订阅的消息日志logger
30 | SubLogger *logger.Logger
31 |
32 | //-----消费者相关-----
33 | UpdateConfigInterval time.Duration
34 |
35 | //-----数据库相关-----
36 | DbHost string
37 | DbPort string
38 | DbUsername string
39 | DbPassword string
40 | DbName string
41 |
42 | //-----本机相关-----
43 | //本机IP
44 | InternalIP string
45 |
46 | //全局配置，无需自定义
47 | Role string
48 | }
49 |
50 | func NewSystemConfig() {
51 | //参数
52 | var httpAddr = flag.String("httpAddr", "0.0.0.0:19421", "监听的HTTP端口")
53 | var masterAddr = flag.String("masterAddr", "", "主库IP端口，为空则本机为主机")
54 | //nsq相关
55 | var nsqlookupdHTTP = flag.String("nsqlookupdHTTP", "127.0.0.1:4161", "nsqLookupd的HTTP地址，多个用逗号分割如'127.0.0.1:4161,127.0.0.1:4163'")
56 | //log相关
57 | var logLevel = flag.String("logLevel", "info", "日志等级，可选有debug、info、warning、error、fatal")
58 | var logPath = flag.String("logPath", "logs/proxy.log", "系统日志路径")
59 | var subLogPath = flag.String("subLogPath", "logs/sub.log", "消费log，由于量大成功消费log仅在日志等级为debug时启用")
60 | //MySQL
61 | var dbHost = flag.String("dbHost", "127.0.0.1", "MySQL的IP")
62 | var dbPort = flag.String("dbPort", "3306", "MySQL的端口")
63 | var dbUsername = flag.String("dbUsername", "root", "MySQL的账号")
64 | var dbPassword = flag.String("dbPassword", "", "MySQL的密码")
65 | var dbName = flag.String("dbName", "nsqproxy", "MySQL的库名")
66 | //消费者相关
67 | var updateConfigInterval = flag.Int64("updateConfigInterval", 60, "更新配置间隔")
68 |
69 | flag.Parse()
70 | //本机相关
71 | SystemConfig.HttpAddr = *httpAddr
72 | if len(SystemConfig.HttpAddr) <= 0 {
73 | panic("httpAddr参数缺失")
74 | }
75 | SystemConfig.MasterAddr = *masterAddr
76 | //nsqlookupd相关
77 | SystemConfig.NsqlookupdHttpAddrList = strings.Split(*nsqlookupdHTTP, ",")
78 | if len(SystemConfig.NsqlookupdHttpAddrList) <= 0 {
79 | panic("nsqlookupdHTTP 缺失")
80 | }
81 | //日志相关
82 | logLevelLower := strings.ToLower(*logLevel)
83 | logLevelList := map[string]struct{}{"debug": struct{}{}, "info": struct{}{}, "warning": struct{}{}, "error": struct{}{}, "fatal": struct{}{}}
84 | if _, ok := logLevelList[logLevelLower]; !ok {
85 | panic("logLevel可选值为debug、info、warning、error、fatal")
86 | }
87 | logger.Init(*logPath, logLevelLower)
88 | SystemConfig.SubLogger = logger.NewLogger(*subLogPath, "", logLevelLower)
89 | //数据库
90 | SystemConfig.DbHost = *dbHost
91 | SystemConfig.DbPort = *dbPort
92 | SystemConfig.DbUsername = *dbUsername
93 | SystemConfig.DbPassword = *dbPassword
94 | SystemConfig.DbName = *dbName
95 | //消费者相关
96 | SystemConfig.UpdateConfigInterval = time.Duration(*updateConfigInterval) * time.Second
97 | //版本
98 | SystemConfig.InternalIP = tool.GetInternalIP()
99 |
100 | //全局配置，无需自定义
101 | SystemConfig.Role = RoleBackup
102 | }
103 |
104 | func (s *systemConfig) Close() bool {
105 | logger.Close()
106 | s.SubLogger.Close()
107 | s = nil
108 | return true
109 | }
110 |
--------------------------------------------------------------------------------
/internal/model/common.go:
--------------------------------------------------------------------------------
1 | package model
2 |
3 | import (
4 | "errors"
5 | "gorm.io/driver/mysql"
6 | "gorm.io/gorm"
7 | "strings"
8 | "time"
9 | )
10 |
11 | const (
12 | InvalidAvailable = 0 //可用
13 | InvalidUnavailable = 1 //不可用
14 | )
15 |
16 | type PageResult struct {
17 | Total int64 `json:"total"`
18 | Page int `json:"page"`
19 | Result interface{} `json:"result"`
20 | }
21 |
22 | var db *gorm.DB
23 |
24 | func NewDB(host, port, username, password, dbname string) {
25 | var err error
26 | dsn := username + ":" + password + "@tcp(" + host + ":" + port + ")/" + dbname + "?charset=utf8&parseTime=true&loc=Local"
27 | db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{})
28 | if err != nil {
29 | panic("open mysql error: " + err.Error())
30 | }
31 | err = ConsumeConfig{}.CreateTable()
32 | if err != nil {
33 | panic("create table error: " + err.Error())
34 | }
35 | err = ConsumeServerMap{}.CreateTable()
36 | if err != nil {
37 | panic("create table error: " + err.Error())
38 | }
39 | err = WorkServer{}.CreateTable()
40 | if err != nil {
41 | panic("create table error: " + err.Error())
42 | }
43 | }
44 |
45 | func IsErrRecordNotFound(err error) bool {
46 | if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
47 | return true
48 | }
49 | return false
50 | }
51 |
52 | //获取可用的消费者列表
53 | func GetAvailableConsumeList() ([]ConsumeConfig, error) {
54 | //执行sql
55 | consumeConfigList := make([]ConsumeConfig, 0)
56 | consumeConfigListDB := make([]ConsumeConfig, 0)
57 | if db == nil {
58 | return consumeConfigList, errors.New("db is nil")
59 | }
60 | result := db.Where("invalid = ?", InvalidAvailable).Find(&consumeConfigListDB)
61 | if result.Error != nil {
62 | if IsErrRecordNotFound(result.Error) {
63 | return consumeConfigList, nil
64 | }
65 | //数据库出错，直接跳过本次查库
66 | return consumeConfigList, result.Error
67 | }
68 | if result.RowsAffected <= 0 {
69 | return consumeConfigList, nil
70 | }
71 | for _, consumeConfig := range consumeConfigListDB {
72 | consumeConfig.Consumer = nil
73 | consumeConfig.TimeoutWrite = consumeConfig.TimeoutWrite * time.Second
74 | consumeConfig.TimeoutRead = consumeConfig.TimeoutRead * time.Second
75 | consumeConfig.TimeoutDial = consumeConfig.TimeoutDial * time.Second
76 | consumeConfig.SetStatusWait()
77 |
78 | //获取消费者配置和worker机关联关系
79 | consumeServerMapList := make([]ConsumeServerMap, 0)
80 | consumeServerMapListDB := make([]ConsumeServerMap, 0)
81 | result = db.Where("consumeid = ? AND weight > 0 AND invalid = ?", consumeConfig.Id, InvalidAvailable).Find(&consumeServerMapListDB)
82 | if result.Error != nil || result.RowsAffected <= 0 || len(consumeServerMapListDB) <= 0 {
83 | continue
84 | }
85 |
86 | //获取每个消费者对应的work机器列表
87 | for _, consumeServerMap := range consumeServerMapListDB {
88 | workServer := WorkServer{}
89 | result = db.Where("id = ? AND invalid = ?", consumeServerMap.Serverid, InvalidAvailable).First(&workServer)
90 | if result.Error != nil || result.RowsAffected <= 0 || workServer.Id <= 0 {
91 | continue
92 | }
93 | workServer.Protocol = strings.ToLower(workServer.Protocol)
94 | workServer.SetStatusAvailable()
95 | consumeServerMap.WorkServer = workServer
96 | consumeServerMapList = append(consumeServerMapList, consumeServerMap)
97 | }
98 | if len(consumeServerMapList) <= 0 {
99 | continue
100 | }
101 | consumeConfig.ServerList = consumeServerMapList
102 | consumeConfigList = append(consumeConfigList, consumeConfig)
103 | }
104 | return consumeConfigList, nil
105 | }
106 |
--------------------------------------------------------------------------------
/internal/httper/workserver.go:
--------------------------------------------------------------------------------
1 | package httper
2 |
3 | import (
4 | "github.com/changba/nsqproxy/internal/model"
5 | "net/http"
6 | "strconv"
7 | )
8 |
9 | type WorkServer struct {
10 | }
11 |
12 | func (WorkServer) Create(w http.ResponseWriter, r *http.Request) {
13 | work := &model.WorkServer{}
14 | work.Addr = r.FormValue("addr")
15 | work.Protocol = r.FormValue("protocol")
16 | work.Extra = r.FormValue("extra")
17 | work.Description = r.FormValue("description")
18 | work.Owner = r.FormValue("owner")
19 | work.Invalid = model.InvalidAvailable
20 | invalid, err := strconv.Atoi(r.FormValue("invalid"))
21 | if err != nil || invalid <= 0 || (invalid != model.InvalidAvailable && invalid != model.InvalidUnavailable) {
22 | invalid = model.InvalidAvailable
23 | }
24 | work.Invalid = invalid
25 | id, err := work.Create()
26 | if err != nil {
27 | Failed(w, HttpCodeBadRequest, "create failed. err: "+err.Error())
28 | return
29 | }
30 | if id <= 0 {
31 | Failed(w, HttpCodeBadRequest, "id is zero")
32 | return
33 | }
34 | Success(w, work)
35 | }
36 |
37 | func (WorkServer) Delete(w http.ResponseWriter, r *http.Request) {
38 | work := &model.WorkServer{}
39 | var err error
40 | work.Id, err = strconv.Atoi(r.FormValue("id"))
41 | if err != nil || work.Id <= 0 {
42 | Failed(w, HttpCodeBadRequest, "param error")
43 | return
44 | }
45 | _, err = work.Delete()
46 | if err != nil {
47 | Failed(w, HttpCodeBadRequest, "delete failed. err: "+err.Error())
48 | return
49 | }
50 | Success(w, "ok")
51 | }
52 |
53 | func (WorkServer) Update(w http.ResponseWriter, r *http.Request) {
54 | work := &model.WorkServer{}
55 | var err error
56 | work.Id, err = strconv.Atoi(r.FormValue("id"))
57 | if err != nil || work.Id <= 0 {
58 | Failed(w, HttpCodeBadRequest, "param error")
59 | return
60 | }
61 | work.Addr = r.FormValue("addr")
62 | work.Protocol = r.FormValue("protocol")
63 | work.Extra = r.FormValue("extra")
64 | work.Description = r.FormValue("description")
65 | work.Owner = r.FormValue("owner")
66 | invalid, err := strconv.Atoi(r.FormValue("invalid"))
67 | if err != nil || invalid <= 0 || (invalid != model.InvalidAvailable && invalid != model.InvalidUnavailable) {
68 | invalid = model.InvalidAvailable
69 | }
70 | work.Invalid = invalid
71 |
72 | _, err = work.Update()
73 | if err != nil {
74 | Failed(w, HttpCodeBadRequest, "upload failed. err: "+err.Error())
75 | return
76 | }
77 | Success(w, "ok")
78 | }
79 |
80 | func (WorkServer) Get(w http.ResponseWriter, r *http.Request) {
81 | work := &model.WorkServer{}
82 | var err error
83 | work.Id, err = strconv.Atoi(r.FormValue("id"))
84 | if err != nil || work.Id <= 0 {
85 | Failed(w, HttpCodeBadRequest, "param error")
86 | return
87 | }
88 | n, err := work.Get()
89 | if err != nil || n == 0 || work.Id <= 0 {
90 | Failed(w, HttpCodeNotFound, "not found")
91 | return
92 | }
93 | Success(w, work)
94 | }
95 |
96 | func (WorkServer) Page(w http.ResponseWriter, r *http.Request) {
97 | work := &model.WorkServer{}
98 | page, err := strconv.Atoi(r.FormValue("page"))
99 | if err != nil || page <= 0 {
100 | page = 1
101 | }
102 | pageResult, err := work.Page(page)
103 | if err != nil {
104 | Failed(w, HttpCodeInternalServerError, "please try again. err: "+err.Error())
105 | return
106 | }
107 | Success(w, pageResult)
108 | }
109 |
110 | func (WorkServer) All(w http.ResponseWriter, r *http.Request) {
111 | work := &model.WorkServer{}
112 | wList, err := work.All()
113 | if err != nil {
114 | Failed(w, HttpCodeInternalServerError, "please try again. err: "+err.Error())
115 | return
116 | }
117 | Success(w, wList)
118 | }
119 |
--------------------------------------------------------------------------------
/internal/proxy/handle.go:
--------------------------------------------------------------------------------
1 | package proxy
2 |
3 | import (
4 | "errors"
5 | "github.com/changba/nsqproxy/config"
6 | "github.com/changba/nsqproxy/internal/model"
7 | "github.com/changba/nsqproxy/internal/module/logger"
8 | "github.com/changba/nsqproxy/internal/worker"
9 | "github.com/nsqio/go-nsq"
10 | "strconv"
11 | )
12 |
13 | //响应成功
14 | const WorkResponseSuccess = "200"
15 |
16 | type Handler struct {
17 | consumeConfig model.ConsumeConfig
18 | workerList *loadBalance
19 | }
20 |
21 | func NewHandler(consumeConfig model.ConsumeConfig) Handler {
22 | return Handler{
23 | consumeConfig: consumeConfig,
24 | workerList: newLoadBalance(LoadBalanceMethodLoop, consumeConfig.ServerList),
25 | }
26 | }
27 |
28 | //处理收到的订阅的消息
29 | //返回error就会重新入队，返回nil则不会
30 | func (h *Handler) HandleMessage(message *nsq.Message) error {
31 | //根据权重选一个work
32 | workServer, err := h.workerList.pickWorker()
33 | if err != nil {
34 | h.recordSubLog(logger.LOG_ERROR, message.ID, message.Body, workServer.WorkServer.Addr, "pick work error. "+err.Error())
35 | return err
36 | }
37 | h.recordSubLog(logger.LOG_DEBUG, message.ID, message.Body, workServer.WorkServer.Addr, "")
38 | //初始化Worker
39 | w, err := worker.NewWorker(workServer.WorkServer.Addr, workServer.WorkServer.Protocol, workServer.WorkServer.Extra, h.consumeConfig.TimeoutDial, h.consumeConfig.TimeoutWrite, h.consumeConfig.TimeoutRead)
40 | if err != nil {
41 | h.recordSubLog(logger.LOG_ERROR, message.ID, message.Body, workServer.WorkServer.Addr, "new worker error. "+err.Error())
42 | return err
43 | }
44 | //向Worker发送数据
45 | response, err := w.Send(message)
46 | if err != nil {
47 | h.recordSubLog(logger.LOG_ERROR, message.ID, message.Body, workServer.WorkServer.Addr, "send worker error. "+err.Error())
48 | if h.isRequeueByError(err) {
49 | return err
50 | }
51 | }
52 | //返回值长度判断
53 | if len(response) < 3 {
54 | //约定返回值前3位为状态码，如：200 success
55 | errMsg := "response length less 3. response[" + strconv.Itoa(len(response)) + "]: " + string(response)
56 | h.recordSubLog(logger.LOG_WARNING, message.ID, message.Body, workServer.WorkServer.Addr, errMsg)
57 | //是否需要重新入队
58 | if h.consumeConfig.IsRequeue {
59 | return errors.New(errMsg)
60 | } else {
61 | return nil
62 | }
63 | }
64 | codeString := string(response[0:3])
65 | //返回值判断
66 | if codeString == WorkResponseSuccess {
67 | return nil
68 | }
69 | errMsg := "response code error. code: " + codeString + ", content: " + string(response)
70 | h.recordSubLog(logger.LOG_WARNING, message.ID, message.Body, workServer.WorkServer.Addr, errMsg)
71 | //是否需要重新入队
72 | if !h.consumeConfig.IsRequeue {
73 | return nil
74 | }
75 | return errors.New(errMsg)
76 | }
77 |
78 | //是否重新入队，根据error类型和配置决定
79 | //连不上worker直接重新入队，write失败直接重新入队，read失败根据配置决定是否重新入队
80 | func (h *Handler) isRequeueByError(err error) bool {
81 | if worker.IsErrorConnect(err) {
82 | return true
83 | } else if worker.IsErrorWrite(err) {
84 | return true
85 | } else if worker.IsErrorRead(err) && h.consumeConfig.IsRequeue {
86 | return true
87 | }
88 | return false
89 | }
90 |
91 | // 处理失败时会调用此方法
92 | // 连续失败五次次，第六次时在go-nsq客户端giving up时会执行
93 | func (h *Handler) LogFailedMessage(message nsq.Message) {
94 | h.recordSubLog(logger.LOG_FATAL, message.ID, message.Body, "", "LogFailedMessage giving up")
95 | }
96 |
97 | //封装写log方法，上面代码太啰嗦
98 | func (h *Handler) recordSubLog(logLevel logger.LogLevel, messageId nsq.MessageID, messageBody []byte, workAddr, errMsg string) {
99 | if len(workAddr) <= 0 {
100 | workAddr = "null"
101 | }
102 | if len(errMsg) <= 0 {
103 | errMsg = "nil"
104 | }
105 | config.SystemConfig.SubLogger.WithLevelf(logLevel, "[%s/%s] nsqproxy %s %s messageid:%s messagebody:%s",
106 | h.consumeConfig.Topic, h.consumeConfig.Channel,
107 | workAddr, errMsg, string(messageId[:]), string(messageBody),
108 | )
109 | }
110 |
--------------------------------------------------------------------------------
/web/vue-admin/vue.config.js:
--------------------------------------------------------------------------------
1 | 'use strict'
2 | const path = require('path')
3 | const defaultSettings = require('./src/settings.js')
4 |
5 | function resolve(dir) {
6 | return path.join(__dirname, dir)
7 | }
8 |
9 | const name = defaultSettings.title || 'vue Admin Template' // page title
10 |
11 | // If your port is set to 80,
12 | // use administrator privileges to execute the command line.
13 | // For example, Mac: sudo npm run
14 | // You can change the port by the following methods:
15 | // port = 9528 npm run dev OR npm run dev --port = 9528
16 | const port = process.env.port || process.env.npm_config_port || 9528 // dev port
17 |
18 | // All configuration item explanations can be find in https://cli.vuejs.org/config/
19 | module.exports = {
20 | publicPath: '/admin',
21 | outputDir: 'dist',
22 | assetsDir: 'static',
23 | lintOnSave: process.env.NODE_ENV === 'development',
24 | productionSourceMap: false,
25 | devServer: {
26 | port: port,
27 | open: true,
28 | overlay: {
29 | warnings: false,
30 | errors: true
31 | },
32 | proxy: {
33 | '/admin/api': {
34 | target: 'http://localhost:19421',
35 | changeOrigin: true
36 | }
37 | }
38 | },
39 | configureWebpack: {
40 | name: name,
41 | resolve: {
42 | alias: {
43 | '@': resolve('src')
44 | }
45 | }
46 | },
47 | chainWebpack(config) {
48 | config.plugin('preload').tap(() => [
49 | {
50 | rel: 'preload',
51 | fileBlacklist: [/\.map$/, /hot-update\.js$/, /runtime\..*\.js$/],
52 | include: 'initial'
53 | }
54 | ])
55 |
56 | config.plugins.delete('prefetch')
57 |
58 | config.module
59 | .rule('svg')
60 | .exclude.add(resolve('src/icons'))
61 | .end()
62 | config.module
63 | .rule('icons')
64 | .test(/\.svg$/)
65 | .include.add(resolve('src/icons'))
66 | .end()
67 | .use('svg-sprite-loader')
68 | .loader('svg-sprite-loader')
69 | .options({
70 | symbolId: 'icon-[name]'
71 | })
72 | .end()
73 |
74 | config
75 | .when(process.env.NODE_ENV !== 'development',
76 | config => {
77 | config
78 | .plugin('ScriptExtHtmlWebpackPlugin')
79 | .after('html')
80 | .use('script-ext-html-webpack-plugin', [{
81 | // `runtime` must same as runtimeChunk name. default is `runtime`
82 | inline: /runtime\..*\.js$/
83 | }])
84 | .end()
85 | config
86 | .optimization.splitChunks({
87 | chunks: 'all',
88 | cacheGroups: {
89 | libs: {
90 | name: 'chunk-libs',
91 | test: /[\\/]node_modules[\\/]/,
92 | priority: 10,
93 | chunks: 'initial' // only package third parties that are initially dependent
94 | },
95 | elementUI: {
96 | name: 'chunk-elementUI', // split elementUI into a single package
97 | priority: 20, // the weight needs to be larger than libs and app or it will be packaged into libs or app
98 | test: /[\\/]node_modules[\\/]_?element-ui(.*)/ // in order to adapt to cnpm
99 | },
100 | commons: {
101 | name: 'chunk-commons',
102 | test: resolve('src/components'), // can customize your rules
103 | minChunks: 3, // minimum common number
104 | priority: 5,
105 | reuseExistingChunk: true
106 | }
107 | }
108 | })
109 | // https:// webpack.js.org/configuration/optimization/#optimizationruntimechunk
110 | config.optimization.runtimeChunk('single')
111 | }
112 | )
113 | }
114 | }
115 |
--------------------------------------------------------------------------------
/internal/model/workserver.go:
--------------------------------------------------------------------------------
1 | package model
2 |
3 | import (
4 | "errors"
5 | "sync/atomic"
6 | "time"
7 | )
8 |
9 | //worker当前可用
10 | const workserverStatusAvailable = 1
11 |
12 | //worker当前不可用
13 | const workserverStatusUnavailable = 0
14 |
15 | type WorkServer struct {
16 | Id int `json:"id" gorm:"primaryKey"`
17 | //地址，IP:PORT
18 | Addr string `json:"addr"`
19 | //协议，如HTTP、FastCGI、CBNSQ
20 | Protocol string `json:"protocol"`
21 | //扩展字段
22 | Extra string `json:"extra"`
23 | //描述
24 | Description string `json:"description"`
25 | //责任人
26 | Owner string `json:"owner"`
27 | //是否有效
28 | Invalid int `json:"invalid"`
29 | //创建时间
30 | CreatedAt time.Time `json:"createdAt"`
31 | //更新时间
32 | UpdatedAt time.Time `json:"updatedAt"`
33 |
34 | status int32 `gorm:"-"`
35 | }
36 |
37 | func (WorkServer) TableName() string {
38 | return "nsqproxy_work_server"
39 | }
40 |
41 | func (WorkServer) CreateTable() error {
42 | sql := "CREATE TABLE IF NOT EXISTS `nsqproxy_work_server` (" +
43 | "`id` int(11) unsigned NOT NULL AUTO_INCREMENT," +
44 | "`addr` varchar(255) NOT NULL DEFAULT '' COMMENT '地址'," +
45 | "`protocol` varchar(11) DEFAULT 'CBNSQ' COMMENT '使用的协议，支持HTTP、FastCGI、CBNSQ'," +
46 | "`extra` varchar(1000) DEFAULT NULL COMMENT '扩展字段，比如协议是FastCGI时，需要传入PHP-FPM的执行的PHP文件的路径'," +
47 | "`description` varchar(1000) DEFAULT '' COMMENT '描述'," +
48 | "`owner` varchar(12) DEFAULT NULL COMMENT '责任人'," +
49 | "`invalid` tinyint(4) DEFAULT '0' COMMENT '是否有效，0是有效'," +
50 | "`created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '添加时间'," +
51 | "`updated_at` timestamp NULL DEFAULT NULL COMMENT '更新时间'," +
52 | "PRIMARY KEY (`id`)," +
53 | "UNIQUE KEY `index_uq_addr` (`addr`)" +
54 | ") ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COMMENT='所有的可以消费的服务器列表';"
55 | return db.Exec(sql).Error
56 | }
57 |
58 | //两份配置是否相等
59 | func (w WorkServer) IsEqual(newWork WorkServer) bool {
60 | if w.Id != newWork.Id || w.Addr != newWork.Addr || w.Protocol != newWork.Protocol || w.Extra != newWork.Extra || w.Description != newWork.Description {
61 | return false
62 | }
63 | if w.Owner != newWork.Owner || w.Invalid != newWork.Invalid {
64 | return false
65 | }
66 | return true
67 | }
68 |
69 | func (w WorkServer) GetStatus() {
70 | atomic.LoadInt32(&w.status)
71 | }
72 |
73 | func (w *WorkServer) SetStatusAvailable() {
74 | atomic.StoreInt32(&w.status, workserverStatusAvailable)
75 | }
76 |
77 | func (w *WorkServer) SetStatusUnAvailable() {
78 | atomic.StoreInt32(&w.status, workserverStatusUnavailable)
79 | }
80 |
81 | func (w *WorkServer) Create() (int, error) {
82 | result := db.Create(w)
83 | if result.Error != nil {
84 | return 0, result.Error
85 | } else if result.RowsAffected <= 0 {
86 | return 0, errors.New("RowsAffected is zero")
87 | } else if w.Id <= 0 {
88 | return 0, errors.New("primaryKey is zero")
89 | }
90 | return w.Id, nil
91 | }
92 |
93 | func (w *WorkServer) Delete() (int64, error) {
94 | if w.Id <= 0 {
95 | return 0, errors.New("primaryKey is zero")
96 | }
97 | result := db.Delete(w, w.Id)
98 | return result.RowsAffected, result.Error
99 | }
100 |
101 | func (w *WorkServer) Update() (int64, error) {
102 | if w.Id <= 0 {
103 | return 0, errors.New("primaryKey is zero")
104 | }
105 | result := db.Select("Id", "Addr", "Protocol", "Extra", "Description", "Owner", "Invalid", "UpdatedAt").Updates(w)
106 | if result.Error != nil {
107 | return 0, result.Error
108 | }
109 | return result.RowsAffected, nil
110 | }
111 |
112 | func (w *WorkServer) Get() (int64, error) {
113 | if w.Id <= 0 {
114 | return 0, errors.New("primaryKey is zero")
115 | }
116 | result := db.First(w)
117 | return result.RowsAffected, result.Error
118 | }
119 |
120 | func (w *WorkServer) Page(page int) (PageResult, error) {
121 | var wList []WorkServer
122 | d := db.Table(w.TableName())
123 | //count部分
124 | var total int64
125 | result := d.Count(&total)
126 | if result.Error != nil || result.RowsAffected != 1 {
127 | total = 0
128 | }
129 | //page部分
130 | if page <= 0 {
131 | page = 1
132 | }
133 | result = d.Offset((page - 1) * 20).Limit(20).Find(&wList)
134 | pageRet := PageResult{
135 | Total: total,
136 | Page: page,
137 | Result: wList,
138 | }
139 | return pageRet, result.Error
140 | }
141 |
142 | func (w *WorkServer) All() ([]WorkServer, error) {
143 | var wList []WorkServer
144 | result := db.Find(&wList)
145 | return wList, result.Error
146 | }
147 |
--------------------------------------------------------------------------------
/internal/httper/consumeservermap.go:
--------------------------------------------------------------------------------
1 | package httper
2 |
3 | import (
4 | "github.com/changba/nsqproxy/internal/model"
5 | "net/http"
6 | "strconv"
7 | "strings"
8 | )
9 |
10 | type ConsumeServerMap struct {
11 | }
12 |
13 | func (ConsumeServerMap) Create(w http.ResponseWriter, r *http.Request) {
14 | consumeid, err := strconv.Atoi(r.FormValue("consumeid"))
15 | if err != nil || consumeid <= 0 {
16 | Failed(w, HttpCodeBadRequest, "param error. consumeid is required")
17 | return
18 | }
19 | weight, err := strconv.Atoi(r.FormValue("weight"))
20 | if err != nil || weight <= 0 {
21 | Failed(w, HttpCodeBadRequest, "param error. weight is required")
22 | return
23 | }
24 | invalid, err := strconv.Atoi(r.FormValue("invalid"))
25 | if err != nil || invalid <= 0 || (invalid != model.InvalidAvailable && invalid != model.InvalidUnavailable) {
26 | invalid = model.InvalidAvailable
27 | }
28 | //支持多选，逗号分割
29 | if len(r.FormValue("serverid")) < 0 {
30 | Failed(w, HttpCodeBadRequest, "param error. serverid is required")
31 | return
32 | }
33 | serveridStrList := strings.Split(r.FormValue("serverid"), ",")
34 | csMapList := make([]*model.ConsumeServerMap, 0)
35 | for _, serveridStr := range serveridStrList {
36 | serverid, err := strconv.Atoi(serveridStr)
37 | if err != nil || serverid <= 0 {
38 | Failed(w, HttpCodeBadRequest, "param error. serverid must be int")
39 | return
40 | }
41 | csMap := &model.ConsumeServerMap{
42 | Consumeid: consumeid,
43 | Serverid: serverid,
44 | Weight: weight,
45 | Invalid: invalid,
46 | }
47 | id, err := csMap.Create()
48 | if err != nil {
49 | Failed(w, HttpCodeBadRequest, "create failed. err: "+err.Error())
50 | return
51 | }
52 | if id <= 0 {
53 | Failed(w, HttpCodeBadRequest, "id is zero")
54 | return
55 | }
56 | csMapList = append(csMapList, csMap)
57 | }
58 | Success(w, csMapList)
59 | }
60 |
61 | func (ConsumeServerMap) Delete(w http.ResponseWriter, r *http.Request) {
62 | csMap := &model.ConsumeServerMap{}
63 | var err error
64 | csMap.Id, err = strconv.Atoi(r.FormValue("id"))
65 | if err != nil || csMap.Id <= 0 {
66 | Failed(w, HttpCodeBadRequest, "param error")
67 | return
68 | }
69 | _, err = csMap.Delete()
70 | if err != nil {
71 | Failed(w, HttpCodeBadRequest, "delete failed. err: "+err.Error())
72 | return
73 | }
74 | Success(w, "ok")
75 | }
76 |
77 | func (ConsumeServerMap) Update(w http.ResponseWriter, r *http.Request) {
78 | csMap := &model.ConsumeServerMap{}
79 | var err error
80 | csMap.Id, err = strconv.Atoi(r.FormValue("id"))
81 | if err != nil || csMap.Id <= 0 {
82 | Failed(w, HttpCodeBadRequest, "param error")
83 | return
84 | }
85 | csMap.Consumeid, err = strconv.Atoi(r.FormValue("consumeid"))
86 | if err != nil || csMap.Consumeid <= 0 {
87 | Failed(w, HttpCodeBadRequest, "param error. consumeid is required")
88 | return
89 | }
90 | csMap.Weight, err = strconv.Atoi(r.FormValue("weight"))
91 | if err != nil || csMap.Weight <= 0 {
92 | Failed(w, HttpCodeBadRequest, "param error. weight is required")
93 | return
94 | }
95 | csMap.Serverid, err = strconv.Atoi(r.FormValue("serverid"))
96 | if err != nil || csMap.Serverid <= 0 {
97 | Failed(w, HttpCodeBadRequest, "param error. serverid is required")
98 | return
99 | }
100 | invalid, err := strconv.Atoi(r.FormValue("invalid"))
101 | if err != nil || invalid <= 0 || (invalid != model.InvalidAvailable && invalid != model.InvalidUnavailable) {
102 | invalid = model.InvalidAvailable
103 | }
104 | csMap.Invalid = invalid
105 |
106 | _, err = csMap.Update()
107 | if err != nil {
108 | Failed(w, HttpCodeBadRequest, "upload failed. err: "+err.Error())
109 | return
110 | }
111 | Success(w, "ok")
112 | }
113 |
114 | func (ConsumeServerMap) GetWork(w http.ResponseWriter, r *http.Request) {
115 | csMap := &model.ConsumeServerMap{}
116 | var err error
117 | csMap.Id, err = strconv.Atoi(r.FormValue("id"))
118 | if err != nil || csMap.Id <= 0 {
119 | Failed(w, HttpCodeBadRequest, "param error")
120 | return
121 | }
122 | n, err := csMap.Get()
123 | if n == 0 || csMap.Id <= 0 {
124 | Failed(w, HttpCodeNotFound, "not found")
125 | return
126 | }
127 | if csMap.Serverid <= 0 {
128 | Failed(w, HttpCodeBadRequest, "work.Id is empty")
129 | return
130 | }
131 | work := model.WorkServer{}
132 | work.Id = csMap.Serverid
133 | n, err = work.Get()
134 | if err != nil || n == 0 || work.Id <= 0 {
135 | Failed(w, HttpCodeNotFound, "work not found")
136 | return
137 | }
138 | csMap.WorkServer = work
139 | Success(w, csMap)
140 | }
141 |
--------------------------------------------------------------------------------
/web/vue-admin/src/styles/sidebar.scss:
--------------------------------------------------------------------------------
1 | #app {
2 |
3 | .main-container {
4 | min-height: 100%;
5 | transition: margin-left .28s;
6 | margin-left: $sideBarWidth;
7 | position: relative;
8 | }
9 |
10 | .sidebar-container {
11 | transition: width 0.28s;
12 | width: $sideBarWidth !important;
13 | background-color: $menuBg;
14 | height: 100%;
15 | position: fixed;
16 | font-size: 0px;
17 | top: 0;
18 | bottom: 0;
19 | left: 0;
20 | z-index: 1001;
21 | overflow: hidden;
22 |
23 | // reset element-ui css
24 | .horizontal-collapse-transition {
25 | transition: 0s width ease-in-out, 0s padding-left ease-in-out, 0s padding-right ease-in-out;
26 | }
27 |
28 | .scrollbar-wrapper {
29 | overflow-x: hidden !important;
30 | }
31 |
32 | .el-scrollbar__bar.is-vertical {
33 | right: 0px;
34 | }
35 |
36 | .el-scrollbar {
37 | height: 100%;
38 | }
39 |
40 | &.has-logo {
41 | .el-scrollbar {
42 | height: calc(100% - 50px);
43 | }
44 | }
45 |
46 | .is-horizontal {
47 | display: none;
48 | }
49 |
50 | a {
51 | display: inline-block;
52 | width: 100%;
53 | overflow: hidden;
54 | }
55 |
56 | .svg-icon {
57 | margin-right: 16px;
58 | }
59 |
60 | .sub-el-icon {
61 | margin-right: 12px;
62 | margin-left: -2px;
63 | }
64 |
65 | .el-menu {
66 | border: none;
67 | height: 100%;
68 | width: 100% !important;
69 | }
70 |
71 | // menu hover
72 | .submenu-title-noDropdown,
73 | .el-submenu__title {
74 | &:hover {
75 | background-color: $menuHover !important;
76 | }
77 | }
78 |
79 | .is-active>.el-submenu__title {
80 | color: $subMenuActiveText !important;
81 | }
82 |
83 | & .nest-menu .el-submenu>.el-submenu__title,
84 | & .el-submenu .el-menu-item {
85 | min-width: $sideBarWidth !important;
86 | background-color: $subMenuBg !important;
87 |
88 | &:hover {
89 | background-color: $subMenuHover !important;
90 | }
91 | }
92 | }
93 |
94 | .hideSidebar {
95 | .sidebar-container {
96 | width: 54px !important;
97 | }
98 |
99 | .main-container {
100 | margin-left: 54px;
101 | }
102 |
103 | .submenu-title-noDropdown {
104 | padding: 0 !important;
105 | position: relative;
106 |
107 | .el-tooltip {
108 | padding: 0 !important;
109 |
110 | .svg-icon {
111 | margin-left: 20px;
112 | }
113 |
114 | .sub-el-icon {
115 | margin-left: 19px;
116 | }
117 | }
118 | }
119 |
120 | .el-submenu {
121 | overflow: hidden;
122 |
123 | &>.el-submenu__title {
124 | padding: 0 !important;
125 |
126 | .svg-icon {
127 | margin-left: 20px;
128 | }
129 |
130 | .sub-el-icon {
131 | margin-left: 19px;
132 | }
133 |
134 | .el-submenu__icon-arrow {
135 | display: none;
136 | }
137 | }
138 | }
139 |
140 | .el-menu--collapse {
141 | .el-submenu {
142 | &>.el-submenu__title {
143 | &>span {
144 | height: 0;
145 | width: 0;
146 | overflow: hidden;
147 | visibility: hidden;
148 | display: inline-block;
149 | }
150 | }
151 | }
152 | }
153 | }
154 |
155 | .el-menu--collapse .el-menu .el-submenu {
156 | min-width: $sideBarWidth !important;
157 | }
158 |
159 | // mobile responsive
160 | .mobile {
161 | .main-container {
162 | margin-left: 0px;
163 | }
164 |
165 | .sidebar-container {
166 | transition: transform .28s;
167 | width: $sideBarWidth !important;
168 | }
169 |
170 | &.hideSidebar {
171 | .sidebar-container {
172 | pointer-events: none;
173 | transition-duration: 0.3s;
174 | transform: translate3d(-$sideBarWidth, 0, 0);
175 | }
176 | }
177 | }
178 |
179 | .withoutAnimation {
180 |
181 | .main-container,
182 | .sidebar-container {
183 | transition: none;
184 | }
185 | }
186 | }
187 |
188 | // when menu collapsed
189 | .el-menu--vertical {
190 | &>.el-menu {
191 | .svg-icon {
192 | margin-right: 16px;
193 | }
194 | .sub-el-icon {
195 | margin-right: 12px;
196 | margin-left: -2px;
197 | }
198 | }
199 |
200 | .nest-menu .el-submenu>.el-submenu__title,
201 | .el-menu-item {
202 | &:hover {
203 | // you can use $subMenuHover
204 | background-color: $menuHover !important;
205 | }
206 | }
207 |
208 | // the scroll bar appears when the subMenu is too long
209 | >.el-menu--popup {
210 | max-height: 100vh;
211 | overflow-y: auto;
212 |
213 | &::-webkit-scrollbar-track-piece {
214 | background: #d3dce6;
215 | }
216 |
217 | &::-webkit-scrollbar {
218 | width: 6px;
219 | }
220 |
221 | &::-webkit-scrollbar-thumb {
222 | background: #99a9bf;
223 | border-radius: 20px;
224 | }
225 | }
226 | }
227 |
--------------------------------------------------------------------------------
/web/vue-admin/.eslintrc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | root: true,
3 | parserOptions: {
4 | parser: 'babel-eslint',
5 | sourceType: 'module'
6 | },
7 | env: {
8 | browser: true,
9 | node: true,
10 | es6: true,
11 | },
12 | extends: ['plugin:vue/recommended', 'eslint:recommended'],
13 |
14 | // add your custom rules here
15 | //it is base on https://github.com/vuejs/eslint-config-vue
16 | rules: {
17 | "vue/max-attributes-per-line": [2, {
18 | "singleline": 10,
19 | "multiline": {
20 | "max": 1,
21 | "allowFirstLine": false
22 | }
23 | }],
24 | "vue/singleline-html-element-content-newline": "off",
25 | "vue/multiline-html-element-content-newline":"off",
26 | "vue/name-property-casing": ["error", "PascalCase"],
27 | "vue/no-v-html": "off",
28 | 'accessor-pairs': 2,
29 | 'arrow-spacing': [2, {
30 | 'before': true,
31 | 'after': true
32 | }],
33 | 'block-spacing': [2, 'always'],
34 | 'brace-style': [2, '1tbs', {
35 | 'allowSingleLine': true
36 | }],
37 | 'camelcase': [0, {
38 | 'properties': 'always'
39 | }],
40 | 'comma-dangle': [2, 'never'],
41 | 'comma-spacing': [2, {
42 | 'before': false,
43 | 'after': true
44 | }],
45 | 'comma-style': [2, 'last'],
46 | 'constructor-super': 2,
47 | 'curly': [2, 'multi-line'],
48 | 'dot-location': [2, 'property'],
49 | 'eol-last': 2,
50 | 'eqeqeq': ["error", "always", {"null": "ignore"}],
51 | 'generator-star-spacing': [2, {
52 | 'before': true,
53 | 'after': true
54 | }],
55 | 'handle-callback-err': [2, '^(err|error)$'],
56 | 'indent': [2, 2, {
57 | 'SwitchCase': 1
58 | }],
59 | 'jsx-quotes': [2, 'prefer-single'],
60 | 'key-spacing': [2, {
61 | 'beforeColon': false,
62 | 'afterColon': true
63 | }],
64 | 'keyword-spacing': [2, {
65 | 'before': true,
66 | 'after': true
67 | }],
68 | 'new-cap': [2, {
69 | 'newIsCap': true,
70 | 'capIsNew': false
71 | }],
72 | 'new-parens': 2,
73 | 'no-array-constructor': 2,
74 | 'no-caller': 2,
75 | 'no-console': 'off',
76 | 'no-class-assign': 2,
77 | 'no-cond-assign': 2,
78 | 'no-const-assign': 2,
79 | 'no-control-regex': 0,
80 | 'no-delete-var': 2,
81 | 'no-dupe-args': 2,
82 | 'no-dupe-class-members': 2,
83 | 'no-dupe-keys': 2,
84 | 'no-duplicate-case': 2,
85 | 'no-empty-character-class': 2,
86 | 'no-empty-pattern': 2,
87 | 'no-eval': 2,
88 | 'no-ex-assign': 2,
89 | 'no-extend-native': 2,
90 | 'no-extra-bind': 2,
91 | 'no-extra-boolean-cast': 2,
92 | 'no-extra-parens': [2, 'functions'],
93 | 'no-fallthrough': 2,
94 | 'no-floating-decimal': 2,
95 | 'no-func-assign': 2,
96 | 'no-implied-eval': 2,
97 | 'no-inner-declarations': [2, 'functions'],
98 | 'no-invalid-regexp': 2,
99 | 'no-irregular-whitespace': 2,
100 | 'no-iterator': 2,
101 | 'no-label-var': 2,
102 | 'no-labels': [2, {
103 | 'allowLoop': false,
104 | 'allowSwitch': false
105 | }],
106 | 'no-lone-blocks': 2,
107 | 'no-mixed-spaces-and-tabs': 2,
108 | 'no-multi-spaces': 2,
109 | 'no-multi-str': 2,
110 | 'no-multiple-empty-lines': [2, {
111 | 'max': 1
112 | }],
113 | 'no-native-reassign': 2,
114 | 'no-negated-in-lhs': 2,
115 | 'no-new-object': 2,
116 | 'no-new-require': 2,
117 | 'no-new-symbol': 2,
118 | 'no-new-wrappers': 2,
119 | 'no-obj-calls': 2,
120 | 'no-octal': 2,
121 | 'no-octal-escape': 2,
122 | 'no-path-concat': 2,
123 | 'no-proto': 2,
124 | 'no-redeclare': 2,
125 | 'no-regex-spaces': 2,
126 | 'no-return-assign': [2, 'except-parens'],
127 | 'no-self-assign': 2,
128 | 'no-self-compare': 2,
129 | 'no-sequences': 2,
130 | 'no-shadow-restricted-names': 2,
131 | 'no-spaced-func': 2,
132 | 'no-sparse-arrays': 2,
133 | 'no-this-before-super': 2,
134 | 'no-throw-literal': 2,
135 | 'no-trailing-spaces': 2,
136 | 'no-undef': 2,
137 | 'no-undef-init': 2,
138 | 'no-unexpected-multiline': 2,
139 | 'no-unmodified-loop-condition': 2,
140 | 'no-unneeded-ternary': [2, {
141 | 'defaultAssignment': false
142 | }],
143 | 'no-unreachable': 2,
144 | 'no-unsafe-finally': 2,
145 | 'no-unused-vars': [2, {
146 | 'vars': 'all',
147 | 'args': 'none'
148 | }],
149 | 'no-useless-call': 2,
150 | 'no-useless-computed-key': 2,
151 | 'no-useless-constructor': 2,
152 | 'no-useless-escape': 0,
153 | 'no-whitespace-before-property': 2,
154 | 'no-with': 2,
155 | 'one-var': [2, {
156 | 'initialized': 'never'
157 | }],
158 | 'operator-linebreak': [2, 'after', {
159 | 'overrides': {
160 | '?': 'before',
161 | ':': 'before'
162 | }
163 | }],
164 | 'padded-blocks': [2, 'never'],
165 | 'quotes': [2, 'single', {
166 | 'avoidEscape': true,
167 | 'allowTemplateLiterals': true
168 | }],
169 | 'semi': [2, 'never'],
170 | 'semi-spacing': [2, {
171 | 'before': false,
172 | 'after': true
173 | }],
174 | 'space-before-blocks': [2, 'always'],
175 | 'space-before-function-paren': [2, 'never'],
176 | 'space-in-parens': [2, 'never'],
177 | 'space-infix-ops': 2,
178 | 'space-unary-ops': [2, {
179 | 'words': true,
180 | 'nonwords': false
181 | }],
182 | 'spaced-comment': [2, 'always', {
183 | 'markers': ['global', 'globals', 'eslint', 'eslint-disable', '*package', '!', ',']
184 | }],
185 | 'template-curly-spacing': [2, 'never'],
186 | 'use-isnan': 2,
187 | 'valid-typeof': 2,
188 | 'wrap-iife': [2, 'any'],
189 | 'yield-star-spacing': [2, 'both'],
190 | 'yoda': [2, 'never'],
191 | 'prefer-const': 2,
192 | 'no-debugger': process.env.NODE_ENV === 'production' ? 2 : 0,
193 | 'object-curly-spacing': [2, 'always', {
194 | objectsInObjects: false
195 | }],
196 | 'array-bracket-spacing': [2, 'never']
197 | }
198 | }
199 |
--------------------------------------------------------------------------------