├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── README_EN.md ├── build_exec.sh ├── build_image.sh ├── cmd └── opennotrd │ └── main.go ├── coverage.sh ├── doc ├── README.md ├── assets │ ├── httpdemo.jpg │ ├── logo.jpg │ ├── logo.png │ ├── opennotr.jpg │ ├── qrcode.jpg │ ├── tcpdemo.jpg │ └── udpdemo.jpg └── zh │ ├── opennotr技术实现细节.md │ └── opennotr技术架构.md ├── docker-build ├── Dockerfile ├── docker-compose.yaml └── start.sh ├── etc ├── opennotr.yaml └── opennotrd.yaml ├── example └── README.md ├── go.mod ├── go.sum ├── internal ├── logs │ ├── README.md │ ├── alils │ │ ├── alils.go │ │ ├── config.go │ │ ├── log.pb.go │ │ ├── log_config.go │ │ ├── log_project.go │ │ ├── log_store.go │ │ ├── machine_group.go │ │ ├── request.go │ │ └── signature.go │ ├── color.go │ ├── color_windows.go │ ├── color_windows_test.go │ ├── conn.go │ ├── conn_test.go │ ├── console.go │ ├── console_test.go │ ├── es │ │ └── es.go │ ├── file.go │ ├── file_test.go │ ├── jianliao.go │ ├── log.go │ ├── logger.go │ ├── logger_test.go │ ├── multifile.go │ ├── multifile_test.go │ ├── slack.go │ ├── smtp.go │ └── smtp_test.go └── proto │ └── cs.go ├── opennotr ├── client.go ├── config.go └── main.go └── opennotrd ├── core ├── config.go ├── dhcp.go ├── net.go ├── resolver.go ├── server.go ├── session.go ├── tcpforward.go ├── tcpforward_test.go └── udpforward.go ├── plugin ├── dummy │ └── dummy.go ├── plugin.go ├── restyproxy │ └── restyproxy.go ├── tcpproxy │ └── tcpproxy.go └── udpproxy │ └── udpproxy.go ├── plugins.go ├── run.go └── test ├── httpecho ├── httpclient.go └── httpserver.go ├── plugin_test.go ├── tcpecho ├── tcpclient.go └── tcpserver.go ├── udpecho ├── client.go └── server.go └── websocket ├── wsclient.go └── wsserver.go /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | *.log 3 | *_temp 4 | *.prof 5 | *.pprof 6 | *.svg 7 | *.txt 8 | *.out -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.13 4 | before_install: 5 | install: 6 | - go get github.com/ICKelin/opennotr/opennotrd 7 | - go get github.com/ICKelin/opennotr/opennotr 8 | before_script: 9 | script: 10 | - cd $HOME/gopath/src/github.com/ICKelin/opennotr 11 | - chmod +x coverage.sh 12 | # - ./coverage.sh 13 | - chmod +x build_exec.sh 14 | - ./build_exec.sh 15 | 16 | after_success: 17 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 ICKelin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 | 6 |

7 | 8 | 9 | 10 | 11 | go report 12 | 13 | 14 | 15 | Build Status 16 | 17 | 18 | license 19 | 20 |

21 | 22 | [English](README_EN.md) | 简体中文 23 | 24 | ## 介绍 25 | 26 | **状态: Stable** 27 | 28 | **[在线试用](https://github.com/ICKelin/opennotr/wiki/opennotr%E5%9C%A8%E7%BA%BF%E4%BD%93%E9%AA%8C)** 29 | 30 | opennotr是一款开源的内网穿透软件，内部使用`透明代理`实现流量劫持，同时，基于虚拟IP构建虚拟局域网，整体技术与k8s的service技术类似，虚拟IP与service ip类似，只在opennotr的服务端所在的机器可以访问，在其他任何位置都无法访问该虚拟ip。 31 | 32 | opennotr支持多种协议，包括http，https，grpc，tcp，udp，为了实现http，https，grpc协议端口复用，opennotr引入了openresty作为网关，多个客户端不同域名可以共享http的80，https的443端口，不需要额外的端口。 33 | 34 | opennotr支持定制化插件，我们内置了http, https, grpc, tcp, udp代理，可以覆盖大部分场景，同时，opennotr允许自己开发协议插件，比如说，如果您希望使用apisix来作为前置的网关，您可以开发opennotr的apisix插件，opennotr会把一些基本信息传递给插件，其余功能均由插件自己进行。 35 | 36 | opennotr内置的协议也是以插件的形式存在的，只是默认导入到程序当中。 37 | 38 | ## 目录 39 | - [介绍](#介绍) 40 | - [功能特性](#功能特性) 41 | - [opennotr的技术原理](#opennotr的技术原理) 42 | - [如何开始使用](#如何开始使用) 43 | - [安装opennotrd](#安装opennotrd) 44 | - [运行opennotr](#运行opennotr) 45 | - [相关文章与视频](#相关文章和视频) 46 | - [插件开发](#插件开发) 47 | - [有问题怎么办](#有问题怎么办) 48 | - [关于作者](#关于作者) 49 | 50 | ## 功能特性 51 | 52 | - 支持多种协议，可覆盖大部分内网穿透场景 53 | - 支持定制化插件，opennotr提供一个虚拟局域网的基础功能，您开发的插件运行在这个局域网内 54 | - 引入openresty作为网关，网络处理性能问题可以得到保证 55 | - 支持动态域名解析，可支持引入coredns作为dns的nameserver。 56 | 57 | ## opennotr的技术原理 58 | 59 | ![opennotr.jpg](doc/assets/opennotr.jpg) 60 | 61 | 最下层是客户端所在的机器/设备，同时也作为虚拟局域网当中的一台机器，具备该局域网的IP地址`100.64.240.100` 62 | 63 | 最上层是服务端所在的机器，同时也作为虚拟局域网当中的网关，具备该局域网的IP地址`100.64.240.1`，opennotr提供能够在`Public Cloud`层可通过`100.64.240.100`访问`Device/PC behind Routed`的能力。在这个能力的基础之上构建整个内网穿透体系。 64 | 65 | 上图中，虚拟IP不存在任何网卡，只是内部虚拟出来的一个ip，便于流量劫持，opennotr通过透明代理，将访问虚拟ip的流量转到本机监听的端口，然后查找该虚拟ip对应的客户端长连接，将流量通过长连接下发到客户端，从而实现穿透。 66 | 67 | 上图中，openresty是基于我openresty开发的http网关，目的是支持动态upstream，可通过接口增删upstream，感兴趣的可以查看[resty-upstream](https://github.com/ICKelin/resty-upstream)，当有客户端连接上来时，会调用`resty-upstream`提供的接口新增配置，当客户端断开连接时，会调用`resty-upstream`提供的删除接口删除配置。 68 | 69 | 从整个系统的层面，openresty的upstream的地址为opennotr客户端分配的lan地址，至于如何与客户端lan地址通信，这是opennotr处理的事情，对于openresty而言是透明的。 70 | 71 | 上图中，针对tcp和udp，opennotr倒是没有使用openresty的功能，而是自己开发的代理程序，当前也是集成在opennotrd程序当中，具体可参考以下两个文件。 72 | 73 | - [tcpproxy.go](https://github.com/ICKelin/opennotr/blob/master/opennotrd/plugin/tcpproxy/tcpproxy.go) 74 | - [udpproxy.go](https://github.com/ICKelin/opennotr/blob/master/opennotrd/plugin/udpproxy/udpproxy.go) 75 | 76 | 不使用openresty处理tcp和udp主要基于以下考虑： 77 | 78 | - openresty本身的stream功能并不能满足要求，这里的场景是事先不知道需要监听哪个端口，而openresty的server配置块需要预先知道监听的端口，否则需要重写配置再reload。 79 | 80 | - openresty的stream模块和http是分割的，原本使用http接口实现的动态upstream针对tcp并不生效，因此需要使用lua的tcp编程接受请求，作者自身对lua和openresty了解不深，不贸贸然行动。 81 | 82 | - 不使用openresty构建tcp/udp代理难度并不大，http使用openresty是为了端口复用，根据host区分，而tcp则没有类似的机制，也就是说，即使用openresty，也是单独的端口。 83 | 84 | 后续有兴趣的同学可以考虑将openresty完全换成golang实现，或者将golang的tcp/udp部分换成openresty。笔者注意到市面上已经有类似的实现，比如[apache apisix](https://github.com/apache/apisix)是使用的openresty实现的http,tcp,udp网关，这个是很不错的项目，我的[resty-upstream](https://github.com/ICKelin/resty-upstream)项目大部分都是从该项目学习而来 85 | 86 | [返回目录](#目录) 87 | 88 | ## 如何使用 89 | 90 | opennotr包括服务端程序`opennotrd`和客户端程序`opennotr` 91 | 92 | ### 安装opennotrd 93 | 94 | **强烈建议使用docker运行opennotrd** 95 | 96 | 为了运行opennotrd，您需要准备好程序运行的配置文件和tls证书等信息，以下为笔者的一个示例 97 | 98 | ``` 99 | root@iZwz97kfjnf78copv1ae65Z:/opt/data/opennotrd# tree 100 | . 101 | |-- cert ---------------------> 证书，秘钥目录 102 | | |-- upstream.crt 103 | | `-- upstream.key 104 | `-- notrd.yaml ---------------> opennotrd启动的配置文件 105 | 106 | 2 directories, 5 files 107 | ``` 108 | 109 | 这里主要关注`notrd.yaml`文件的内容，该文件是opennotrd运行时的配置文件. 110 | 111 | ```yml 112 | server: 113 | listen: ":10100" 114 | authKey: "client server exchange key" 115 | domain: "open.notr.tech" 116 | 117 | tcpforward: 118 | listen: ":4398" 119 | 120 | udpforward: 121 | listen: ":4399" 122 | 123 | dhcp: 124 | cidr: "100.64.242.1/24" 125 | ip: "100.64.242.1" 126 | 127 | plugin: 128 | tcp: | 129 | {} 130 | 131 | udp: | 132 | {} 133 | 134 | http: | 135 | { 136 | "adminUrl": "http://127.0.0.1:81/upstreams" 137 | } 138 | 139 | https: | 140 | { 141 | "adminUrl": "http://127.0.0.1:81/upstreams" 142 | } 143 | 144 | h2c: | 145 | { 146 | "adminUrl": "http://127.0.0.1:81/upstreams" 147 | } 148 | ``` 149 | 150 | 大部分情况下，上述配置只需要调整`server`区块相关配置，其他保留默认即可。 151 | 152 | 准备好配置之后运行以下命令即可开始启动: 153 | 154 | `docker run --privileged --net=host -v /opt/logs/opennotr:/opt/resty-upstream/logs -v /opt/data/opennotrd:/opt/conf -d ickelin/opennotr:v1.0.4` 155 | 156 | 需要配置volume，将主机目录`/opt/data/opennotrd`挂载到容器的`/opt/conf`当中，`/opt/data/opennotrd`为在上一步当中创建的配置文件，证书目录。 157 | 158 | 或者您也可以使用docker-compose启动 159 | 160 | ``` 161 | wget https://github.com/ICKelin/opennotr/blob/master/docker-build/docker-compose.yaml 162 | 163 | docker-compose up -d opennotrd 164 | ``` 165 | 166 | ### 运行opennotr 167 | opennotr的启动比较简单，首先需要准备配置. 168 | 169 | ```yaml 170 | serverAddr: "demo.notr.tech:10100" 171 | key: "client server exchange key" 172 | domain: "cloud.dahuizong.com" 173 | 174 | # 转发表信息 175 | # ports字段解析: 公网端口: 本机监听的端口 176 | forwards: 177 | - protocol: tcp 178 | ports: 179 | 2222: 2222 180 | 181 | - protocol: udp 182 | ports: 183 | 53: 53 184 | 185 | - protocol: http 186 | ports: 187 | 0: 8080 188 | 189 | - protocol: https 190 | ports: 191 | 0: 8081 192 | 193 | - protocol: h2c 194 | ports: 195 | 0: 50052 196 | 197 | ``` 198 | 199 | 配置准备好之后，启动opennotr即可 200 | 201 | `./opennotr -conf config.yaml` 202 | 203 | 204 | [返回目录](#目录) 205 | 206 | ### 相关文章和视频 207 | 208 | - [opennotr基本用法](https://www.zhihu.com/zvideo/1348958178885963776) 209 | - [opennotr进阶-使用域名](https://www.zhihu.com/zvideo/1357007720181293056) 210 | 211 | ## 插件开发 212 | 要开发opennotr支持的插件，您需要实现以下接口: 213 | 214 | ```golang 215 | 216 | // IPlugin defines plugin interface 217 | // Plugin should implements the IPlugin 218 | type IPlugin interface { 219 | // Setup calls at the begin of plugin system initialize 220 | // plugin system will pass the raw message to plugin's Setup function 221 | Setup(json.RawMessage) error 222 | 223 | // Close a proxy, it may be called by client's connection close 224 | StopProxy(item *PluginMeta) 225 | 226 | // Run a proxy, it may be called by client's connection established 227 | RunProxy(item *PluginMeta) error 228 | } 229 | 230 | ``` 231 | 232 | `Setup`函数负责初始化您的插件，由插件管理程序调用，开发者无需手动调用，参数是插件运行需要的配置，格式为`json`格式 233 | 234 | `RunProxy`和`StopProxy`也是由我们插件管理程序调用的，需要由开发者自己实现。 235 | 236 | 实现以上三个接口之后，需要在插件当中调用注册函数，将插件注册到系统当中，比如: 237 | 238 | ```golang 239 | package tcpproxy 240 | 241 | import ( 242 | "encoding/json" 243 | "fmt" 244 | 245 | "github.com/ICKelin/opennotr/opennotrd/plugin" 246 | ) 247 | 248 | func init() { 249 | plugin.Register("tcp", &TCPProxy{}) 250 | } 251 | 252 | type TCPProxy struct{} 253 | 254 | func (t *TCPProxy) Setup(config json.RawMessage) error { return nil } 255 | 256 | func (t *TCPProxy) StopProxy(item *plugin.PluginMeta) {} 257 | 258 | func (t *TCPProxy) RunProxy(item *plugin.PluginMeta) error { 259 | return fmt.Errorf("TODO://") 260 | } 261 | 262 | ``` 263 | 264 | 最后，需要在`opennotrd/plugins.go`当中导入您的插件所在的包。 265 | 266 | ```golang 267 | import ( 268 | // plugin import 269 | _ "github.com/ICKelin/opennotr/opennotrd/plugin/tcpproxy" 270 | ) 271 | ``` 272 | 273 | 示例插件可参考[tcpproxy.go](https://github.com/ICKelin/opennotr/blob/master/opennotrd/plugin/tcpproxy/tcpproxy.go) 274 | 275 | [返回目录](#目录) 276 | 277 | ## 有问题怎么办 278 | 279 | - [wiki](https://github.com/ICKelin/opennotr/wiki) 280 | - [查看文档](https://github.com/ICKelin/opennotr/tree/master/doc) 281 | - [提交issue](https://github.com/ICKelin/opennotr/issues) 282 | - [查看源码](https://github.com/ICKelin/opennotr) 283 | - [联系作者交流解决](#关于作者) 284 | 285 | 286 | [返回目录](#目录) 287 | 288 | ## 关于作者 289 | 一个爱好编程的人，网名叫ICKelin。对于以下任何问题，包括 290 | 291 | - 项目实现细节 292 | - 项目使用问题 293 | - 项目建议，代码问题 294 | - 案例分享 295 | - 技术交流 296 | 297 | 可加微信: zyj995139094 -------------------------------------------------------------------------------- /README_EN.md: -------------------------------------------------------------------------------- 1 | ## opennotr 2 | [![Build Status](https://travis-ci.org/ICKelin/opennotr.svg?branch=master)](https://travis-ci.org/ICKelin/opennotr) 3 | [![Go Report Card](https://goreportcard.com/badge/github.com/ICKelin/opennotr)](https://goreportcard.com/report/github.com/ICKelin/opennotr) 4 | 5 | opennotr is a nat tranversal application base on `tproxy` and openresty. 6 | 7 | opennotr provides http, https, grpc, tcp and udp nat traversal. For http, https, grpc, opennotr supports multi client share the 80/443 ports, it maybe useful for wechat, facebook webhook debug. 8 | 9 | **Status: Stable** 10 | 11 | The technical architecture of opennotr 12 | 13 | ![opennotr](doc/assets/opennotr.jpg) 14 | 15 | Table of Contents 16 | ================= 17 | - [Features](#Features) 18 | - [Build](#build) 19 | - [Install](#Install) 20 | - [Plugin](#Plugin) 21 | - [Technology details](#Technology-details) 22 | - [Author](#Author) 23 | 24 | Features 25 | ========= 26 | opennotr provides these features: 27 | 28 | - Supports multi protocol, http, https, grpc, tcp, udp. 29 | - Multi client shares the same http, https, grpc port, for example: client A use `a.notr.tech` domain, client B use `b.notr.tech`, they can both use 80 port for http. Opennotr use openresty for dynamic upstream. 30 | - Dynamic dns support, opennotr use coredns and etcd for dynamic dns. 31 | - Support plugin 32 | 33 | [Back to TOC](#table-of-contents) 34 | 35 | Build 36 | ===== 37 | 38 | **Build binary:** 39 | 40 | `./build_exec.sh` 41 | 42 | The binary file will created in bin folder. 43 | 44 | **Build docker image:** 45 | 46 | `./build_image.sh` 47 | 48 | This scripts will run `build_exec.sh` and build an image name `opennotr` 49 | 50 | [Back to TOC](#table-of-contents) 51 | 52 | Install 53 | ========= 54 | 55 | **Install via docker-compose** 56 | 57 | 1. create configuration file 58 | 59 | `mkdir /opt/data/opennotrd` 60 | 61 | An example of configuration folder tree is: 62 | 63 | ``` 64 | root@iZwz97kfjnf78copv1ae65Z:/opt/data/opennotrd# tree 65 | . 66 | |-- cert ---------------------> cert folder 67 | | |-- upstream.crt 68 | | `-- upstream.key 69 | `-- notrd.yaml ---------------> opennotr config file 70 | 71 | 2 directories, 5 files 72 | ``` 73 | 74 | the cert folder MUST be created and the crt and key file MUST created too. 75 | 76 | ```yml 77 | server: 78 | listen: ":10100" 79 | authKey: "client server exchange key" 80 | domain: "open.notr.tech" 81 | tcplisten: ":4398" 82 | udplisten: ":4399" 83 | 84 | dhcp: 85 | cidr: "100.64.242.1/24" 86 | ip: "100.64.242.1" 87 | 88 | plugin: 89 | tcp: | 90 | { 91 | "PortMin": 10000, 92 | "PortMax": 20000 93 | } 94 | 95 | udp: | 96 | { 97 | "PortMin": 20000, 98 | "PortMax": 30000 99 | } 100 | 101 | http: | 102 | { 103 | "adminUrl": "http://127.0.0.1:81/upstreams" 104 | } 105 | 106 | https: | 107 | { 108 | "adminUrl": "http://127.0.0.1:81/upstreams" 109 | } 110 | 111 | h2c: | 112 | { 113 | "adminUrl": "http://127.0.0.1:81/upstreams" 114 | } 115 | ``` 116 | 117 | the only one configuration item you should change is `domain: "open.notr.tech"`, replace `open.notr.tech` with your own domain. 118 | 119 | 2. Run with docker 120 | 121 | `docker run --privileged --net=host -v /opt/logs/opennotr:/opt/resty-upstream/logs -v /opt/data/opennotrd:/opt/conf -d opennotr` 122 | 123 | Or use docker-compose 124 | 125 | 126 | ``` 127 | wget https://github.com/ICKelin/opennotr/blob/develop/docker-build/docker-compose.yaml 128 | 129 | docker-compose up -d opennotrd 130 | ``` 131 | 132 | **Run opennotr client** 133 | 134 | prepare config file`config.yaml` 135 | ```yaml 136 | serverAddr: "demo.notr.tech:10100" 137 | key: "client server exchange key" 138 | domain: "cloud.dahuizong.com" 139 | 140 | # forward table 141 | forwards: 142 | - protocol: tcp 143 | ports: 144 | # public port: local port 145 | 2222: 2222 146 | 147 | - protocol: udp 148 | ports: 149 | 53: 53 150 | 151 | - protocol: http 152 | ports: 153 | 0: 8080 154 | 155 | - protocol: https 156 | ports: 157 | 0: 8081 158 | 159 | - protocol: h2c 160 | ports: 161 | 0: 50052 162 | 163 | ``` 164 | 165 | and then you can run the opennotr client using `./opennotr -conf config.yaml` command 166 | 167 | [Back to TOC](#table-of-contents) 168 | 169 | Plugin 170 | ======= 171 | opennotr provide plugin interface for developer, Yes, tcp and udp are buildin plugins. 172 | 173 | For a new plugin, you should implement the IPlugin interface which contains RunProxy method. 174 | 175 | ```golang 176 | // IPlugin defines plugin interface 177 | // Plugin should implements the IPlugin 178 | type IPlugin interface { 179 | // Setup calls at the begin of plugin system initialize 180 | // plugin system will pass the raw message to plugin's Setup function 181 | Setup(json.RawMessage) error 182 | 183 | // Close a proxy, it may be called by client's connection close 184 | StopProxy(item *PluginMeta) 185 | 186 | // Run a proxy, it may be called by client's connection established 187 | RunProxy(item *PluginMeta) error 188 | } 189 | ``` 190 | 191 | And then implement the interface 192 | 193 | ```golang 194 | package tcpproxy 195 | 196 | import ( 197 | "encoding/json" 198 | "fmt" 199 | 200 | "github.com/ICKelin/opennotr/opennotrd/plugin" 201 | ) 202 | 203 | func init() { 204 | plugin.Register("tcp", &TCPProxy{}) 205 | } 206 | 207 | type TCPProxy struct{} 208 | 209 | func (t *TCPProxy) Setup(config json.RawMessage) error { return nil } 210 | 211 | func (t *TCPProxy) StopProxy(item *plugin.PluginMeta) {} 212 | 213 | func (t *TCPProxy) RunProxy(item *plugin.PluginMeta) error { 214 | return fmt.Errorf("TODO://") 215 | } 216 | ``` 217 | 218 | and then import the plugin package 219 | ```golang 220 | import ( 221 | // plugin import 222 | _ "github.com/ICKelin/opennotr/opennotrd/plugin/tcpproxy" 223 | ) 224 | ``` 225 | 226 | Technology details 227 | ================== 228 | 229 | - [opennotr architecture]() 230 | - [opennotr dynamic upstream implement]() 231 | - [opennotr vpn implement]() 232 | 233 | [Back to TOC](#table-of-contents) 234 | 235 | Author 236 | ====== 237 | A programer name ICKelin. -------------------------------------------------------------------------------- /build_exec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | WORKSPACE=`pwd` 3 | BIN=$WORKSPACE/bin/ 4 | EXEC_PREFIX=opennotrd 5 | 6 | cd $WORKSPACE/opennotr 7 | 8 | echo 'building client...' 9 | GOOS=darwin go build -o $BIN/$EXEC_PREFIX_darwin_amd64 10 | GOOS=linux go build -o $BIN/$EXEC_PREFIX_linux_amd64 11 | GOARCH=arm GOOS=linux go build -o $BIN/$EXEC_PREFIX_arm 12 | GOARCH=arm64 GOOS=linux go build -o $BIN/$EXEC_PREFIX_arm64 13 | 14 | echo 'building server...' 15 | cd $WORKSPACE/cmd/opennotrd 16 | GOOS=linux go build -o $BIN/$EXEC_PREFIX_linux_amd64 17 | -------------------------------------------------------------------------------- /build_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ./build_exec.sh 3 | 4 | cd $WORKSPACE/docker-build 5 | docker build . -t opennotrd 6 | -------------------------------------------------------------------------------- /cmd/opennotrd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/ICKelin/opennotr/opennotrd" 4 | 5 | func main() { 6 | opennotrd.Run() 7 | } 8 | -------------------------------------------------------------------------------- /coverage.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | echo "" > coverage.txt 5 | 6 | for d in $(go list ./... | grep -v pkg/logs); do 7 | go test -race -coverprofile=profile.out -covermode=atomic $d -v 8 | if [ -f profile.out ]; then 9 | cat profile.out >> coverage.txt 10 | rm profile.out 11 | fi 12 | done -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | opennotr相关文档 2 | =============== 3 | 建设中. -------------------------------------------------------------------------------- /doc/assets/httpdemo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ICKelin/opennotr/c7fa36165ea9766f8065e593fb9dbc625c19cd31/doc/assets/httpdemo.jpg -------------------------------------------------------------------------------- /doc/assets/logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ICKelin/opennotr/c7fa36165ea9766f8065e593fb9dbc625c19cd31/doc/assets/logo.jpg -------------------------------------------------------------------------------- /doc/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ICKelin/opennotr/c7fa36165ea9766f8065e593fb9dbc625c19cd31/doc/assets/logo.png -------------------------------------------------------------------------------- /doc/assets/opennotr.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ICKelin/opennotr/c7fa36165ea9766f8065e593fb9dbc625c19cd31/doc/assets/opennotr.jpg -------------------------------------------------------------------------------- /doc/assets/qrcode.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ICKelin/opennotr/c7fa36165ea9766f8065e593fb9dbc625c19cd31/doc/assets/qrcode.jpg -------------------------------------------------------------------------------- /doc/assets/tcpdemo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ICKelin/opennotr/c7fa36165ea9766f8065e593fb9dbc625c19cd31/doc/assets/tcpdemo.jpg -------------------------------------------------------------------------------- /doc/assets/udpdemo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ICKelin/opennotr/c7fa36165ea9766f8065e593fb9dbc625c19cd31/doc/assets/udpdemo.jpg -------------------------------------------------------------------------------- /doc/zh/opennotr技术实现细节.md: -------------------------------------------------------------------------------- 1 | ## opennotr技术实现细节 2 | 3 | ## 内部线程模型 4 | 5 | ## openresty动态upstream 6 | 7 | ## tcp/udp代理实现 8 | -------------------------------------------------------------------------------- /doc/zh/opennotr技术架构.md: -------------------------------------------------------------------------------- 1 | ## opennotr技术架构 2 | ![技术架构](../../opennotr.jpg) -------------------------------------------------------------------------------- /docker-build/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ickelin/resty-upstream:latest 2 | COPY opennotrd /opt/ 3 | COPY start.sh /opt/ 4 | RUN chmod +x /opt/start.sh && chmod +x /opt/opennotrd 5 | CMD /opt/start.sh 6 | -------------------------------------------------------------------------------- /docker-build/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | opennotrd: 4 | image: ickelin/opennotr:latest 5 | network_mode: host 6 | container_name: opennotrd 7 | restart: always 8 | privileged: true 9 | volumes: 10 | - /opt/logs/opennotr:/opt/resty-upstream/logs 11 | - /opt/data/opennotrd:/opt/conf 12 | environment: 13 | TIME_ZONE: Asia/Shanghai 14 | -------------------------------------------------------------------------------- /docker-build/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | /opt/openresty/bin/openresty -p /opt/resty-upstream -c /opt/conf/nginx-conf/nginx.conf 3 | /opt/opennotrd -conf /opt/conf/notrd.yaml 4 | -------------------------------------------------------------------------------- /etc/opennotr.yaml: -------------------------------------------------------------------------------- 1 | serverAddr: "demo.notr.tech:10100" 2 | key: "http://www.notr.tech" 3 | forwards: 4 | - protocol: tcp 5 | ports: 6 | 222: 2222 7 | 8 | - protocol: udp 9 | ports: 10 | 0: 53 11 | 12 | - protocol: http 13 | ports: 14 | 0: 8080 15 | 16 | - protocol: https 17 | ports: 18 | 0: 8081 19 | 20 | - protocol: h2c 21 | ports: 22 | 0: 50052 23 | -------------------------------------------------------------------------------- /etc/opennotrd.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | listen: ":10100" 3 | authKey: "client server exchange key" 4 | domain: "open.notr.tech" 5 | 6 | tcpforward: 7 | listen: ":4398" 8 | 9 | udpforward: 10 | listen: ":4399" 11 | 12 | dhcp: 13 | cidr: "100.64.242.1/24" 14 | ip: "100.64.242.1" 15 | 16 | # resolver: 17 | # etcdEndpoints: 18 | # - 127.0.0.1:2379 19 | 20 | plugin: 21 | tcp: | 22 | {} 23 | 24 | udp: | 25 | { 26 | "sessionTimeout": 30 27 | } 28 | 29 | http: | 30 | { 31 | "adminUrl": "http://127.0.0.1:81/upstreams" 32 | } 33 | 34 | https: | 35 | { 36 | "adminUrl": "http://127.0.0.1:81/upstreams" 37 | } 38 | 39 | h2c: | 40 | { 41 | "adminUrl": "http://127.0.0.1:81/upstreams" 42 | } 43 | dummy: | 44 | {} 45 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ICKelin/opennotr/c7fa36165ea9766f8065e593fb9dbc625c19cd31/example/README.md -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ICKelin/opennotr 2 | 3 | go 1.13 4 | 5 | replace google.golang.org/grpc => google.golang.org/grpc v1.26.0 6 | 7 | require ( 8 | github.com/astaxie/beego v1.12.3 9 | github.com/belogik/goes v0.0.0-20151229125003-e54d722c3aff 10 | github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 11 | github.com/coreos/bbolt v1.3.2 // indirect 12 | github.com/coreos/etcd v3.3.22+incompatible 13 | github.com/coreos/go-semver v0.3.0 // indirect 14 | github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect 15 | github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect 16 | github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect 17 | github.com/dustin/go-humanize v1.0.0 // indirect 18 | github.com/gogo/protobuf v1.3.1 19 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect 20 | github.com/google/btree v1.0.0 // indirect 21 | github.com/google/uuid v1.1.1 // indirect 22 | github.com/gorilla/websocket v1.4.0 23 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect 24 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 25 | github.com/grpc-ecosystem/grpc-gateway v1.9.5 // indirect 26 | github.com/jonboulle/clockwork v0.1.0 // indirect 27 | github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect 28 | github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0 29 | github.com/soheilhy/cmux v0.1.4 // indirect 30 | github.com/stretchr/testify v1.5.1 // indirect 31 | github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect 32 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect 33 | github.com/xtaci/smux v2.0.1+incompatible 34 | go.etcd.io/bbolt v1.3.3 // indirect 35 | go.uber.org/zap v1.15.0 // indirect 36 | golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 // indirect 37 | golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect 38 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect 39 | golang.org/x/text v0.3.2 // indirect 40 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect 41 | golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 // indirect 42 | google.golang.org/genproto v0.0.0-20200711021454-869866162049 // indirect 43 | google.golang.org/grpc v1.29.1 // indirect 44 | gopkg.in/yaml.v2 v2.3.0 45 | honnef.co/go/tools v0.0.1-2020.1.3 // indirect 46 | sigs.k8s.io/yaml v1.2.0 // indirect 47 | ) 48 | -------------------------------------------------------------------------------- /internal/logs/README.md: -------------------------------------------------------------------------------- 1 | ## logs 2 | logs is a Go logs manager. It can use many logs adapters. The repo is inspired by `database/sql` . 3 | 4 | 5 | ## How to install? 6 | 7 | go get github.com/astaxie/beego/logs 8 | 9 | 10 | ## What adapters are supported? 11 | 12 | As of now this logs support console, file,smtp and conn. 13 | 14 | 15 | ## How to use it? 16 | 17 | First you must import it 18 | 19 | import ( 20 | "github.com/astaxie/beego/logs" 21 | ) 22 | 23 | Then init a Log (example with console adapter) 24 | 25 | log := NewLogger(10000) 26 | log.SetLogger("console", "") 27 | 28 | > the first params stand for how many channel 29 | 30 | Use it like this: 31 | 32 | log.Trace("trace") 33 | log.Info("info") 34 | log.Warn("warning") 35 | log.Debug("debug") 36 | log.Critical("critical") 37 | 38 | 39 | ## File adapter 40 | 41 | Configure file adapter like this: 42 | 43 | log := NewLogger(10000) 44 | log.SetLogger("file", `{"filename":"test.log"}`) 45 | 46 | 47 | ## Conn adapter 48 | 49 | Configure like this: 50 | 51 | log := NewLogger(1000) 52 | log.SetLogger("conn", `{"net":"tcp","addr":":7020"}`) 53 | log.Info("info") 54 | 55 | 56 | ## Smtp adapter 57 | 58 | Configure like this: 59 | 60 | log := NewLogger(10000) 61 | log.SetLogger("smtp", `{"username":"beegotest@gmail.com","password":"xxxxxxxx","host":"smtp.gmail.com:587","sendTos":["xiemengjun@gmail.com"]}`) 62 | log.Critical("sendmail critical") 63 | time.Sleep(time.Second * 30) 64 | -------------------------------------------------------------------------------- /internal/logs/alils/alils.go: -------------------------------------------------------------------------------- 1 | package alils 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "sync" 7 | "time" 8 | 9 | "github.com/astaxie/beego/logs" 10 | "github.com/gogo/protobuf/proto" 11 | ) 12 | 13 | const ( 14 | // CacheSize set the flush size 15 | CacheSize int = 64 16 | // Delimiter define the topic delimiter 17 | Delimiter string = "##" 18 | ) 19 | 20 | // Config is the Config for Ali Log 21 | type Config struct { 22 | Project string `json:"project"` 23 | Endpoint string `json:"endpoint"` 24 | KeyID string `json:"key_id"` 25 | KeySecret string `json:"key_secret"` 26 | LogStore string `json:"log_store"` 27 | Topics []string `json:"topics"` 28 | Source string `json:"source"` 29 | Level int `json:"level"` 30 | FlushWhen int `json:"flush_when"` 31 | } 32 | 33 | // aliLSWriter implements LoggerInterface. 34 | // it writes messages in keep-live tcp connection. 35 | type aliLSWriter struct { 36 | store *LogStore 37 | group []*LogGroup 38 | withMap bool 39 | groupMap map[string]*LogGroup 40 | lock *sync.Mutex 41 | Config 42 | } 43 | 44 | // NewAliLS create a new Logger 45 | func NewAliLS() logs.Logger { 46 | alils := new(aliLSWriter) 47 | alils.Level = logs.LevelTrace 48 | return alils 49 | } 50 | 51 | // Init parse config and init struct 52 | func (c *aliLSWriter) Init(jsonConfig string) (err error) { 53 | 54 | json.Unmarshal([]byte(jsonConfig), c) 55 | 56 | if c.FlushWhen > CacheSize { 57 | c.FlushWhen = CacheSize 58 | } 59 | 60 | prj := &LogProject{ 61 | Name: c.Project, 62 | Endpoint: c.Endpoint, 63 | AccessKeyID: c.KeyID, 64 | AccessKeySecret: c.KeySecret, 65 | } 66 | 67 | c.store, err = prj.GetLogStore(c.LogStore) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | // Create default Log Group 73 | c.group = append(c.group, &LogGroup{ 74 | Topic: proto.String(""), 75 | Source: proto.String(c.Source), 76 | Logs: make([]*Log, 0, c.FlushWhen), 77 | }) 78 | 79 | // Create other Log Group 80 | c.groupMap = make(map[string]*LogGroup) 81 | for _, topic := range c.Topics { 82 | 83 | lg := &LogGroup{ 84 | Topic: proto.String(topic), 85 | Source: proto.String(c.Source), 86 | Logs: make([]*Log, 0, c.FlushWhen), 87 | } 88 | 89 | c.group = append(c.group, lg) 90 | c.groupMap[topic] = lg 91 | } 92 | 93 | if len(c.group) == 1 { 94 | c.withMap = false 95 | } else { 96 | c.withMap = true 97 | } 98 | 99 | c.lock = &sync.Mutex{} 100 | 101 | return nil 102 | } 103 | 104 | // WriteMsg write message in connection. 105 | // if connection is down, try to re-connect. 106 | func (c *aliLSWriter) WriteMsg(when time.Time, msg string, level int) (err error) { 107 | 108 | if level > c.Level { 109 | return nil 110 | } 111 | 112 | var topic string 113 | var content string 114 | var lg *LogGroup 115 | if c.withMap { 116 | 117 | // Topic，LogGroup 118 | strs := strings.SplitN(msg, Delimiter, 2) 119 | if len(strs) == 2 { 120 | pos := strings.LastIndex(strs[0], " ") 121 | topic = strs[0][pos+1 : len(strs[0])] 122 | content = strs[0][0:pos] + strs[1] 123 | lg = c.groupMap[topic] 124 | } 125 | 126 | // send to empty Topic 127 | if lg == nil { 128 | content = msg 129 | lg = c.group[0] 130 | } 131 | } else { 132 | content = msg 133 | lg = c.group[0] 134 | } 135 | 136 | c1 := &LogContent{ 137 | Key: proto.String("msg"), 138 | Value: proto.String(content), 139 | } 140 | 141 | l := &Log{ 142 | Time: proto.Uint32(uint32(when.Unix())), 143 | Contents: []*LogContent{ 144 | c1, 145 | }, 146 | } 147 | 148 | c.lock.Lock() 149 | lg.Logs = append(lg.Logs, l) 150 | c.lock.Unlock() 151 | 152 | if len(lg.Logs) >= c.FlushWhen { 153 | c.flush(lg) 154 | } 155 | 156 | return nil 157 | } 158 | 159 | // Flush implementing method. empty. 160 | func (c *aliLSWriter) Flush() { 161 | 162 | // flush all group 163 | for _, lg := range c.group { 164 | c.flush(lg) 165 | } 166 | } 167 | 168 | // Destroy destroy connection writer and close tcp listener. 169 | func (c *aliLSWriter) Destroy() { 170 | } 171 | 172 | func (c *aliLSWriter) flush(lg *LogGroup) { 173 | 174 | c.lock.Lock() 175 | defer c.lock.Unlock() 176 | err := c.store.PutLogs(lg) 177 | if err != nil { 178 | return 179 | } 180 | 181 | lg.Logs = make([]*Log, 0, c.FlushWhen) 182 | } 183 | 184 | func init() { 185 | logs.Register(logs.AdapterAliLS, NewAliLS) 186 | } 187 | -------------------------------------------------------------------------------- /internal/logs/alils/config.go: -------------------------------------------------------------------------------- 1 | package alils 2 | 3 | const ( 4 | version = "0.5.0" // SDK version 5 | signatureMethod = "hmac-sha1" // Signature method 6 | 7 | // OffsetNewest stands for the log head offset, i.e. the offset that will be 8 | // assigned to the next message that will be produced to the shard. 9 | OffsetNewest = "end" 10 | // OffsetOldest stands for the oldest offset available on the logstore for a 11 | // shard. 12 | OffsetOldest = "begin" 13 | ) 14 | -------------------------------------------------------------------------------- /internal/logs/alils/log_config.go: -------------------------------------------------------------------------------- 1 | package alils 2 | 3 | // InputDetail define log detail 4 | type InputDetail struct { 5 | LogType string `json:"logType"` 6 | LogPath string `json:"logPath"` 7 | FilePattern string `json:"filePattern"` 8 | LocalStorage bool `json:"localStorage"` 9 | TimeFormat string `json:"timeFormat"` 10 | LogBeginRegex string `json:"logBeginRegex"` 11 | Regex string `json:"regex"` 12 | Keys []string `json:"key"` 13 | FilterKeys []string `json:"filterKey"` 14 | FilterRegex []string `json:"filterRegex"` 15 | TopicFormat string `json:"topicFormat"` 16 | } 17 | 18 | // OutputDetail define the output detail 19 | type OutputDetail struct { 20 | Endpoint string `json:"endpoint"` 21 | LogStoreName string `json:"logstoreName"` 22 | } 23 | 24 | // LogConfig define Log Config 25 | type LogConfig struct { 26 | Name string `json:"configName"` 27 | InputType string `json:"inputType"` 28 | InputDetail InputDetail `json:"inputDetail"` 29 | OutputType string `json:"outputType"` 30 | OutputDetail OutputDetail `json:"outputDetail"` 31 | 32 | CreateTime uint32 33 | LastModifyTime uint32 34 | 35 | project *LogProject 36 | } 37 | 38 | // GetAppliedMachineGroup returns applied machine group of this config. 39 | func (c *LogConfig) GetAppliedMachineGroup(confName string) (groupNames []string, err error) { 40 | groupNames, err = c.project.GetAppliedMachineGroups(c.Name) 41 | return 42 | } 43 | -------------------------------------------------------------------------------- /internal/logs/alils/log_store.go: -------------------------------------------------------------------------------- 1 | package alils 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "net/http/httputil" 9 | "strconv" 10 | 11 | lz4 "github.com/cloudflare/golz4" 12 | "github.com/gogo/protobuf/proto" 13 | ) 14 | 15 | // LogStore Store the logs 16 | type LogStore struct { 17 | Name string `json:"logstoreName"` 18 | TTL int 19 | ShardCount int 20 | 21 | CreateTime uint32 22 | LastModifyTime uint32 23 | 24 | project *LogProject 25 | } 26 | 27 | // Shard define the Log Shard 28 | type Shard struct { 29 | ShardID int `json:"shardID"` 30 | } 31 | 32 | // ListShards returns shard id list of this logstore. 33 | func (s *LogStore) ListShards() (shardIDs []int, err error) { 34 | h := map[string]string{ 35 | "x-sls-bodyrawsize": "0", 36 | } 37 | 38 | uri := fmt.Sprintf("/logstores/%v/shards", s.Name) 39 | r, err := request(s.project, "GET", uri, h, nil) 40 | if err != nil { 41 | return 42 | } 43 | 44 | buf, err := ioutil.ReadAll(r.Body) 45 | if err != nil { 46 | return 47 | } 48 | 49 | if r.StatusCode != http.StatusOK { 50 | errMsg := &errorMessage{} 51 | err = json.Unmarshal(buf, errMsg) 52 | if err != nil { 53 | err = fmt.Errorf("failed to list logstore") 54 | dump, _ := httputil.DumpResponse(r, true) 55 | fmt.Println(dump) 56 | return 57 | } 58 | err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) 59 | return 60 | } 61 | 62 | var shards []*Shard 63 | err = json.Unmarshal(buf, &shards) 64 | if err != nil { 65 | return 66 | } 67 | 68 | for _, v := range shards { 69 | shardIDs = append(shardIDs, v.ShardID) 70 | } 71 | return 72 | } 73 | 74 | // PutLogs put logs into logstore. 75 | // The callers should transform user logs into LogGroup. 76 | func (s *LogStore) PutLogs(lg *LogGroup) (err error) { 77 | body, err := proto.Marshal(lg) 78 | if err != nil { 79 | return 80 | } 81 | 82 | // Compresse body with lz4 83 | out := make([]byte, lz4.CompressBound(body)) 84 | n, err := lz4.Compress(body, out) 85 | if err != nil { 86 | return 87 | } 88 | 89 | h := map[string]string{ 90 | "x-sls-compresstype": "lz4", 91 | "x-sls-bodyrawsize": fmt.Sprintf("%v", len(body)), 92 | "Content-Type": "application/x-protobuf", 93 | } 94 | 95 | uri := fmt.Sprintf("/logstores/%v", s.Name) 96 | r, err := request(s.project, "POST", uri, h, out[:n]) 97 | if err != nil { 98 | return 99 | } 100 | 101 | buf, err := ioutil.ReadAll(r.Body) 102 | if err != nil { 103 | return 104 | } 105 | 106 | if r.StatusCode != http.StatusOK { 107 | errMsg := &errorMessage{} 108 | err = json.Unmarshal(buf, errMsg) 109 | if err != nil { 110 | err = fmt.Errorf("failed to put logs") 111 | dump, _ := httputil.DumpResponse(r, true) 112 | fmt.Println(dump) 113 | return 114 | } 115 | err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) 116 | return 117 | } 118 | return 119 | } 120 | 121 | // GetCursor gets log cursor of one shard specified by shardID. 122 | // The from can be in three form: a) unix timestamp in seccond, b) "begin", c) "end". 123 | // For more detail please read: http://gitlab.alibaba-inc.com/sls/doc/blob/master/api/shard.md#logstore 124 | func (s *LogStore) GetCursor(shardID int, from string) (cursor string, err error) { 125 | h := map[string]string{ 126 | "x-sls-bodyrawsize": "0", 127 | } 128 | 129 | uri := fmt.Sprintf("/logstores/%v/shards/%v?type=cursor&from=%v", 130 | s.Name, shardID, from) 131 | 132 | r, err := request(s.project, "GET", uri, h, nil) 133 | if err != nil { 134 | return 135 | } 136 | 137 | buf, err := ioutil.ReadAll(r.Body) 138 | if err != nil { 139 | return 140 | } 141 | 142 | if r.StatusCode != http.StatusOK { 143 | errMsg := &errorMessage{} 144 | err = json.Unmarshal(buf, errMsg) 145 | if err != nil { 146 | err = fmt.Errorf("failed to get cursor") 147 | dump, _ := httputil.DumpResponse(r, true) 148 | fmt.Println(dump) 149 | return 150 | } 151 | err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) 152 | return 153 | } 154 | 155 | type Body struct { 156 | Cursor string 157 | } 158 | body := &Body{} 159 | 160 | err = json.Unmarshal(buf, body) 161 | if err != nil { 162 | return 163 | } 164 | cursor = body.Cursor 165 | return 166 | } 167 | 168 | // GetLogsBytes gets logs binary data from shard specified by shardID according cursor. 169 | // The logGroupMaxCount is the max number of logGroup could be returned. 170 | // The nextCursor is the next curosr can be used to read logs at next time. 171 | func (s *LogStore) GetLogsBytes(shardID int, cursor string, 172 | logGroupMaxCount int) (out []byte, nextCursor string, err error) { 173 | 174 | h := map[string]string{ 175 | "x-sls-bodyrawsize": "0", 176 | "Accept": "application/x-protobuf", 177 | "Accept-Encoding": "lz4", 178 | } 179 | 180 | uri := fmt.Sprintf("/logstores/%v/shards/%v?type=logs&cursor=%v&count=%v", 181 | s.Name, shardID, cursor, logGroupMaxCount) 182 | 183 | r, err := request(s.project, "GET", uri, h, nil) 184 | if err != nil { 185 | return 186 | } 187 | 188 | buf, err := ioutil.ReadAll(r.Body) 189 | if err != nil { 190 | return 191 | } 192 | 193 | if r.StatusCode != http.StatusOK { 194 | errMsg := &errorMessage{} 195 | err = json.Unmarshal(buf, errMsg) 196 | if err != nil { 197 | err = fmt.Errorf("failed to get cursor") 198 | dump, _ := httputil.DumpResponse(r, true) 199 | fmt.Println(dump) 200 | return 201 | } 202 | err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) 203 | return 204 | } 205 | 206 | v, ok := r.Header["X-Sls-Compresstype"] 207 | if !ok || len(v) == 0 { 208 | err = fmt.Errorf("can't find 'x-sls-compresstype' header") 209 | return 210 | } 211 | if v[0] != "lz4" { 212 | err = fmt.Errorf("unexpected compress type:%v", v[0]) 213 | return 214 | } 215 | 216 | v, ok = r.Header["X-Sls-Cursor"] 217 | if !ok || len(v) == 0 { 218 | err = fmt.Errorf("can't find 'x-sls-cursor' header") 219 | return 220 | } 221 | nextCursor = v[0] 222 | 223 | v, ok = r.Header["X-Sls-Bodyrawsize"] 224 | if !ok || len(v) == 0 { 225 | err = fmt.Errorf("can't find 'x-sls-bodyrawsize' header") 226 | return 227 | } 228 | bodyRawSize, err := strconv.Atoi(v[0]) 229 | if err != nil { 230 | return 231 | } 232 | 233 | out = make([]byte, bodyRawSize) 234 | err = lz4.Uncompress(buf, out) 235 | if err != nil { 236 | return 237 | } 238 | 239 | return 240 | } 241 | 242 | // LogsBytesDecode decodes logs binary data retruned by GetLogsBytes API 243 | func LogsBytesDecode(data []byte) (gl *LogGroupList, err error) { 244 | 245 | gl = &LogGroupList{} 246 | err = proto.Unmarshal(data, gl) 247 | if err != nil { 248 | return 249 | } 250 | 251 | return 252 | } 253 | 254 | // GetLogs gets logs from shard specified by shardID according cursor. 255 | // The logGroupMaxCount is the max number of logGroup could be returned. 256 | // The nextCursor is the next curosr can be used to read logs at next time. 257 | func (s *LogStore) GetLogs(shardID int, cursor string, 258 | logGroupMaxCount int) (gl *LogGroupList, nextCursor string, err error) { 259 | 260 | out, nextCursor, err := s.GetLogsBytes(shardID, cursor, logGroupMaxCount) 261 | if err != nil { 262 | return 263 | } 264 | 265 | gl, err = LogsBytesDecode(out) 266 | if err != nil { 267 | return 268 | } 269 | 270 | return 271 | } 272 | -------------------------------------------------------------------------------- /internal/logs/alils/machine_group.go: -------------------------------------------------------------------------------- 1 | package alils 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "net/http/httputil" 9 | ) 10 | 11 | // MachineGroupAttribute define the Attribute 12 | type MachineGroupAttribute struct { 13 | ExternalName string `json:"externalName"` 14 | TopicName string `json:"groupTopic"` 15 | } 16 | 17 | // MachineGroup define the machine Group 18 | type MachineGroup struct { 19 | Name string `json:"groupName"` 20 | Type string `json:"groupType"` 21 | MachineIDType string `json:"machineIdentifyType"` 22 | MachineIDList []string `json:"machineList"` 23 | 24 | Attribute MachineGroupAttribute `json:"groupAttribute"` 25 | 26 | CreateTime uint32 27 | LastModifyTime uint32 28 | 29 | project *LogProject 30 | } 31 | 32 | // Machine define the Machine 33 | type Machine struct { 34 | IP string 35 | UniqueID string `json:"machine-uniqueid"` 36 | UserdefinedID string `json:"userdefined-id"` 37 | } 38 | 39 | // MachineList define the Machine List 40 | type MachineList struct { 41 | Total int 42 | Machines []*Machine 43 | } 44 | 45 | // ListMachines returns machine list of this machine group. 46 | func (m *MachineGroup) ListMachines() (ms []*Machine, total int, err error) { 47 | h := map[string]string{ 48 | "x-sls-bodyrawsize": "0", 49 | } 50 | 51 | uri := fmt.Sprintf("/machinegroups/%v/machines", m.Name) 52 | r, err := request(m.project, "GET", uri, h, nil) 53 | if err != nil { 54 | return 55 | } 56 | 57 | buf, err := ioutil.ReadAll(r.Body) 58 | if err != nil { 59 | return 60 | } 61 | 62 | if r.StatusCode != http.StatusOK { 63 | errMsg := &errorMessage{} 64 | err = json.Unmarshal(buf, errMsg) 65 | if err != nil { 66 | err = fmt.Errorf("failed to remove config from machine group") 67 | dump, _ := httputil.DumpResponse(r, true) 68 | fmt.Println(dump) 69 | return 70 | } 71 | err = fmt.Errorf("%v:%v", errMsg.Code, errMsg.Message) 72 | return 73 | } 74 | 75 | body := &MachineList{} 76 | err = json.Unmarshal(buf, body) 77 | if err != nil { 78 | return 79 | } 80 | 81 | ms = body.Machines 82 | total = body.Total 83 | 84 | return 85 | } 86 | 87 | // GetAppliedConfigs returns applied configs of this machine group. 88 | func (m *MachineGroup) GetAppliedConfigs() (confNames []string, err error) { 89 | confNames, err = m.project.GetAppliedConfigs(m.Name) 90 | return 91 | } 92 | -------------------------------------------------------------------------------- /internal/logs/alils/request.go: -------------------------------------------------------------------------------- 1 | package alils 2 | 3 | import ( 4 | "bytes" 5 | "crypto/md5" 6 | "fmt" 7 | "net/http" 8 | ) 9 | 10 | // request sends a request to SLS. 11 | func request(project *LogProject, method, uri string, headers map[string]string, 12 | body []byte) (resp *http.Response, err error) { 13 | 14 | // The caller should provide 'x-sls-bodyrawsize' header 15 | if _, ok := headers["x-sls-bodyrawsize"]; !ok { 16 | err = fmt.Errorf("Can't find 'x-sls-bodyrawsize' header") 17 | return 18 | } 19 | 20 | // SLS public request headers 21 | headers["Host"] = project.Name + "." + project.Endpoint 22 | headers["Date"] = nowRFC1123() 23 | headers["x-sls-apiversion"] = version 24 | headers["x-sls-signaturemethod"] = signatureMethod 25 | if body != nil { 26 | bodyMD5 := fmt.Sprintf("%X", md5.Sum(body)) 27 | headers["Content-MD5"] = bodyMD5 28 | 29 | if _, ok := headers["Content-Type"]; !ok { 30 | err = fmt.Errorf("Can't find 'Content-Type' header") 31 | return 32 | } 33 | } 34 | 35 | // Calc Authorization 36 | // Authorization = "SLS :" 37 | digest, err := signature(project, method, uri, headers) 38 | if err != nil { 39 | return 40 | } 41 | auth := fmt.Sprintf("SLS %v:%v", project.AccessKeyID, digest) 42 | headers["Authorization"] = auth 43 | 44 | // Initialize http request 45 | reader := bytes.NewReader(body) 46 | urlStr := fmt.Sprintf("http://%v.%v%v", project.Name, project.Endpoint, uri) 47 | req, err := http.NewRequest(method, urlStr, reader) 48 | if err != nil { 49 | return 50 | } 51 | for k, v := range headers { 52 | req.Header.Add(k, v) 53 | } 54 | 55 | // Get ready to do request 56 | resp, err = http.DefaultClient.Do(req) 57 | if err != nil { 58 | return 59 | } 60 | 61 | return 62 | } 63 | -------------------------------------------------------------------------------- /internal/logs/alils/signature.go: -------------------------------------------------------------------------------- 1 | package alils 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha1" 6 | "encoding/base64" 7 | "fmt" 8 | "net/url" 9 | "sort" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | // GMT location 15 | var gmtLoc = time.FixedZone("GMT", 0) 16 | 17 | // NowRFC1123 returns now time in RFC1123 format with GMT timezone, 18 | // eg. "Mon, 02 Jan 2006 15:04:05 GMT". 19 | func nowRFC1123() string { 20 | return time.Now().In(gmtLoc).Format(time.RFC1123) 21 | } 22 | 23 | // signature calculates a request's signature digest. 24 | func signature(project *LogProject, method, uri string, 25 | headers map[string]string) (digest string, err error) { 26 | var contentMD5, contentType, date, canoHeaders, canoResource string 27 | var slsHeaderKeys sort.StringSlice 28 | 29 | // SignString = VERB + "\n" 30 | // + CONTENT-MD5 + "\n" 31 | // + CONTENT-TYPE + "\n" 32 | // + DATE + "\n" 33 | // + CanonicalizedSLSHeaders + "\n" 34 | // + CanonicalizedResource 35 | 36 | if val, ok := headers["Content-MD5"]; ok { 37 | contentMD5 = val 38 | } 39 | 40 | if val, ok := headers["Content-Type"]; ok { 41 | contentType = val 42 | } 43 | 44 | date, ok := headers["Date"] 45 | if !ok { 46 | err = fmt.Errorf("Can't find 'Date' header") 47 | return 48 | } 49 | 50 | // Calc CanonicalizedSLSHeaders 51 | slsHeaders := make(map[string]string, len(headers)) 52 | for k, v := range headers { 53 | l := strings.TrimSpace(strings.ToLower(k)) 54 | if strings.HasPrefix(l, "x-sls-") { 55 | slsHeaders[l] = strings.TrimSpace(v) 56 | slsHeaderKeys = append(slsHeaderKeys, l) 57 | } 58 | } 59 | 60 | sort.Sort(slsHeaderKeys) 61 | for i, k := range slsHeaderKeys { 62 | canoHeaders += k + ":" + slsHeaders[k] 63 | if i+1 < len(slsHeaderKeys) { 64 | canoHeaders += "\n" 65 | } 66 | } 67 | 68 | // Calc CanonicalizedResource 69 | u, err := url.Parse(uri) 70 | if err != nil { 71 | return 72 | } 73 | 74 | canoResource += url.QueryEscape(u.Path) 75 | if u.RawQuery != "" { 76 | var keys sort.StringSlice 77 | 78 | vals := u.Query() 79 | for k := range vals { 80 | keys = append(keys, k) 81 | } 82 | 83 | sort.Sort(keys) 84 | canoResource += "?" 85 | for i, k := range keys { 86 | if i > 0 { 87 | canoResource += "&" 88 | } 89 | 90 | for _, v := range vals[k] { 91 | canoResource += k + "=" + v 92 | } 93 | } 94 | } 95 | 96 | signStr := method + "\n" + 97 | contentMD5 + "\n" + 98 | contentType + "\n" + 99 | date + "\n" + 100 | canoHeaders + "\n" + 101 | canoResource 102 | 103 | // Signature = base64(hmac-sha1(UTF8-Encoding-Of(SignString)，AccessKeySecret)) 104 | mac := hmac.New(sha1.New, []byte(project.AccessKeySecret)) 105 | _, err = mac.Write([]byte(signStr)) 106 | if err != nil { 107 | return 108 | } 109 | digest = base64.StdEncoding.EncodeToString(mac.Sum(nil)) 110 | return 111 | } 112 | -------------------------------------------------------------------------------- /internal/logs/color.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build !windows 16 | 17 | package logs 18 | 19 | import "io" 20 | 21 | type ansiColorWriter struct { 22 | w io.Writer 23 | mode outputMode 24 | } 25 | 26 | func (cw *ansiColorWriter) Write(p []byte) (int, error) { 27 | return cw.w.Write(p) 28 | } 29 | -------------------------------------------------------------------------------- /internal/logs/color_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build windows 16 | 17 | package logs 18 | 19 | import ( 20 | "bytes" 21 | "io" 22 | "strings" 23 | "syscall" 24 | "unsafe" 25 | ) 26 | 27 | type ( 28 | csiState int 29 | parseResult int 30 | ) 31 | 32 | const ( 33 | outsideCsiCode csiState = iota 34 | firstCsiCode 35 | secondCsiCode 36 | ) 37 | 38 | const ( 39 | noConsole parseResult = iota 40 | changedColor 41 | unknown 42 | ) 43 | 44 | type ansiColorWriter struct { 45 | w io.Writer 46 | mode outputMode 47 | state csiState 48 | paramStartBuf bytes.Buffer 49 | paramBuf bytes.Buffer 50 | } 51 | 52 | const ( 53 | firstCsiChar byte = '\x1b' 54 | secondeCsiChar byte = '[' 55 | separatorChar byte = ';' 56 | sgrCode byte = 'm' 57 | ) 58 | 59 | const ( 60 | foregroundBlue = uint16(0x0001) 61 | foregroundGreen = uint16(0x0002) 62 | foregroundRed = uint16(0x0004) 63 | foregroundIntensity = uint16(0x0008) 64 | backgroundBlue = uint16(0x0010) 65 | backgroundGreen = uint16(0x0020) 66 | backgroundRed = uint16(0x0040) 67 | backgroundIntensity = uint16(0x0080) 68 | underscore = uint16(0x8000) 69 | 70 | foregroundMask = foregroundBlue | foregroundGreen | foregroundRed | foregroundIntensity 71 | backgroundMask = backgroundBlue | backgroundGreen | backgroundRed | backgroundIntensity 72 | ) 73 | 74 | const ( 75 | ansiReset = "0" 76 | ansiIntensityOn = "1" 77 | ansiIntensityOff = "21" 78 | ansiUnderlineOn = "4" 79 | ansiUnderlineOff = "24" 80 | ansiBlinkOn = "5" 81 | ansiBlinkOff = "25" 82 | 83 | ansiForegroundBlack = "30" 84 | ansiForegroundRed = "31" 85 | ansiForegroundGreen = "32" 86 | ansiForegroundYellow = "33" 87 | ansiForegroundBlue = "34" 88 | ansiForegroundMagenta = "35" 89 | ansiForegroundCyan = "36" 90 | ansiForegroundWhite = "37" 91 | ansiForegroundDefault = "39" 92 | 93 | ansiBackgroundBlack = "40" 94 | ansiBackgroundRed = "41" 95 | ansiBackgroundGreen = "42" 96 | ansiBackgroundYellow = "43" 97 | ansiBackgroundBlue = "44" 98 | ansiBackgroundMagenta = "45" 99 | ansiBackgroundCyan = "46" 100 | ansiBackgroundWhite = "47" 101 | ansiBackgroundDefault = "49" 102 | 103 | ansiLightForegroundGray = "90" 104 | ansiLightForegroundRed = "91" 105 | ansiLightForegroundGreen = "92" 106 | ansiLightForegroundYellow = "93" 107 | ansiLightForegroundBlue = "94" 108 | ansiLightForegroundMagenta = "95" 109 | ansiLightForegroundCyan = "96" 110 | ansiLightForegroundWhite = "97" 111 | 112 | ansiLightBackgroundGray = "100" 113 | ansiLightBackgroundRed = "101" 114 | ansiLightBackgroundGreen = "102" 115 | ansiLightBackgroundYellow = "103" 116 | ansiLightBackgroundBlue = "104" 117 | ansiLightBackgroundMagenta = "105" 118 | ansiLightBackgroundCyan = "106" 119 | ansiLightBackgroundWhite = "107" 120 | ) 121 | 122 | type drawType int 123 | 124 | const ( 125 | foreground drawType = iota 126 | background 127 | ) 128 | 129 | type winColor struct { 130 | code uint16 131 | drawType drawType 132 | } 133 | 134 | var colorMap = map[string]winColor{ 135 | ansiForegroundBlack: {0, foreground}, 136 | ansiForegroundRed: {foregroundRed, foreground}, 137 | ansiForegroundGreen: {foregroundGreen, foreground}, 138 | ansiForegroundYellow: {foregroundRed | foregroundGreen, foreground}, 139 | ansiForegroundBlue: {foregroundBlue, foreground}, 140 | ansiForegroundMagenta: {foregroundRed | foregroundBlue, foreground}, 141 | ansiForegroundCyan: {foregroundGreen | foregroundBlue, foreground}, 142 | ansiForegroundWhite: {foregroundRed | foregroundGreen | foregroundBlue, foreground}, 143 | ansiForegroundDefault: {foregroundRed | foregroundGreen | foregroundBlue, foreground}, 144 | 145 | ansiBackgroundBlack: {0, background}, 146 | ansiBackgroundRed: {backgroundRed, background}, 147 | ansiBackgroundGreen: {backgroundGreen, background}, 148 | ansiBackgroundYellow: {backgroundRed | backgroundGreen, background}, 149 | ansiBackgroundBlue: {backgroundBlue, background}, 150 | ansiBackgroundMagenta: {backgroundRed | backgroundBlue, background}, 151 | ansiBackgroundCyan: {backgroundGreen | backgroundBlue, background}, 152 | ansiBackgroundWhite: {backgroundRed | backgroundGreen | backgroundBlue, background}, 153 | ansiBackgroundDefault: {0, background}, 154 | 155 | ansiLightForegroundGray: {foregroundIntensity, foreground}, 156 | ansiLightForegroundRed: {foregroundIntensity | foregroundRed, foreground}, 157 | ansiLightForegroundGreen: {foregroundIntensity | foregroundGreen, foreground}, 158 | ansiLightForegroundYellow: {foregroundIntensity | foregroundRed | foregroundGreen, foreground}, 159 | ansiLightForegroundBlue: {foregroundIntensity | foregroundBlue, foreground}, 160 | ansiLightForegroundMagenta: {foregroundIntensity | foregroundRed | foregroundBlue, foreground}, 161 | ansiLightForegroundCyan: {foregroundIntensity | foregroundGreen | foregroundBlue, foreground}, 162 | ansiLightForegroundWhite: {foregroundIntensity | foregroundRed | foregroundGreen | foregroundBlue, foreground}, 163 | 164 | ansiLightBackgroundGray: {backgroundIntensity, background}, 165 | ansiLightBackgroundRed: {backgroundIntensity | backgroundRed, background}, 166 | ansiLightBackgroundGreen: {backgroundIntensity | backgroundGreen, background}, 167 | ansiLightBackgroundYellow: {backgroundIntensity | backgroundRed | backgroundGreen, background}, 168 | ansiLightBackgroundBlue: {backgroundIntensity | backgroundBlue, background}, 169 | ansiLightBackgroundMagenta: {backgroundIntensity | backgroundRed | backgroundBlue, background}, 170 | ansiLightBackgroundCyan: {backgroundIntensity | backgroundGreen | backgroundBlue, background}, 171 | ansiLightBackgroundWhite: {backgroundIntensity | backgroundRed | backgroundGreen | backgroundBlue, background}, 172 | } 173 | 174 | var ( 175 | kernel32 = syscall.NewLazyDLL("kernel32.dll") 176 | procSetConsoleTextAttribute = kernel32.NewProc("SetConsoleTextAttribute") 177 | procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo") 178 | defaultAttr *textAttributes 179 | ) 180 | 181 | func init() { 182 | screenInfo := getConsoleScreenBufferInfo(uintptr(syscall.Stdout)) 183 | if screenInfo != nil { 184 | colorMap[ansiForegroundDefault] = winColor{ 185 | screenInfo.WAttributes & (foregroundRed | foregroundGreen | foregroundBlue), 186 | foreground, 187 | } 188 | colorMap[ansiBackgroundDefault] = winColor{ 189 | screenInfo.WAttributes & (backgroundRed | backgroundGreen | backgroundBlue), 190 | background, 191 | } 192 | defaultAttr = convertTextAttr(screenInfo.WAttributes) 193 | } 194 | } 195 | 196 | type coord struct { 197 | X, Y int16 198 | } 199 | 200 | type smallRect struct { 201 | Left, Top, Right, Bottom int16 202 | } 203 | 204 | type consoleScreenBufferInfo struct { 205 | DwSize coord 206 | DwCursorPosition coord 207 | WAttributes uint16 208 | SrWindow smallRect 209 | DwMaximumWindowSize coord 210 | } 211 | 212 | func getConsoleScreenBufferInfo(hConsoleOutput uintptr) *consoleScreenBufferInfo { 213 | var csbi consoleScreenBufferInfo 214 | ret, _, _ := procGetConsoleScreenBufferInfo.Call( 215 | hConsoleOutput, 216 | uintptr(unsafe.Pointer(&csbi))) 217 | if ret == 0 { 218 | return nil 219 | } 220 | return &csbi 221 | } 222 | 223 | func setConsoleTextAttribute(hConsoleOutput uintptr, wAttributes uint16) bool { 224 | ret, _, _ := procSetConsoleTextAttribute.Call( 225 | hConsoleOutput, 226 | uintptr(wAttributes)) 227 | return ret != 0 228 | } 229 | 230 | type textAttributes struct { 231 | foregroundColor uint16 232 | backgroundColor uint16 233 | foregroundIntensity uint16 234 | backgroundIntensity uint16 235 | underscore uint16 236 | otherAttributes uint16 237 | } 238 | 239 | func convertTextAttr(winAttr uint16) *textAttributes { 240 | fgColor := winAttr & (foregroundRed | foregroundGreen | foregroundBlue) 241 | bgColor := winAttr & (backgroundRed | backgroundGreen | backgroundBlue) 242 | fgIntensity := winAttr & foregroundIntensity 243 | bgIntensity := winAttr & backgroundIntensity 244 | underline := winAttr & underscore 245 | otherAttributes := winAttr &^ (foregroundMask | backgroundMask | underscore) 246 | return &textAttributes{fgColor, bgColor, fgIntensity, bgIntensity, underline, otherAttributes} 247 | } 248 | 249 | func convertWinAttr(textAttr *textAttributes) uint16 { 250 | var winAttr uint16 251 | winAttr |= textAttr.foregroundColor 252 | winAttr |= textAttr.backgroundColor 253 | winAttr |= textAttr.foregroundIntensity 254 | winAttr |= textAttr.backgroundIntensity 255 | winAttr |= textAttr.underscore 256 | winAttr |= textAttr.otherAttributes 257 | return winAttr 258 | } 259 | 260 | func changeColor(param []byte) parseResult { 261 | screenInfo := getConsoleScreenBufferInfo(uintptr(syscall.Stdout)) 262 | if screenInfo == nil { 263 | return noConsole 264 | } 265 | 266 | winAttr := convertTextAttr(screenInfo.WAttributes) 267 | strParam := string(param) 268 | if len(strParam) <= 0 { 269 | strParam = "0" 270 | } 271 | csiParam := strings.Split(strParam, string(separatorChar)) 272 | for _, p := range csiParam { 273 | c, ok := colorMap[p] 274 | switch { 275 | case !ok: 276 | switch p { 277 | case ansiReset: 278 | winAttr.foregroundColor = defaultAttr.foregroundColor 279 | winAttr.backgroundColor = defaultAttr.backgroundColor 280 | winAttr.foregroundIntensity = defaultAttr.foregroundIntensity 281 | winAttr.backgroundIntensity = defaultAttr.backgroundIntensity 282 | winAttr.underscore = 0 283 | winAttr.otherAttributes = 0 284 | case ansiIntensityOn: 285 | winAttr.foregroundIntensity = foregroundIntensity 286 | case ansiIntensityOff: 287 | winAttr.foregroundIntensity = 0 288 | case ansiUnderlineOn: 289 | winAttr.underscore = underscore 290 | case ansiUnderlineOff: 291 | winAttr.underscore = 0 292 | case ansiBlinkOn: 293 | winAttr.backgroundIntensity = backgroundIntensity 294 | case ansiBlinkOff: 295 | winAttr.backgroundIntensity = 0 296 | default: 297 | // unknown code 298 | } 299 | case c.drawType == foreground: 300 | winAttr.foregroundColor = c.code 301 | case c.drawType == background: 302 | winAttr.backgroundColor = c.code 303 | } 304 | } 305 | winTextAttribute := convertWinAttr(winAttr) 306 | setConsoleTextAttribute(uintptr(syscall.Stdout), winTextAttribute) 307 | 308 | return changedColor 309 | } 310 | 311 | func parseEscapeSequence(command byte, param []byte) parseResult { 312 | if defaultAttr == nil { 313 | return noConsole 314 | } 315 | 316 | switch command { 317 | case sgrCode: 318 | return changeColor(param) 319 | default: 320 | return unknown 321 | } 322 | } 323 | 324 | func (cw *ansiColorWriter) flushBuffer() (int, error) { 325 | return cw.flushTo(cw.w) 326 | } 327 | 328 | func (cw *ansiColorWriter) resetBuffer() (int, error) { 329 | return cw.flushTo(nil) 330 | } 331 | 332 | func (cw *ansiColorWriter) flushTo(w io.Writer) (int, error) { 333 | var n1, n2 int 334 | var err error 335 | 336 | startBytes := cw.paramStartBuf.Bytes() 337 | cw.paramStartBuf.Reset() 338 | if w != nil { 339 | n1, err = cw.w.Write(startBytes) 340 | if err != nil { 341 | return n1, err 342 | } 343 | } else { 344 | n1 = len(startBytes) 345 | } 346 | paramBytes := cw.paramBuf.Bytes() 347 | cw.paramBuf.Reset() 348 | if w != nil { 349 | n2, err = cw.w.Write(paramBytes) 350 | if err != nil { 351 | return n1 + n2, err 352 | } 353 | } else { 354 | n2 = len(paramBytes) 355 | } 356 | return n1 + n2, nil 357 | } 358 | 359 | func isParameterChar(b byte) bool { 360 | return ('0' <= b && b <= '9') || b == separatorChar 361 | } 362 | 363 | func (cw *ansiColorWriter) Write(p []byte) (int, error) { 364 | var r, nw, first, last int 365 | if cw.mode != DiscardNonColorEscSeq { 366 | cw.state = outsideCsiCode 367 | cw.resetBuffer() 368 | } 369 | 370 | var err error 371 | for i, ch := range p { 372 | switch cw.state { 373 | case outsideCsiCode: 374 | if ch == firstCsiChar { 375 | cw.paramStartBuf.WriteByte(ch) 376 | cw.state = firstCsiCode 377 | } 378 | case firstCsiCode: 379 | switch ch { 380 | case firstCsiChar: 381 | cw.paramStartBuf.WriteByte(ch) 382 | break 383 | case secondeCsiChar: 384 | cw.paramStartBuf.WriteByte(ch) 385 | cw.state = secondCsiCode 386 | last = i - 1 387 | default: 388 | cw.resetBuffer() 389 | cw.state = outsideCsiCode 390 | } 391 | case secondCsiCode: 392 | if isParameterChar(ch) { 393 | cw.paramBuf.WriteByte(ch) 394 | } else { 395 | nw, err = cw.w.Write(p[first:last]) 396 | r += nw 397 | if err != nil { 398 | return r, err 399 | } 400 | first = i + 1 401 | result := parseEscapeSequence(ch, cw.paramBuf.Bytes()) 402 | if result == noConsole || (cw.mode == OutputNonColorEscSeq && result == unknown) { 403 | cw.paramBuf.WriteByte(ch) 404 | nw, err := cw.flushBuffer() 405 | if err != nil { 406 | return r, err 407 | } 408 | r += nw 409 | } else { 410 | n, _ := cw.resetBuffer() 411 | // Add one more to the size of the buffer for the last ch 412 | r += n + 1 413 | } 414 | 415 | cw.state = outsideCsiCode 416 | } 417 | default: 418 | cw.state = outsideCsiCode 419 | } 420 | } 421 | 422 | if cw.mode != DiscardNonColorEscSeq || cw.state == outsideCsiCode { 423 | nw, err = cw.w.Write(p[first:]) 424 | r += nw 425 | } 426 | 427 | return r, err 428 | } 429 | -------------------------------------------------------------------------------- /internal/logs/color_windows_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // +build windows 16 | 17 | package logs 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "syscall" 23 | "testing" 24 | ) 25 | 26 | var GetConsoleScreenBufferInfo = getConsoleScreenBufferInfo 27 | 28 | func ChangeColor(color uint16) { 29 | setConsoleTextAttribute(uintptr(syscall.Stdout), color) 30 | } 31 | 32 | func ResetColor() { 33 | ChangeColor(uint16(0x0007)) 34 | } 35 | 36 | func TestWritePlanText(t *testing.T) { 37 | inner := bytes.NewBufferString("") 38 | w := NewAnsiColorWriter(inner) 39 | expected := "plain text" 40 | fmt.Fprintf(w, expected) 41 | actual := inner.String() 42 | if actual != expected { 43 | t.Errorf("Get %q, want %q", actual, expected) 44 | } 45 | } 46 | 47 | func TestWriteParseText(t *testing.T) { 48 | inner := bytes.NewBufferString("") 49 | w := NewAnsiColorWriter(inner) 50 | 51 | inputTail := "\x1b[0mtail text" 52 | expectedTail := "tail text" 53 | fmt.Fprintf(w, inputTail) 54 | actualTail := inner.String() 55 | inner.Reset() 56 | if actualTail != expectedTail { 57 | t.Errorf("Get %q, want %q", actualTail, expectedTail) 58 | } 59 | 60 | inputHead := "head text\x1b[0m" 61 | expectedHead := "head text" 62 | fmt.Fprintf(w, inputHead) 63 | actualHead := inner.String() 64 | inner.Reset() 65 | if actualHead != expectedHead { 66 | t.Errorf("Get %q, want %q", actualHead, expectedHead) 67 | } 68 | 69 | inputBothEnds := "both ends \x1b[0m text" 70 | expectedBothEnds := "both ends text" 71 | fmt.Fprintf(w, inputBothEnds) 72 | actualBothEnds := inner.String() 73 | inner.Reset() 74 | if actualBothEnds != expectedBothEnds { 75 | t.Errorf("Get %q, want %q", actualBothEnds, expectedBothEnds) 76 | } 77 | 78 | inputManyEsc := "\x1b\x1b\x1b\x1b[0m many esc" 79 | expectedManyEsc := "\x1b\x1b\x1b many esc" 80 | fmt.Fprintf(w, inputManyEsc) 81 | actualManyEsc := inner.String() 82 | inner.Reset() 83 | if actualManyEsc != expectedManyEsc { 84 | t.Errorf("Get %q, want %q", actualManyEsc, expectedManyEsc) 85 | } 86 | 87 | expectedSplit := "split text" 88 | for _, ch := range "split \x1b[0m text" { 89 | fmt.Fprintf(w, string(ch)) 90 | } 91 | actualSplit := inner.String() 92 | inner.Reset() 93 | if actualSplit != expectedSplit { 94 | t.Errorf("Get %q, want %q", actualSplit, expectedSplit) 95 | } 96 | } 97 | 98 | type screenNotFoundError struct { 99 | error 100 | } 101 | 102 | func writeAnsiColor(expectedText, colorCode string) (actualText string, actualAttributes uint16, err error) { 103 | inner := bytes.NewBufferString("") 104 | w := NewAnsiColorWriter(inner) 105 | fmt.Fprintf(w, "\x1b[%sm%s", colorCode, expectedText) 106 | 107 | actualText = inner.String() 108 | screenInfo := GetConsoleScreenBufferInfo(uintptr(syscall.Stdout)) 109 | if screenInfo != nil { 110 | actualAttributes = screenInfo.WAttributes 111 | } else { 112 | err = &screenNotFoundError{} 113 | } 114 | return 115 | } 116 | 117 | type testParam struct { 118 | text string 119 | attributes uint16 120 | ansiColor string 121 | } 122 | 123 | func TestWriteAnsiColorText(t *testing.T) { 124 | screenInfo := GetConsoleScreenBufferInfo(uintptr(syscall.Stdout)) 125 | if screenInfo == nil { 126 | t.Fatal("Could not get ConsoleScreenBufferInfo") 127 | } 128 | defer ChangeColor(screenInfo.WAttributes) 129 | defaultFgColor := screenInfo.WAttributes & uint16(0x0007) 130 | defaultBgColor := screenInfo.WAttributes & uint16(0x0070) 131 | defaultFgIntensity := screenInfo.WAttributes & uint16(0x0008) 132 | defaultBgIntensity := screenInfo.WAttributes & uint16(0x0080) 133 | 134 | fgParam := []testParam{ 135 | {"foreground black ", uint16(0x0000 | 0x0000), "30"}, 136 | {"foreground red ", uint16(0x0004 | 0x0000), "31"}, 137 | {"foreground green ", uint16(0x0002 | 0x0000), "32"}, 138 | {"foreground yellow ", uint16(0x0006 | 0x0000), "33"}, 139 | {"foreground blue ", uint16(0x0001 | 0x0000), "34"}, 140 | {"foreground magenta", uint16(0x0005 | 0x0000), "35"}, 141 | {"foreground cyan ", uint16(0x0003 | 0x0000), "36"}, 142 | {"foreground white ", uint16(0x0007 | 0x0000), "37"}, 143 | {"foreground default", defaultFgColor | 0x0000, "39"}, 144 | {"foreground light gray ", uint16(0x0000 | 0x0008 | 0x0000), "90"}, 145 | {"foreground light red ", uint16(0x0004 | 0x0008 | 0x0000), "91"}, 146 | {"foreground light green ", uint16(0x0002 | 0x0008 | 0x0000), "92"}, 147 | {"foreground light yellow ", uint16(0x0006 | 0x0008 | 0x0000), "93"}, 148 | {"foreground light blue ", uint16(0x0001 | 0x0008 | 0x0000), "94"}, 149 | {"foreground light magenta", uint16(0x0005 | 0x0008 | 0x0000), "95"}, 150 | {"foreground light cyan ", uint16(0x0003 | 0x0008 | 0x0000), "96"}, 151 | {"foreground light white ", uint16(0x0007 | 0x0008 | 0x0000), "97"}, 152 | } 153 | 154 | bgParam := []testParam{ 155 | {"background black ", uint16(0x0007 | 0x0000), "40"}, 156 | {"background red ", uint16(0x0007 | 0x0040), "41"}, 157 | {"background green ", uint16(0x0007 | 0x0020), "42"}, 158 | {"background yellow ", uint16(0x0007 | 0x0060), "43"}, 159 | {"background blue ", uint16(0x0007 | 0x0010), "44"}, 160 | {"background magenta", uint16(0x0007 | 0x0050), "45"}, 161 | {"background cyan ", uint16(0x0007 | 0x0030), "46"}, 162 | {"background white ", uint16(0x0007 | 0x0070), "47"}, 163 | {"background default", uint16(0x0007) | defaultBgColor, "49"}, 164 | {"background light gray ", uint16(0x0007 | 0x0000 | 0x0080), "100"}, 165 | {"background light red ", uint16(0x0007 | 0x0040 | 0x0080), "101"}, 166 | {"background light green ", uint16(0x0007 | 0x0020 | 0x0080), "102"}, 167 | {"background light yellow ", uint16(0x0007 | 0x0060 | 0x0080), "103"}, 168 | {"background light blue ", uint16(0x0007 | 0x0010 | 0x0080), "104"}, 169 | {"background light magenta", uint16(0x0007 | 0x0050 | 0x0080), "105"}, 170 | {"background light cyan ", uint16(0x0007 | 0x0030 | 0x0080), "106"}, 171 | {"background light white ", uint16(0x0007 | 0x0070 | 0x0080), "107"}, 172 | } 173 | 174 | resetParam := []testParam{ 175 | {"all reset", defaultFgColor | defaultBgColor | defaultFgIntensity | defaultBgIntensity, "0"}, 176 | {"all reset", defaultFgColor | defaultBgColor | defaultFgIntensity | defaultBgIntensity, ""}, 177 | } 178 | 179 | boldParam := []testParam{ 180 | {"bold on", uint16(0x0007 | 0x0008), "1"}, 181 | {"bold off", uint16(0x0007), "21"}, 182 | } 183 | 184 | underscoreParam := []testParam{ 185 | {"underscore on", uint16(0x0007 | 0x8000), "4"}, 186 | {"underscore off", uint16(0x0007), "24"}, 187 | } 188 | 189 | blinkParam := []testParam{ 190 | {"blink on", uint16(0x0007 | 0x0080), "5"}, 191 | {"blink off", uint16(0x0007), "25"}, 192 | } 193 | 194 | mixedParam := []testParam{ 195 | {"both black, bold, underline, blink", uint16(0x0000 | 0x0000 | 0x0008 | 0x8000 | 0x0080), "30;40;1;4;5"}, 196 | {"both red, bold, underline, blink", uint16(0x0004 | 0x0040 | 0x0008 | 0x8000 | 0x0080), "31;41;1;4;5"}, 197 | {"both green, bold, underline, blink", uint16(0x0002 | 0x0020 | 0x0008 | 0x8000 | 0x0080), "32;42;1;4;5"}, 198 | {"both yellow, bold, underline, blink", uint16(0x0006 | 0x0060 | 0x0008 | 0x8000 | 0x0080), "33;43;1;4;5"}, 199 | {"both blue, bold, underline, blink", uint16(0x0001 | 0x0010 | 0x0008 | 0x8000 | 0x0080), "34;44;1;4;5"}, 200 | {"both magenta, bold, underline, blink", uint16(0x0005 | 0x0050 | 0x0008 | 0x8000 | 0x0080), "35;45;1;4;5"}, 201 | {"both cyan, bold, underline, blink", uint16(0x0003 | 0x0030 | 0x0008 | 0x8000 | 0x0080), "36;46;1;4;5"}, 202 | {"both white, bold, underline, blink", uint16(0x0007 | 0x0070 | 0x0008 | 0x8000 | 0x0080), "37;47;1;4;5"}, 203 | {"both default, bold, underline, blink", uint16(defaultFgColor | defaultBgColor | 0x0008 | 0x8000 | 0x0080), "39;49;1;4;5"}, 204 | } 205 | 206 | assertTextAttribute := func(expectedText string, expectedAttributes uint16, ansiColor string) { 207 | actualText, actualAttributes, err := writeAnsiColor(expectedText, ansiColor) 208 | if actualText != expectedText { 209 | t.Errorf("Get %q, want %q", actualText, expectedText) 210 | } 211 | if err != nil { 212 | t.Fatal("Could not get ConsoleScreenBufferInfo") 213 | } 214 | if actualAttributes != expectedAttributes { 215 | t.Errorf("Text: %q, Get 0x%04x, want 0x%04x", expectedText, actualAttributes, expectedAttributes) 216 | } 217 | } 218 | 219 | for _, v := range fgParam { 220 | ResetColor() 221 | assertTextAttribute(v.text, v.attributes, v.ansiColor) 222 | } 223 | 224 | for _, v := range bgParam { 225 | ChangeColor(uint16(0x0070 | 0x0007)) 226 | assertTextAttribute(v.text, v.attributes, v.ansiColor) 227 | } 228 | 229 | for _, v := range resetParam { 230 | ChangeColor(uint16(0x0000 | 0x0070 | 0x0008)) 231 | assertTextAttribute(v.text, v.attributes, v.ansiColor) 232 | } 233 | 234 | ResetColor() 235 | for _, v := range boldParam { 236 | assertTextAttribute(v.text, v.attributes, v.ansiColor) 237 | } 238 | 239 | ResetColor() 240 | for _, v := range underscoreParam { 241 | assertTextAttribute(v.text, v.attributes, v.ansiColor) 242 | } 243 | 244 | ResetColor() 245 | for _, v := range blinkParam { 246 | assertTextAttribute(v.text, v.attributes, v.ansiColor) 247 | } 248 | 249 | for _, v := range mixedParam { 250 | ResetColor() 251 | assertTextAttribute(v.text, v.attributes, v.ansiColor) 252 | } 253 | } 254 | 255 | func TestIgnoreUnknownSequences(t *testing.T) { 256 | inner := bytes.NewBufferString("") 257 | w := NewModeAnsiColorWriter(inner, OutputNonColorEscSeq) 258 | 259 | inputText := "\x1b[=decpath mode" 260 | expectedTail := inputText 261 | fmt.Fprintf(w, inputText) 262 | actualTail := inner.String() 263 | inner.Reset() 264 | if actualTail != expectedTail { 265 | t.Errorf("Get %q, want %q", actualTail, expectedTail) 266 | } 267 | 268 | inputText = "\x1b[=tailing esc and bracket\x1b[" 269 | expectedTail = inputText 270 | fmt.Fprintf(w, inputText) 271 | actualTail = inner.String() 272 | inner.Reset() 273 | if actualTail != expectedTail { 274 | t.Errorf("Get %q, want %q", actualTail, expectedTail) 275 | } 276 | 277 | inputText = "\x1b[?tailing esc\x1b" 278 | expectedTail = inputText 279 | fmt.Fprintf(w, inputText) 280 | actualTail = inner.String() 281 | inner.Reset() 282 | if actualTail != expectedTail { 283 | t.Errorf("Get %q, want %q", actualTail, expectedTail) 284 | } 285 | 286 | inputText = "\x1b[1h;3punended color code invalid\x1b3" 287 | expectedTail = inputText 288 | fmt.Fprintf(w, inputText) 289 | actualTail = inner.String() 290 | inner.Reset() 291 | if actualTail != expectedTail { 292 | t.Errorf("Get %q, want %q", actualTail, expectedTail) 293 | } 294 | } 295 | -------------------------------------------------------------------------------- /internal/logs/conn.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "encoding/json" 19 | "io" 20 | "net" 21 | "time" 22 | ) 23 | 24 | // connWriter implements LoggerInterface. 25 | // it writes messages in keep-live tcp connection. 26 | type connWriter struct { 27 | lg *logWriter 28 | innerWriter io.WriteCloser 29 | ReconnectOnMsg bool `json:"reconnectOnMsg"` 30 | Reconnect bool `json:"reconnect"` 31 | Net string `json:"net"` 32 | Addr string `json:"addr"` 33 | Level int `json:"level"` 34 | } 35 | 36 | // NewConn create new ConnWrite returning as LoggerInterface. 37 | func NewConn() Logger { 38 | conn := new(connWriter) 39 | conn.Level = LevelTrace 40 | return conn 41 | } 42 | 43 | // Init init connection writer with json config. 44 | // json config only need key "level". 45 | func (c *connWriter) Init(jsonConfig string) error { 46 | return json.Unmarshal([]byte(jsonConfig), c) 47 | } 48 | 49 | // WriteMsg write message in connection. 50 | // if connection is down, try to re-connect. 51 | func (c *connWriter) WriteMsg(when time.Time, msg string, level int) error { 52 | if level > c.Level { 53 | return nil 54 | } 55 | if c.needToConnectOnMsg() { 56 | err := c.connect() 57 | if err != nil { 58 | return err 59 | } 60 | } 61 | 62 | if c.ReconnectOnMsg { 63 | defer c.innerWriter.Close() 64 | } 65 | 66 | c.lg.println(when, msg) 67 | return nil 68 | } 69 | 70 | // Flush implementing method. empty. 71 | func (c *connWriter) Flush() { 72 | 73 | } 74 | 75 | // Destroy destroy connection writer and close tcp listener. 76 | func (c *connWriter) Destroy() { 77 | if c.innerWriter != nil { 78 | c.innerWriter.Close() 79 | } 80 | } 81 | 82 | func (c *connWriter) connect() error { 83 | if c.innerWriter != nil { 84 | c.innerWriter.Close() 85 | c.innerWriter = nil 86 | } 87 | 88 | conn, err := net.Dial(c.Net, c.Addr) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | if tcpConn, ok := conn.(*net.TCPConn); ok { 94 | tcpConn.SetKeepAlive(true) 95 | } 96 | 97 | c.innerWriter = conn 98 | c.lg = newLogWriter(conn) 99 | return nil 100 | } 101 | 102 | func (c *connWriter) needToConnectOnMsg() bool { 103 | if c.Reconnect { 104 | c.Reconnect = false 105 | return true 106 | } 107 | 108 | if c.innerWriter == nil { 109 | return true 110 | } 111 | 112 | return c.ReconnectOnMsg 113 | } 114 | 115 | func init() { 116 | Register(AdapterConn, NewConn) 117 | } 118 | -------------------------------------------------------------------------------- /internal/logs/conn_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestConn(t *testing.T) { 22 | log := NewLogger(1000) 23 | log.SetLogger("conn", `{"net":"tcp","addr":":7020"}`) 24 | log.Informational("informational") 25 | } 26 | -------------------------------------------------------------------------------- /internal/logs/console.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "encoding/json" 19 | "os" 20 | "runtime" 21 | "time" 22 | ) 23 | 24 | // brush is a color join function 25 | type brush func(string) string 26 | 27 | // newBrush return a fix color Brush 28 | func newBrush(color string) brush { 29 | pre := "\033[" 30 | reset := "\033[0m" 31 | return func(text string) string { 32 | return pre + color + "m" + text + reset 33 | } 34 | } 35 | 36 | var colors = []brush{ 37 | newBrush("1;37"), // Emergency white 38 | newBrush("1;36"), // Alert cyan 39 | newBrush("1;35"), // Critical magenta 40 | newBrush("1;31"), // Error red 41 | newBrush("1;33"), // Warning yellow 42 | newBrush("1;32"), // Notice green 43 | newBrush("1;34"), // Informational blue 44 | newBrush("1;44"), // Debug Background blue 45 | } 46 | 47 | // consoleWriter implements LoggerInterface and writes messages to terminal. 48 | type consoleWriter struct { 49 | lg *logWriter 50 | Level int `json:"level"` 51 | Colorful bool `json:"color"` //this filed is useful only when system's terminal supports color 52 | } 53 | 54 | // NewConsole create ConsoleWriter returning as LoggerInterface. 55 | func NewConsole() Logger { 56 | cw := &consoleWriter{ 57 | lg: newLogWriter(os.Stdout), 58 | Level: LevelDebug, 59 | Colorful: runtime.GOOS != "windows", 60 | } 61 | return cw 62 | } 63 | 64 | // Init init console logger. 65 | // jsonConfig like '{"level":LevelTrace}'. 66 | func (c *consoleWriter) Init(jsonConfig string) error { 67 | if len(jsonConfig) == 0 { 68 | return nil 69 | } 70 | err := json.Unmarshal([]byte(jsonConfig), c) 71 | if runtime.GOOS == "windows" { 72 | c.Colorful = false 73 | } 74 | return err 75 | } 76 | 77 | // WriteMsg write message in console. 78 | func (c *consoleWriter) WriteMsg(when time.Time, msg string, level int) error { 79 | if level > c.Level { 80 | return nil 81 | } 82 | if c.Colorful { 83 | msg = colors[level](msg) 84 | } 85 | c.lg.println(when, msg) 86 | return nil 87 | } 88 | 89 | // Destroy implementing method. empty. 90 | func (c *consoleWriter) Destroy() { 91 | 92 | } 93 | 94 | // Flush implementing method. empty. 95 | func (c *consoleWriter) Flush() { 96 | 97 | } 98 | 99 | func init() { 100 | Register(AdapterConsole, NewConsole) 101 | } 102 | -------------------------------------------------------------------------------- /internal/logs/console_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | // Try each log level in decreasing order of priority. 22 | func testConsoleCalls(bl *BeeLogger) { 23 | bl.Emergency("emergency") 24 | bl.Alert("alert") 25 | bl.Critical("critical") 26 | bl.Error("error") 27 | bl.Warning("warning") 28 | bl.Notice("notice") 29 | bl.Informational("informational") 30 | bl.Debug("debug") 31 | } 32 | 33 | // Test console logging by visually comparing the lines being output with and 34 | // without a log level specification. 35 | func TestConsole(t *testing.T) { 36 | log1 := NewLogger(10000) 37 | log1.EnableFuncCallDepth(true) 38 | log1.SetLogger("console", "") 39 | testConsoleCalls(log1) 40 | 41 | log2 := NewLogger(100) 42 | log2.SetLogger("console", `{"level":3}`) 43 | testConsoleCalls(log2) 44 | } 45 | 46 | // Test console without color 47 | func TestConsoleNoColor(t *testing.T) { 48 | log := NewLogger(100) 49 | log.SetLogger("console", `{"color":false}`) 50 | testConsoleCalls(log) 51 | } 52 | -------------------------------------------------------------------------------- /internal/logs/es/es.go: -------------------------------------------------------------------------------- 1 | package es 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "net" 8 | "net/url" 9 | "time" 10 | 11 | "github.com/astaxie/beego/logs" 12 | "github.com/belogik/goes" 13 | ) 14 | 15 | // NewES return a LoggerInterface 16 | func NewES() logs.Logger { 17 | cw := &esLogger{ 18 | Level: logs.LevelDebug, 19 | } 20 | return cw 21 | } 22 | 23 | type esLogger struct { 24 | *goes.Connection 25 | DSN string `json:"dsn"` 26 | Level int `json:"level"` 27 | } 28 | 29 | // {"dsn":"http://localhost:9200/","level":1} 30 | func (el *esLogger) Init(jsonconfig string) error { 31 | err := json.Unmarshal([]byte(jsonconfig), el) 32 | if err != nil { 33 | return err 34 | } 35 | if el.DSN == "" { 36 | return errors.New("empty dsn") 37 | } else if u, err := url.Parse(el.DSN); err != nil { 38 | return err 39 | } else if u.Path == "" { 40 | return errors.New("missing prefix") 41 | } else if host, port, err := net.SplitHostPort(u.Host); err != nil { 42 | return err 43 | } else { 44 | conn := goes.NewConnection(host, port) 45 | el.Connection = conn 46 | } 47 | return nil 48 | } 49 | 50 | // WriteMsg will write the msg and level into es 51 | func (el *esLogger) WriteMsg(when time.Time, msg string, level int) error { 52 | if level > el.Level { 53 | return nil 54 | } 55 | 56 | vals := make(map[string]interface{}) 57 | vals["@timestamp"] = when.Format(time.RFC3339) 58 | vals["@msg"] = msg 59 | d := goes.Document{ 60 | Index: fmt.Sprintf("%04d.%02d.%02d", when.Year(), when.Month(), when.Day()), 61 | Type: "logs", 62 | Fields: vals, 63 | } 64 | _, err := el.Index(d, nil) 65 | return err 66 | } 67 | 68 | // Destroy is a empty method 69 | func (el *esLogger) Destroy() { 70 | 71 | } 72 | 73 | // Flush is a empty method 74 | func (el *esLogger) Flush() { 75 | 76 | } 77 | 78 | func init() { 79 | logs.Register(logs.AdapterEs, NewES) 80 | } 81 | -------------------------------------------------------------------------------- /internal/logs/file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "bytes" 19 | "encoding/json" 20 | "errors" 21 | "fmt" 22 | "io" 23 | "os" 24 | "path/filepath" 25 | "strconv" 26 | "strings" 27 | "sync" 28 | "time" 29 | ) 30 | 31 | // fileLogWriter implements LoggerInterface. 32 | // It writes messages by lines limit, file size limit, or time frequency. 33 | type fileLogWriter struct { 34 | sync.RWMutex // write log order by order and atomic incr maxLinesCurLines and maxSizeCurSize 35 | // The opened file 36 | Filename string `json:"filename"` 37 | fileWriter *os.File 38 | 39 | // Rotate at line 40 | MaxLines int `json:"maxlines"` 41 | maxLinesCurLines int 42 | 43 | // Rotate at size 44 | MaxSize int `json:"maxsize"` 45 | maxSizeCurSize int 46 | 47 | // Rotate daily 48 | Daily bool `json:"daily"` 49 | MaxDays int64 `json:"maxdays"` 50 | dailyOpenDate int 51 | dailyOpenTime time.Time 52 | 53 | Rotate bool `json:"rotate"` 54 | 55 | Level int `json:"level"` 56 | 57 | Perm string `json:"perm"` 58 | 59 | fileNameOnly, suffix string // like "project.log", project is fileNameOnly and .log is suffix 60 | } 61 | 62 | // newFileWriter create a FileLogWriter returning as LoggerInterface. 63 | func newFileWriter() Logger { 64 | w := &fileLogWriter{ 65 | Daily: true, 66 | MaxDays: 7, 67 | Rotate: true, 68 | Level: LevelTrace, 69 | Perm: "0660", 70 | } 71 | return w 72 | } 73 | 74 | // Init file logger with json config. 75 | // jsonConfig like: 76 | // { 77 | // "filename":"logs/beego.log", 78 | // "maxLines":10000, 79 | // "maxsize":1024, 80 | // "daily":true, 81 | // "maxDays":15, 82 | // "rotate":true, 83 | // "perm":"0600" 84 | // } 85 | func (w *fileLogWriter) Init(jsonConfig string) error { 86 | err := json.Unmarshal([]byte(jsonConfig), w) 87 | if err != nil { 88 | return err 89 | } 90 | if len(w.Filename) == 0 { 91 | return errors.New("jsonconfig must have filename") 92 | } 93 | w.suffix = filepath.Ext(w.Filename) 94 | w.fileNameOnly = strings.TrimSuffix(w.Filename, w.suffix) 95 | if w.suffix == "" { 96 | w.suffix = ".log" 97 | } 98 | err = w.startLogger() 99 | return err 100 | } 101 | 102 | // start file logger. create log file and set to locker-inside file writer. 103 | func (w *fileLogWriter) startLogger() error { 104 | file, err := w.createLogFile() 105 | if err != nil { 106 | return err 107 | } 108 | if w.fileWriter != nil { 109 | w.fileWriter.Close() 110 | } 111 | w.fileWriter = file 112 | return w.initFd() 113 | } 114 | 115 | func (w *fileLogWriter) needRotate(size int, day int) bool { 116 | return (w.MaxLines > 0 && w.maxLinesCurLines >= w.MaxLines) || 117 | (w.MaxSize > 0 && w.maxSizeCurSize >= w.MaxSize) || 118 | (w.Daily && day != w.dailyOpenDate) 119 | 120 | } 121 | 122 | // WriteMsg write logger message into file. 123 | func (w *fileLogWriter) WriteMsg(when time.Time, msg string, level int) error { 124 | if level > w.Level { 125 | return nil 126 | } 127 | h, d := formatTimeHeader(when) 128 | msg = string(h) + msg + "\n" 129 | if w.Rotate { 130 | w.RLock() 131 | if w.needRotate(len(msg), d) { 132 | w.RUnlock() 133 | w.Lock() 134 | if w.needRotate(len(msg), d) { 135 | if err := w.doRotate(when); err != nil { 136 | fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.Filename, err) 137 | } 138 | } 139 | w.Unlock() 140 | } else { 141 | w.RUnlock() 142 | } 143 | } 144 | 145 | w.Lock() 146 | _, err := w.fileWriter.Write([]byte(msg)) 147 | if err == nil { 148 | w.maxLinesCurLines++ 149 | w.maxSizeCurSize += len(msg) 150 | } 151 | w.Unlock() 152 | return err 153 | } 154 | 155 | func (w *fileLogWriter) createLogFile() (*os.File, error) { 156 | // Open the log file 157 | perm, err := strconv.ParseInt(w.Perm, 8, 64) 158 | if err != nil { 159 | return nil, err 160 | } 161 | fd, err := os.OpenFile(w.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(perm)) 162 | if err == nil { 163 | // Make sure file perm is user set perm cause of `os.OpenFile` will obey umask 164 | os.Chmod(w.Filename, os.FileMode(perm)) 165 | } 166 | return fd, err 167 | } 168 | 169 | func (w *fileLogWriter) initFd() error { 170 | fd := w.fileWriter 171 | fInfo, err := fd.Stat() 172 | if err != nil { 173 | return fmt.Errorf("get stat err: %s", err) 174 | } 175 | w.maxSizeCurSize = int(fInfo.Size()) 176 | w.dailyOpenTime = time.Now() 177 | w.dailyOpenDate = w.dailyOpenTime.Day() 178 | w.maxLinesCurLines = 0 179 | if w.Daily { 180 | go w.dailyRotate(w.dailyOpenTime) 181 | } 182 | if fInfo.Size() > 0 { 183 | count, err := w.lines() 184 | if err != nil { 185 | return err 186 | } 187 | w.maxLinesCurLines = count 188 | } 189 | return nil 190 | } 191 | 192 | func (w *fileLogWriter) dailyRotate(openTime time.Time) { 193 | y, m, d := openTime.Add(24 * time.Hour).Date() 194 | nextDay := time.Date(y, m, d, 0, 0, 0, 0, openTime.Location()) 195 | tm := time.NewTimer(time.Duration(nextDay.UnixNano() - openTime.UnixNano() + 100)) 196 | <-tm.C 197 | w.Lock() 198 | if w.needRotate(0, time.Now().Day()) { 199 | if err := w.doRotate(time.Now()); err != nil { 200 | fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.Filename, err) 201 | } 202 | } 203 | w.Unlock() 204 | } 205 | 206 | func (w *fileLogWriter) lines() (int, error) { 207 | fd, err := os.Open(w.Filename) 208 | if err != nil { 209 | return 0, err 210 | } 211 | defer fd.Close() 212 | 213 | buf := make([]byte, 32768) // 32k 214 | count := 0 215 | lineSep := []byte{'\n'} 216 | 217 | for { 218 | c, err := fd.Read(buf) 219 | if err != nil && err != io.EOF { 220 | return count, err 221 | } 222 | 223 | count += bytes.Count(buf[:c], lineSep) 224 | 225 | if err == io.EOF { 226 | break 227 | } 228 | } 229 | 230 | return count, nil 231 | } 232 | 233 | // DoRotate means it need to write file in new file. 234 | // new file name like xx.2013-01-01.log (daily) or xx.001.log (by line or size) 235 | func (w *fileLogWriter) doRotate(logTime time.Time) error { 236 | // file exists 237 | // Find the next available number 238 | num := 1 239 | fName := "" 240 | 241 | _, err := os.Lstat(w.Filename) 242 | if err != nil { 243 | //even if the file is not exist or other ,we should RESTART the logger 244 | goto RESTART_LOGGER 245 | } 246 | 247 | if w.MaxLines > 0 || w.MaxSize > 0 { 248 | for ; err == nil && num <= 999; num++ { 249 | fName = w.fileNameOnly + fmt.Sprintf(".%s.%03d%s", logTime.Format("2006-01-02"), num, w.suffix) 250 | _, err = os.Lstat(fName) 251 | } 252 | } else { 253 | fName = fmt.Sprintf("%s.%s%s", w.fileNameOnly, w.dailyOpenTime.Format("2006-01-02"), w.suffix) 254 | _, err = os.Lstat(fName) 255 | for ; err == nil && num <= 999; num++ { 256 | fName = w.fileNameOnly + fmt.Sprintf(".%s.%03d%s", w.dailyOpenTime.Format("2006-01-02"), num, w.suffix) 257 | _, err = os.Lstat(fName) 258 | } 259 | } 260 | // return error if the last file checked still existed 261 | if err == nil { 262 | return fmt.Errorf("Rotate: Cannot find free log number to rename %s", w.Filename) 263 | } 264 | 265 | // close fileWriter before rename 266 | w.fileWriter.Close() 267 | 268 | // Rename the file to its new found name 269 | // even if occurs error,we MUST guarantee to restart new logger 270 | err = os.Rename(w.Filename, fName) 271 | if err != nil { 272 | goto RESTART_LOGGER 273 | } 274 | err = os.Chmod(fName, os.FileMode(0440)) 275 | // re-start logger 276 | RESTART_LOGGER: 277 | 278 | startLoggerErr := w.startLogger() 279 | go w.deleteOldLog() 280 | 281 | if startLoggerErr != nil { 282 | return fmt.Errorf("Rotate StartLogger: %s", startLoggerErr) 283 | } 284 | if err != nil { 285 | return fmt.Errorf("Rotate: %s", err) 286 | } 287 | return nil 288 | } 289 | 290 | func (w *fileLogWriter) deleteOldLog() { 291 | dir := filepath.Dir(w.Filename) 292 | filepath.Walk(dir, func(path string, info os.FileInfo, err error) (returnErr error) { 293 | defer func() { 294 | if r := recover(); r != nil { 295 | fmt.Fprintf(os.Stderr, "Unable to delete old log '%s', error: %v\n", path, r) 296 | } 297 | }() 298 | 299 | if info == nil { 300 | return 301 | } 302 | 303 | if !info.IsDir() && info.ModTime().Add(24*time.Hour*time.Duration(w.MaxDays)).Before(time.Now()) { 304 | if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) && 305 | strings.HasSuffix(filepath.Base(path), w.suffix) { 306 | os.Remove(path) 307 | } 308 | } 309 | return 310 | }) 311 | } 312 | 313 | // Destroy close the file description, close file writer. 314 | func (w *fileLogWriter) Destroy() { 315 | w.fileWriter.Close() 316 | } 317 | 318 | // Flush flush file logger. 319 | // there are no buffering messages in file logger in memory. 320 | // flush file means sync file from disk. 321 | func (w *fileLogWriter) Flush() { 322 | w.fileWriter.Sync() 323 | } 324 | 325 | func init() { 326 | Register(AdapterFile, newFileWriter) 327 | } 328 | -------------------------------------------------------------------------------- /internal/logs/file_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "bufio" 19 | "fmt" 20 | "io/ioutil" 21 | "os" 22 | "strconv" 23 | "testing" 24 | "time" 25 | ) 26 | 27 | func TestFilePerm(t *testing.T) { 28 | log := NewLogger(10000) 29 | // use 0666 as test perm cause the default umask is 022 30 | log.SetLogger("file", `{"filename":"test.log", "perm": "0666"}`) 31 | log.Debug("debug") 32 | log.Informational("info") 33 | log.Notice("notice") 34 | log.Warning("warning") 35 | log.Error("error") 36 | log.Alert("alert") 37 | log.Critical("critical") 38 | log.Emergency("emergency") 39 | file, err := os.Stat("test.log") 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | if file.Mode() != 0666 { 44 | t.Fatal("unexpected log file permission") 45 | } 46 | os.Remove("test.log") 47 | } 48 | 49 | func TestFile1(t *testing.T) { 50 | log := NewLogger(10000) 51 | log.SetLogger("file", `{"filename":"test.log"}`) 52 | log.Debug("debug") 53 | log.Informational("info") 54 | log.Notice("notice") 55 | log.Warning("warning") 56 | log.Error("error") 57 | log.Alert("alert") 58 | log.Critical("critical") 59 | log.Emergency("emergency") 60 | f, err := os.Open("test.log") 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | b := bufio.NewReader(f) 65 | lineNum := 0 66 | for { 67 | line, _, err := b.ReadLine() 68 | if err != nil { 69 | break 70 | } 71 | if len(line) > 0 { 72 | lineNum++ 73 | } 74 | } 75 | var expected = LevelDebug + 1 76 | if lineNum != expected { 77 | t.Fatal(lineNum, "not "+strconv.Itoa(expected)+" lines") 78 | } 79 | os.Remove("test.log") 80 | } 81 | 82 | func TestFile2(t *testing.T) { 83 | log := NewLogger(10000) 84 | log.SetLogger("file", fmt.Sprintf(`{"filename":"test2.log","level":%d}`, LevelError)) 85 | log.Debug("debug") 86 | log.Info("info") 87 | log.Notice("notice") 88 | log.Warning("warning") 89 | log.Error("error") 90 | log.Alert("alert") 91 | log.Critical("critical") 92 | log.Emergency("emergency") 93 | f, err := os.Open("test2.log") 94 | if err != nil { 95 | t.Fatal(err) 96 | } 97 | b := bufio.NewReader(f) 98 | lineNum := 0 99 | for { 100 | line, _, err := b.ReadLine() 101 | if err != nil { 102 | break 103 | } 104 | if len(line) > 0 { 105 | lineNum++ 106 | } 107 | } 108 | var expected = LevelError + 1 109 | if lineNum != expected { 110 | t.Fatal(lineNum, "not "+strconv.Itoa(expected)+" lines") 111 | } 112 | os.Remove("test2.log") 113 | } 114 | 115 | func TestFileRotate_01(t *testing.T) { 116 | log := NewLogger(10000) 117 | log.SetLogger("file", `{"filename":"test3.log","maxlines":4}`) 118 | log.Debug("debug") 119 | log.Info("info") 120 | log.Notice("notice") 121 | log.Warning("warning") 122 | log.Error("error") 123 | log.Alert("alert") 124 | log.Critical("critical") 125 | log.Emergency("emergency") 126 | rotateName := "test3" + fmt.Sprintf(".%s.%03d", time.Now().Format("2006-01-02"), 1) + ".log" 127 | b, err := exists(rotateName) 128 | if !b || err != nil { 129 | os.Remove("test3.log") 130 | t.Fatal("rotate not generated") 131 | } 132 | os.Remove(rotateName) 133 | os.Remove("test3.log") 134 | } 135 | 136 | func TestFileRotate_02(t *testing.T) { 137 | fn1 := "rotate_day.log" 138 | fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".log" 139 | testFileRotate(t, fn1, fn2) 140 | } 141 | 142 | func TestFileRotate_03(t *testing.T) { 143 | fn1 := "rotate_day.log" 144 | fn := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".log" 145 | os.Create(fn) 146 | fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".001.log" 147 | testFileRotate(t, fn1, fn2) 148 | os.Remove(fn) 149 | } 150 | 151 | func TestFileRotate_04(t *testing.T) { 152 | fn1 := "rotate_day.log" 153 | fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".log" 154 | testFileDailyRotate(t, fn1, fn2) 155 | } 156 | 157 | func TestFileRotate_05(t *testing.T) { 158 | fn1 := "rotate_day.log" 159 | fn := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".log" 160 | os.Create(fn) 161 | fn2 := "rotate_day." + time.Now().Add(-24*time.Hour).Format("2006-01-02") + ".001.log" 162 | testFileDailyRotate(t, fn1, fn2) 163 | os.Remove(fn) 164 | } 165 | func TestFileRotate_06(t *testing.T) { //test file mode 166 | log := NewLogger(10000) 167 | log.SetLogger("file", `{"filename":"test3.log","maxlines":4}`) 168 | log.Debug("debug") 169 | log.Info("info") 170 | log.Notice("notice") 171 | log.Warning("warning") 172 | log.Error("error") 173 | log.Alert("alert") 174 | log.Critical("critical") 175 | log.Emergency("emergency") 176 | rotateName := "test3" + fmt.Sprintf(".%s.%03d", time.Now().Format("2006-01-02"), 1) + ".log" 177 | s, _ := os.Lstat(rotateName) 178 | if s.Mode() != 0440 { 179 | os.Remove(rotateName) 180 | os.Remove("test3.log") 181 | t.Fatal("rotate file mode error") 182 | } 183 | os.Remove(rotateName) 184 | os.Remove("test3.log") 185 | } 186 | func testFileRotate(t *testing.T, fn1, fn2 string) { 187 | fw := &fileLogWriter{ 188 | Daily: true, 189 | MaxDays: 7, 190 | Rotate: true, 191 | Level: LevelTrace, 192 | Perm: "0660", 193 | } 194 | fw.Init(fmt.Sprintf(`{"filename":"%v","maxdays":1}`, fn1)) 195 | fw.dailyOpenTime = time.Now().Add(-24 * time.Hour) 196 | fw.dailyOpenDate = fw.dailyOpenTime.Day() 197 | fw.WriteMsg(time.Now(), "this is a msg for test", LevelDebug) 198 | 199 | for _, file := range []string{fn1, fn2} { 200 | _, err := os.Stat(file) 201 | if err != nil { 202 | t.FailNow() 203 | } 204 | os.Remove(file) 205 | } 206 | fw.Destroy() 207 | } 208 | 209 | func testFileDailyRotate(t *testing.T, fn1, fn2 string) { 210 | fw := &fileLogWriter{ 211 | Daily: true, 212 | MaxDays: 7, 213 | Rotate: true, 214 | Level: LevelTrace, 215 | Perm: "0660", 216 | } 217 | fw.Init(fmt.Sprintf(`{"filename":"%v","maxdays":1}`, fn1)) 218 | fw.dailyOpenTime = time.Now().Add(-24 * time.Hour) 219 | fw.dailyOpenDate = fw.dailyOpenTime.Day() 220 | today, _ := time.ParseInLocation("2006-01-02", time.Now().Format("2006-01-02"), fw.dailyOpenTime.Location()) 221 | today = today.Add(-1 * time.Second) 222 | fw.dailyRotate(today) 223 | for _, file := range []string{fn1, fn2} { 224 | _, err := os.Stat(file) 225 | if err != nil { 226 | t.FailNow() 227 | } 228 | content, err := ioutil.ReadFile(file) 229 | if err != nil { 230 | t.FailNow() 231 | } 232 | if len(content) > 0 { 233 | t.FailNow() 234 | } 235 | os.Remove(file) 236 | } 237 | fw.Destroy() 238 | } 239 | 240 | func exists(path string) (bool, error) { 241 | _, err := os.Stat(path) 242 | if err == nil { 243 | return true, nil 244 | } 245 | if os.IsNotExist(err) { 246 | return false, nil 247 | } 248 | return false, err 249 | } 250 | 251 | func BenchmarkFile(b *testing.B) { 252 | log := NewLogger(100000) 253 | log.SetLogger("file", `{"filename":"test4.log"}`) 254 | for i := 0; i < b.N; i++ { 255 | log.Debug("debug") 256 | } 257 | os.Remove("test4.log") 258 | } 259 | 260 | func BenchmarkFileAsynchronous(b *testing.B) { 261 | log := NewLogger(100000) 262 | log.SetLogger("file", `{"filename":"test4.log"}`) 263 | log.Async() 264 | for i := 0; i < b.N; i++ { 265 | log.Debug("debug") 266 | } 267 | os.Remove("test4.log") 268 | } 269 | 270 | func BenchmarkFileCallDepth(b *testing.B) { 271 | log := NewLogger(100000) 272 | log.SetLogger("file", `{"filename":"test4.log"}`) 273 | log.EnableFuncCallDepth(true) 274 | log.SetLogFuncCallDepth(2) 275 | for i := 0; i < b.N; i++ { 276 | log.Debug("debug") 277 | } 278 | os.Remove("test4.log") 279 | } 280 | 281 | func BenchmarkFileAsynchronousCallDepth(b *testing.B) { 282 | log := NewLogger(100000) 283 | log.SetLogger("file", `{"filename":"test4.log"}`) 284 | log.EnableFuncCallDepth(true) 285 | log.SetLogFuncCallDepth(2) 286 | log.Async() 287 | for i := 0; i < b.N; i++ { 288 | log.Debug("debug") 289 | } 290 | os.Remove("test4.log") 291 | } 292 | 293 | func BenchmarkFileOnGoroutine(b *testing.B) { 294 | log := NewLogger(100000) 295 | log.SetLogger("file", `{"filename":"test4.log"}`) 296 | for i := 0; i < b.N; i++ { 297 | go log.Debug("debug") 298 | } 299 | os.Remove("test4.log") 300 | } 301 | -------------------------------------------------------------------------------- /internal/logs/jianliao.go: -------------------------------------------------------------------------------- 1 | package logs 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "time" 9 | ) 10 | 11 | // JLWriter implements beego LoggerInterface and is used to send jiaoliao webhook 12 | type JLWriter struct { 13 | AuthorName string `json:"authorname"` 14 | Title string `json:"title"` 15 | WebhookURL string `json:"webhookurl"` 16 | RedirectURL string `json:"redirecturl,omitempty"` 17 | ImageURL string `json:"imageurl,omitempty"` 18 | Level int `json:"level"` 19 | } 20 | 21 | // newJLWriter create jiaoliao writer. 22 | func newJLWriter() Logger { 23 | return &JLWriter{Level: LevelTrace} 24 | } 25 | 26 | // Init JLWriter with json config string 27 | func (s *JLWriter) Init(jsonconfig string) error { 28 | return json.Unmarshal([]byte(jsonconfig), s) 29 | } 30 | 31 | // WriteMsg write message in smtp writer. 32 | // it will send an email with subject and only this message. 33 | func (s *JLWriter) WriteMsg(when time.Time, msg string, level int) error { 34 | if level > s.Level { 35 | return nil 36 | } 37 | 38 | text := fmt.Sprintf("%s %s", when.Format("2006-01-02 15:04:05"), msg) 39 | 40 | form := url.Values{} 41 | form.Add("authorName", s.AuthorName) 42 | form.Add("title", s.Title) 43 | form.Add("text", text) 44 | if s.RedirectURL != "" { 45 | form.Add("redirectUrl", s.RedirectURL) 46 | } 47 | if s.ImageURL != "" { 48 | form.Add("imageUrl", s.ImageURL) 49 | } 50 | 51 | resp, err := http.PostForm(s.WebhookURL, form) 52 | if err != nil { 53 | return err 54 | } 55 | defer resp.Body.Close() 56 | if resp.StatusCode != http.StatusOK { 57 | return fmt.Errorf("Post webhook failed %s %d", resp.Status, resp.StatusCode) 58 | } 59 | return nil 60 | } 61 | 62 | // Flush implementing method. empty. 63 | func (s *JLWriter) Flush() { 64 | } 65 | 66 | // Destroy implementing method. empty. 67 | func (s *JLWriter) Destroy() { 68 | } 69 | 70 | func init() { 71 | Register(AdapterJianLiao, newJLWriter) 72 | } 73 | -------------------------------------------------------------------------------- /internal/logs/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package logs provide a general log interface 16 | // Usage: 17 | // 18 | // import "github.com/astaxie/beego/logs" 19 | // 20 | // log := NewLogger(10000) 21 | // log.SetLogger("console", "") 22 | // 23 | // > the first params stand for how many channel 24 | // 25 | // Use it like this: 26 | // 27 | // log.Trace("trace") 28 | // log.Info("info") 29 | // log.Warn("warning") 30 | // log.Debug("debug") 31 | // log.Critical("critical") 32 | // 33 | // more docs http://beego.me/docs/module/logs.md 34 | package logs 35 | 36 | import ( 37 | "fmt" 38 | "log" 39 | "os" 40 | "path" 41 | "runtime" 42 | "strconv" 43 | "strings" 44 | "sync" 45 | "time" 46 | ) 47 | 48 | // RFC5424 log message levels. 49 | const ( 50 | LevelEmergency = iota 51 | LevelAlert 52 | LevelCritical 53 | LevelError 54 | LevelWarning 55 | LevelNotice 56 | LevelInformational 57 | LevelDebug 58 | ) 59 | 60 | // levelLogLogger is defined to implement log.Logger 61 | // the real log level will be LevelEmergency 62 | const levelLoggerImpl = -1 63 | 64 | // Name for adapter with beego official support 65 | const ( 66 | AdapterConsole = "console" 67 | AdapterFile = "file" 68 | AdapterMultiFile = "multifile" 69 | AdapterMail = "smtp" 70 | AdapterConn = "conn" 71 | AdapterEs = "es" 72 | AdapterJianLiao = "jianliao" 73 | AdapterSlack = "slack" 74 | AdapterAliLS = "alils" 75 | ) 76 | 77 | // Legacy log level constants to ensure backwards compatibility. 78 | const ( 79 | LevelInfo = LevelInformational 80 | LevelTrace = LevelDebug 81 | LevelWarn = LevelWarning 82 | ) 83 | 84 | type newLoggerFunc func() Logger 85 | 86 | // Logger defines the behavior of a log provider. 87 | type Logger interface { 88 | Init(config string) error 89 | WriteMsg(when time.Time, msg string, level int) error 90 | Destroy() 91 | Flush() 92 | } 93 | 94 | var adapters = make(map[string]newLoggerFunc) 95 | var levelPrefix = [LevelDebug + 1]string{"[M] ", "[A] ", "[C] ", "[E] ", "[W] ", "[N] ", "[I] ", "[D] "} 96 | 97 | // Register makes a log provide available by the provided name. 98 | // If Register is called twice with the same name or if driver is nil, 99 | // it panics. 100 | func Register(name string, log newLoggerFunc) { 101 | if log == nil { 102 | panic("logs: Register provide is nil") 103 | } 104 | if _, dup := adapters[name]; dup { 105 | panic("logs: Register called twice for provider " + name) 106 | } 107 | adapters[name] = log 108 | } 109 | 110 | // BeeLogger is default logger in beego application. 111 | // it can contain several providers and log message into all providers. 112 | type BeeLogger struct { 113 | lock sync.Mutex 114 | level int 115 | init bool 116 | enableFuncCallDepth bool 117 | loggerFuncCallDepth int 118 | asynchronous bool 119 | msgChanLen int64 120 | msgChan chan *logMsg 121 | signalChan chan string 122 | wg sync.WaitGroup 123 | outputs []*nameLogger 124 | } 125 | 126 | const defaultAsyncMsgLen = 1e3 127 | 128 | type nameLogger struct { 129 | Logger 130 | name string 131 | } 132 | 133 | type logMsg struct { 134 | level int 135 | msg string 136 | when time.Time 137 | } 138 | 139 | var logMsgPool *sync.Pool 140 | 141 | // NewLogger returns a new BeeLogger. 142 | // channelLen means the number of messages in chan(used where asynchronous is true). 143 | // if the buffering chan is full, logger adapters write to file or other way. 144 | func NewLogger(channelLens ...int64) *BeeLogger { 145 | bl := new(BeeLogger) 146 | bl.level = LevelDebug 147 | bl.loggerFuncCallDepth = 2 148 | bl.msgChanLen = append(channelLens, 0)[0] 149 | if bl.msgChanLen <= 0 { 150 | bl.msgChanLen = defaultAsyncMsgLen 151 | } 152 | bl.signalChan = make(chan string, 1) 153 | bl.setLogger(AdapterConsole) 154 | return bl 155 | } 156 | 157 | // Async set the log to asynchronous and start the goroutine 158 | func (bl *BeeLogger) Async(msgLen ...int64) *BeeLogger { 159 | bl.lock.Lock() 160 | defer bl.lock.Unlock() 161 | if bl.asynchronous { 162 | return bl 163 | } 164 | bl.asynchronous = true 165 | if len(msgLen) > 0 && msgLen[0] > 0 { 166 | bl.msgChanLen = msgLen[0] 167 | } 168 | bl.msgChan = make(chan *logMsg, bl.msgChanLen) 169 | logMsgPool = &sync.Pool{ 170 | New: func() interface{} { 171 | return &logMsg{} 172 | }, 173 | } 174 | bl.wg.Add(1) 175 | go bl.startLogger() 176 | return bl 177 | } 178 | 179 | // SetLogger provides a given logger adapter into BeeLogger with config string. 180 | // config need to be correct JSON as string: {"interval":360}. 181 | func (bl *BeeLogger) setLogger(adapterName string, configs ...string) error { 182 | config := append(configs, "{}")[0] 183 | for _, l := range bl.outputs { 184 | if l.name == adapterName { 185 | return fmt.Errorf("logs: duplicate adaptername %q (you have set this logger before)", adapterName) 186 | } 187 | } 188 | 189 | log, ok := adapters[adapterName] 190 | if !ok { 191 | return fmt.Errorf("logs: unknown adaptername %q (forgotten Register?)", adapterName) 192 | } 193 | 194 | lg := log() 195 | err := lg.Init(config) 196 | if err != nil { 197 | fmt.Fprintln(os.Stderr, "logs.BeeLogger.SetLogger: "+err.Error()) 198 | return err 199 | } 200 | bl.outputs = append(bl.outputs, &nameLogger{name: adapterName, Logger: lg}) 201 | return nil 202 | } 203 | 204 | // SetLogger provides a given logger adapter into BeeLogger with config string. 205 | // config need to be correct JSON as string: {"interval":360}. 206 | func (bl *BeeLogger) SetLogger(adapterName string, configs ...string) error { 207 | bl.lock.Lock() 208 | defer bl.lock.Unlock() 209 | if !bl.init { 210 | bl.outputs = []*nameLogger{} 211 | bl.init = true 212 | } 213 | return bl.setLogger(adapterName, configs...) 214 | } 215 | 216 | // DelLogger remove a logger adapter in BeeLogger. 217 | func (bl *BeeLogger) DelLogger(adapterName string) error { 218 | bl.lock.Lock() 219 | defer bl.lock.Unlock() 220 | outputs := []*nameLogger{} 221 | for _, lg := range bl.outputs { 222 | if lg.name == adapterName { 223 | lg.Destroy() 224 | } else { 225 | outputs = append(outputs, lg) 226 | } 227 | } 228 | if len(outputs) == len(bl.outputs) { 229 | return fmt.Errorf("logs: unknown adaptername %q (forgotten Register?)", adapterName) 230 | } 231 | bl.outputs = outputs 232 | return nil 233 | } 234 | 235 | func (bl *BeeLogger) writeToLoggers(when time.Time, msg string, level int) { 236 | for _, l := range bl.outputs { 237 | err := l.WriteMsg(when, msg, level) 238 | if err != nil { 239 | fmt.Fprintf(os.Stderr, "unable to WriteMsg to adapter:%v,error:%v\n", l.name, err) 240 | } 241 | } 242 | } 243 | 244 | func (bl *BeeLogger) Write(p []byte) (n int, err error) { 245 | if len(p) == 0 { 246 | return 0, nil 247 | } 248 | // writeMsg will always add a '\n' character 249 | if p[len(p)-1] == '\n' { 250 | p = p[0 : len(p)-1] 251 | } 252 | // set levelLoggerImpl to ensure all log message will be write out 253 | err = bl.writeMsg(levelLoggerImpl, string(p)) 254 | if err == nil { 255 | return len(p), err 256 | } 257 | return 0, err 258 | } 259 | 260 | func (bl *BeeLogger) writeMsg(logLevel int, msg string, v ...interface{}) error { 261 | if !bl.init { 262 | bl.lock.Lock() 263 | bl.setLogger(AdapterConsole) 264 | bl.lock.Unlock() 265 | } 266 | 267 | if len(v) > 0 { 268 | msg = fmt.Sprintf(msg, v...) 269 | } 270 | when := time.Now() 271 | if bl.enableFuncCallDepth { 272 | _, file, line, ok := runtime.Caller(bl.loggerFuncCallDepth) 273 | if !ok { 274 | file = "???" 275 | line = 0 276 | } 277 | _, filename := path.Split(file) 278 | msg = "[" + filename + ":" + strconv.Itoa(line) + "] " + msg 279 | } 280 | 281 | //set level info in front of filename info 282 | if logLevel == levelLoggerImpl { 283 | // set to emergency to ensure all log will be print out correctly 284 | logLevel = LevelEmergency 285 | } else { 286 | msg = levelPrefix[logLevel] + msg 287 | } 288 | 289 | if bl.asynchronous { 290 | lm := logMsgPool.Get().(*logMsg) 291 | lm.level = logLevel 292 | lm.msg = msg 293 | lm.when = when 294 | bl.msgChan <- lm 295 | } else { 296 | bl.writeToLoggers(when, msg, logLevel) 297 | } 298 | return nil 299 | } 300 | 301 | // SetLevel Set log message level. 302 | // If message level (such as LevelDebug) is higher than logger level (such as LevelWarning), 303 | // log providers will not even be sent the message. 304 | func (bl *BeeLogger) SetLevel(l int) { 305 | bl.level = l 306 | } 307 | 308 | // SetLogFuncCallDepth set log funcCallDepth 309 | func (bl *BeeLogger) SetLogFuncCallDepth(d int) { 310 | bl.loggerFuncCallDepth = d 311 | } 312 | 313 | // GetLogFuncCallDepth return log funcCallDepth for wrapper 314 | func (bl *BeeLogger) GetLogFuncCallDepth() int { 315 | return bl.loggerFuncCallDepth 316 | } 317 | 318 | // EnableFuncCallDepth enable log funcCallDepth 319 | func (bl *BeeLogger) EnableFuncCallDepth(b bool) { 320 | bl.enableFuncCallDepth = b 321 | } 322 | 323 | // start logger chan reading. 324 | // when chan is not empty, write logs. 325 | func (bl *BeeLogger) startLogger() { 326 | gameOver := false 327 | for { 328 | select { 329 | case bm := <-bl.msgChan: 330 | bl.writeToLoggers(bm.when, bm.msg, bm.level) 331 | logMsgPool.Put(bm) 332 | case sg := <-bl.signalChan: 333 | // Now should only send "flush" or "close" to bl.signalChan 334 | bl.flush() 335 | if sg == "close" { 336 | for _, l := range bl.outputs { 337 | l.Destroy() 338 | } 339 | bl.outputs = nil 340 | gameOver = true 341 | } 342 | bl.wg.Done() 343 | } 344 | if gameOver { 345 | break 346 | } 347 | } 348 | } 349 | 350 | // Emergency Log EMERGENCY level message. 351 | func (bl *BeeLogger) Emergency(format string, v ...interface{}) { 352 | if LevelEmergency > bl.level { 353 | return 354 | } 355 | bl.writeMsg(LevelEmergency, format, v...) 356 | } 357 | 358 | // Alert Log ALERT level message. 359 | func (bl *BeeLogger) Alert(format string, v ...interface{}) { 360 | if LevelAlert > bl.level { 361 | return 362 | } 363 | bl.writeMsg(LevelAlert, format, v...) 364 | } 365 | 366 | // Critical Log CRITICAL level message. 367 | func (bl *BeeLogger) Critical(format string, v ...interface{}) { 368 | if LevelCritical > bl.level { 369 | return 370 | } 371 | bl.writeMsg(LevelCritical, format, v...) 372 | } 373 | 374 | // Error Log ERROR level message. 375 | func (bl *BeeLogger) Error(format string, v ...interface{}) { 376 | if LevelError > bl.level { 377 | return 378 | } 379 | bl.writeMsg(LevelError, format, v...) 380 | } 381 | 382 | // Warning Log WARNING level message. 383 | func (bl *BeeLogger) Warning(format string, v ...interface{}) { 384 | if LevelWarn > bl.level { 385 | return 386 | } 387 | bl.writeMsg(LevelWarn, format, v...) 388 | } 389 | 390 | // Notice Log NOTICE level message. 391 | func (bl *BeeLogger) Notice(format string, v ...interface{}) { 392 | if LevelNotice > bl.level { 393 | return 394 | } 395 | bl.writeMsg(LevelNotice, format, v...) 396 | } 397 | 398 | // Informational Log INFORMATIONAL level message. 399 | func (bl *BeeLogger) Informational(format string, v ...interface{}) { 400 | if LevelInfo > bl.level { 401 | return 402 | } 403 | bl.writeMsg(LevelInfo, format, v...) 404 | } 405 | 406 | // Debug Log DEBUG level message. 407 | func (bl *BeeLogger) Debug(format string, v ...interface{}) { 408 | if LevelDebug > bl.level { 409 | return 410 | } 411 | bl.writeMsg(LevelDebug, format, v...) 412 | } 413 | 414 | // Warn Log WARN level message. 415 | // compatibility alias for Warning() 416 | func (bl *BeeLogger) Warn(format string, v ...interface{}) { 417 | if LevelWarn > bl.level { 418 | return 419 | } 420 | bl.writeMsg(LevelWarn, format, v...) 421 | } 422 | 423 | // Info Log INFO level message. 424 | // compatibility alias for Informational() 425 | func (bl *BeeLogger) Info(format string, v ...interface{}) { 426 | if LevelInfo > bl.level { 427 | return 428 | } 429 | bl.writeMsg(LevelInfo, format, v...) 430 | } 431 | 432 | // Trace Log TRACE level message. 433 | // compatibility alias for Debug() 434 | func (bl *BeeLogger) Trace(format string, v ...interface{}) { 435 | if LevelDebug > bl.level { 436 | return 437 | } 438 | bl.writeMsg(LevelDebug, format, v...) 439 | } 440 | 441 | // Flush flush all chan data. 442 | func (bl *BeeLogger) Flush() { 443 | if bl.asynchronous { 444 | bl.signalChan <- "flush" 445 | bl.wg.Wait() 446 | bl.wg.Add(1) 447 | return 448 | } 449 | bl.flush() 450 | } 451 | 452 | // Close close logger, flush all chan data and destroy all adapters in BeeLogger. 453 | func (bl *BeeLogger) Close() { 454 | if bl.asynchronous { 455 | bl.signalChan <- "close" 456 | bl.wg.Wait() 457 | close(bl.msgChan) 458 | } else { 459 | bl.flush() 460 | for _, l := range bl.outputs { 461 | l.Destroy() 462 | } 463 | bl.outputs = nil 464 | } 465 | close(bl.signalChan) 466 | } 467 | 468 | // Reset close all outputs, and set bl.outputs to nil 469 | func (bl *BeeLogger) Reset() { 470 | bl.Flush() 471 | for _, l := range bl.outputs { 472 | l.Destroy() 473 | } 474 | bl.outputs = nil 475 | } 476 | 477 | func (bl *BeeLogger) flush() { 478 | if bl.asynchronous { 479 | for { 480 | if len(bl.msgChan) > 0 { 481 | bm := <-bl.msgChan 482 | bl.writeToLoggers(bm.when, bm.msg, bm.level) 483 | logMsgPool.Put(bm) 484 | continue 485 | } 486 | break 487 | } 488 | } 489 | for _, l := range bl.outputs { 490 | l.Flush() 491 | } 492 | } 493 | 494 | // beeLogger references the used application logger. 495 | var beeLogger = NewLogger() 496 | 497 | // GetBeeLogger returns the default BeeLogger 498 | func GetBeeLogger() *BeeLogger { 499 | return beeLogger 500 | } 501 | 502 | var beeLoggerMap = struct { 503 | sync.RWMutex 504 | logs map[string]*log.Logger 505 | }{ 506 | logs: map[string]*log.Logger{}, 507 | } 508 | 509 | // GetLogger returns the default BeeLogger 510 | func GetLogger(prefixes ...string) *log.Logger { 511 | prefix := append(prefixes, "")[0] 512 | if prefix != "" { 513 | prefix = fmt.Sprintf(`[%s] `, strings.ToUpper(prefix)) 514 | } 515 | beeLoggerMap.RLock() 516 | l, ok := beeLoggerMap.logs[prefix] 517 | if ok { 518 | beeLoggerMap.RUnlock() 519 | return l 520 | } 521 | beeLoggerMap.RUnlock() 522 | beeLoggerMap.Lock() 523 | defer beeLoggerMap.Unlock() 524 | l, ok = beeLoggerMap.logs[prefix] 525 | if !ok { 526 | l = log.New(beeLogger, prefix, 0) 527 | beeLoggerMap.logs[prefix] = l 528 | } 529 | return l 530 | } 531 | 532 | // Reset will remove all the adapter 533 | func Reset() { 534 | beeLogger.Reset() 535 | } 536 | 537 | // Async set the beelogger with Async mode and hold msglen messages 538 | func Async(msgLen ...int64) *BeeLogger { 539 | return beeLogger.Async(msgLen...) 540 | } 541 | 542 | // SetLevel sets the global log level used by the simple logger. 543 | func SetLevel(l int) { 544 | beeLogger.SetLevel(l) 545 | } 546 | 547 | var levelMap = map[string]int{ 548 | "debug": LevelDebug, 549 | "info": LevelInfo, 550 | "warn": LevelWarn, 551 | "error": LevelError, 552 | "critical": LevelCritical, 553 | } 554 | 555 | func Level(l string) { 556 | lvl, ok := levelMap[l] 557 | if !ok { 558 | lvl = LevelInfo 559 | } 560 | beeLogger.SetLevel(lvl) 561 | } 562 | 563 | func Init(path, level string, maxDay int64) { 564 | param := fmt.Sprintf(`{"filename": "%s", "maxdays": %d}`, path, maxDay) 565 | beeLogger.SetLogger(AdapterFile, param) 566 | beeLogger.SetLogFuncCallDepth(3) 567 | beeLogger.EnableFuncCallDepth(true) 568 | Level(level) 569 | } 570 | 571 | // EnableFuncCallDepth enable log funcCallDepth 572 | func EnableFuncCallDepth(b bool) { 573 | beeLogger.enableFuncCallDepth = b 574 | } 575 | 576 | // SetLogFuncCall set the CallDepth, default is 4 577 | func SetLogFuncCall(b bool) { 578 | beeLogger.EnableFuncCallDepth(b) 579 | beeLogger.SetLogFuncCallDepth(4) 580 | } 581 | 582 | // SetLogFuncCallDepth set log funcCallDepth 583 | func SetLogFuncCallDepth(d int) { 584 | beeLogger.loggerFuncCallDepth = d 585 | } 586 | 587 | // SetLogger sets a new logger. 588 | func SetLogger(adapter string, config ...string) error { 589 | return beeLogger.SetLogger(adapter, config...) 590 | } 591 | 592 | // Emergency logs a message at emergency level. 593 | func Emergency(f interface{}, v ...interface{}) { 594 | beeLogger.Emergency(formatLog(f, v...)) 595 | } 596 | 597 | // Alert logs a message at alert level. 598 | func Alert(f interface{}, v ...interface{}) { 599 | beeLogger.Alert(formatLog(f, v...)) 600 | } 601 | 602 | // Critical logs a message at critical level. 603 | func Critical(f interface{}, v ...interface{}) { 604 | beeLogger.Critical(formatLog(f, v...)) 605 | } 606 | 607 | // Error logs a message at error level. 608 | func Error(f interface{}, v ...interface{}) { 609 | beeLogger.Error(formatLog(f, v...)) 610 | } 611 | 612 | // Warning logs a message at warning level. 613 | func Warning(f interface{}, v ...interface{}) { 614 | beeLogger.Warn(formatLog(f, v...)) 615 | } 616 | 617 | // Warn compatibility alias for Warning() 618 | func Warn(f interface{}, v ...interface{}) { 619 | beeLogger.Warn(formatLog(f, v...)) 620 | } 621 | 622 | // Notice logs a message at notice level. 623 | func Notice(f interface{}, v ...interface{}) { 624 | beeLogger.Notice(formatLog(f, v...)) 625 | } 626 | 627 | // Informational logs a message at info level. 628 | func Informational(f interface{}, v ...interface{}) { 629 | beeLogger.Info(formatLog(f, v...)) 630 | } 631 | 632 | // Info compatibility alias for Warning() 633 | func Info(f interface{}, v ...interface{}) { 634 | beeLogger.Info(formatLog(f, v...)) 635 | } 636 | 637 | // Debug logs a message at debug level. 638 | func Debug(f interface{}, v ...interface{}) { 639 | beeLogger.Debug(formatLog(f, v...)) 640 | } 641 | 642 | // Trace logs a message at trace level. 643 | // compatibility alias for Warning() 644 | func Trace(f interface{}, v ...interface{}) { 645 | beeLogger.Trace(formatLog(f, v...)) 646 | } 647 | 648 | func formatLog(f interface{}, v ...interface{}) string { 649 | var msg string 650 | switch f.(type) { 651 | case string: 652 | msg = f.(string) 653 | if len(v) == 0 { 654 | return msg 655 | } 656 | if strings.Contains(msg, "%") && !strings.Contains(msg, "%%") { 657 | //format string 658 | } else { 659 | //do not contain format char 660 | msg += strings.Repeat(" %v", len(v)) 661 | } 662 | default: 663 | msg = fmt.Sprint(f) 664 | if len(v) == 0 { 665 | return msg 666 | } 667 | msg += strings.Repeat(" %v", len(v)) 668 | } 669 | return fmt.Sprintf(msg, v...) 670 | } 671 | -------------------------------------------------------------------------------- /internal/logs/logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "fmt" 19 | "io" 20 | "os" 21 | "sync" 22 | "time" 23 | ) 24 | 25 | type logWriter struct { 26 | sync.Mutex 27 | writer io.Writer 28 | } 29 | 30 | func newLogWriter(wr io.Writer) *logWriter { 31 | return &logWriter{writer: wr} 32 | } 33 | 34 | func (lg *logWriter) println(when time.Time, msg string) { 35 | lg.Lock() 36 | h, _ := formatTimeHeader(when) 37 | lg.writer.Write(append(append(h, msg...), '\n')) 38 | lg.Unlock() 39 | } 40 | 41 | type outputMode int 42 | 43 | // DiscardNonColorEscSeq supports the divided color escape sequence. 44 | // But non-color escape sequence is not output. 45 | // Please use the OutputNonColorEscSeq If you want to output a non-color 46 | // escape sequences such as ncurses. However, it does not support the divided 47 | // color escape sequence. 48 | const ( 49 | _ outputMode = iota 50 | DiscardNonColorEscSeq 51 | OutputNonColorEscSeq 52 | ) 53 | 54 | // NewAnsiColorWriter creates and initializes a new ansiColorWriter 55 | // using io.Writer w as its initial contents. 56 | // In the console of Windows, which change the foreground and background 57 | // colors of the text by the escape sequence. 58 | // In the console of other systems, which writes to w all text. 59 | func NewAnsiColorWriter(w io.Writer) io.Writer { 60 | return NewModeAnsiColorWriter(w, DiscardNonColorEscSeq) 61 | } 62 | 63 | // NewModeAnsiColorWriter create and initializes a new ansiColorWriter 64 | // by specifying the outputMode. 65 | func NewModeAnsiColorWriter(w io.Writer, mode outputMode) io.Writer { 66 | if _, ok := w.(*ansiColorWriter); !ok { 67 | return &ansiColorWriter{ 68 | w: w, 69 | mode: mode, 70 | } 71 | } 72 | return w 73 | } 74 | 75 | // Cheap integer to fixed-width decimal ASCII. Give a negative width to avoid zero-padding. 76 | func itoa(buf *[]byte, i int, wid int) { 77 | // Assemble decimal in reverse order. 78 | var b [6]byte 79 | bp := len(b) - 1 80 | for i >= 10 || wid > 1 { 81 | wid-- 82 | q := i / 10 83 | b[bp] = byte('0' + i - q*10) 84 | bp-- 85 | i = q 86 | } 87 | // i < 10 88 | b[bp] = byte('0' + i) 89 | *buf = append(*buf, b[bp:]...) 90 | } 91 | 92 | func formatTimeHeader(when time.Time) ([]byte, int) { 93 | buf := make([]byte, 0, 27) 94 | year, month, day := when.Date() 95 | itoa(&buf, year, 4) 96 | buf = append(buf, '/') 97 | itoa(&buf, int(month), 2) 98 | buf = append(buf, '/') 99 | itoa(&buf, day, 2) 100 | buf = append(buf, ' ') 101 | hour, min, sec := when.Clock() 102 | itoa(&buf, hour, 2) 103 | buf = append(buf, ':') 104 | itoa(&buf, min, 2) 105 | buf = append(buf, ':') 106 | itoa(&buf, sec, 2) 107 | buf = append(buf, '.') 108 | itoa(&buf, when.Nanosecond()/1e3, 6) 109 | buf = append(buf, ' ') 110 | return buf, day 111 | } 112 | 113 | var ( 114 | green = string([]byte{27, 91, 57, 55, 59, 52, 50, 109}) 115 | white = string([]byte{27, 91, 57, 48, 59, 52, 55, 109}) 116 | yellow = string([]byte{27, 91, 57, 55, 59, 52, 51, 109}) 117 | red = string([]byte{27, 91, 57, 55, 59, 52, 49, 109}) 118 | blue = string([]byte{27, 91, 57, 55, 59, 52, 52, 109}) 119 | magenta = string([]byte{27, 91, 57, 55, 59, 52, 53, 109}) 120 | cyan = string([]byte{27, 91, 57, 55, 59, 52, 54, 109}) 121 | 122 | w32Green = string([]byte{27, 91, 52, 50, 109}) 123 | w32White = string([]byte{27, 91, 52, 55, 109}) 124 | w32Yellow = string([]byte{27, 91, 52, 51, 109}) 125 | w32Red = string([]byte{27, 91, 52, 49, 109}) 126 | w32Blue = string([]byte{27, 91, 52, 52, 109}) 127 | w32Magenta = string([]byte{27, 91, 52, 53, 109}) 128 | w32Cyan = string([]byte{27, 91, 52, 54, 109}) 129 | 130 | reset = string([]byte{27, 91, 48, 109}) 131 | ) 132 | 133 | // ColorByStatus return color by http code 134 | // 2xx return Green 135 | // 3xx return White 136 | // 4xx return Yellow 137 | // 5xx return Red 138 | func ColorByStatus(cond bool, code int) string { 139 | switch { 140 | case code >= 200 && code < 300: 141 | return map[bool]string{true: green, false: w32Green}[cond] 142 | case code >= 300 && code < 400: 143 | return map[bool]string{true: white, false: w32White}[cond] 144 | case code >= 400 && code < 500: 145 | return map[bool]string{true: yellow, false: w32Yellow}[cond] 146 | default: 147 | return map[bool]string{true: red, false: w32Red}[cond] 148 | } 149 | } 150 | 151 | // ColorByMethod return color by http code 152 | // GET return Blue 153 | // POST return Cyan 154 | // PUT return Yellow 155 | // DELETE return Red 156 | // PATCH return Green 157 | // HEAD return Magenta 158 | // OPTIONS return WHITE 159 | func ColorByMethod(cond bool, method string) string { 160 | switch method { 161 | case "GET": 162 | return map[bool]string{true: blue, false: w32Blue}[cond] 163 | case "POST": 164 | return map[bool]string{true: cyan, false: w32Cyan}[cond] 165 | case "PUT": 166 | return map[bool]string{true: yellow, false: w32Yellow}[cond] 167 | case "DELETE": 168 | return map[bool]string{true: red, false: w32Red}[cond] 169 | case "PATCH": 170 | return map[bool]string{true: green, false: w32Green}[cond] 171 | case "HEAD": 172 | return map[bool]string{true: magenta, false: w32Magenta}[cond] 173 | case "OPTIONS": 174 | return map[bool]string{true: white, false: w32White}[cond] 175 | default: 176 | return reset 177 | } 178 | } 179 | 180 | // Guard Mutex to guarantee atomic of W32Debug(string) function 181 | var mu sync.Mutex 182 | 183 | // W32Debug Helper method to output colored logs in Windows terminals 184 | func W32Debug(msg string) { 185 | mu.Lock() 186 | defer mu.Unlock() 187 | 188 | current := time.Now() 189 | w := NewAnsiColorWriter(os.Stdout) 190 | 191 | fmt.Fprintf(w, "[beego] %v %s\n", current.Format("2006/01/02 - 15:04:05"), msg) 192 | } 193 | -------------------------------------------------------------------------------- /internal/logs/logger_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "bytes" 19 | "testing" 20 | "time" 21 | ) 22 | 23 | func TestFormatHeader_0(t *testing.T) { 24 | tm := time.Now() 25 | if tm.Year() >= 2100 { 26 | t.FailNow() 27 | } 28 | dur := time.Second 29 | for { 30 | if tm.Year() >= 2100 { 31 | break 32 | } 33 | h, _ := formatTimeHeader(tm) 34 | if tm.Format("2006/01/02 15:04:05 ") != string(h) { 35 | t.Log(tm) 36 | t.FailNow() 37 | } 38 | tm = tm.Add(dur) 39 | dur *= 2 40 | } 41 | } 42 | 43 | func TestFormatHeader_1(t *testing.T) { 44 | tm := time.Now() 45 | year := tm.Year() 46 | dur := time.Second 47 | for { 48 | if tm.Year() >= year+1 { 49 | break 50 | } 51 | h, _ := formatTimeHeader(tm) 52 | if tm.Format("2006/01/02 15:04:05 ") != string(h) { 53 | t.Log(tm) 54 | t.FailNow() 55 | } 56 | tm = tm.Add(dur) 57 | } 58 | } 59 | 60 | func TestNewAnsiColor1(t *testing.T) { 61 | inner := bytes.NewBufferString("") 62 | w := NewAnsiColorWriter(inner) 63 | if w == inner { 64 | t.Errorf("Get %#v, want %#v", w, inner) 65 | } 66 | } 67 | 68 | func TestNewAnsiColor2(t *testing.T) { 69 | inner := bytes.NewBufferString("") 70 | w1 := NewAnsiColorWriter(inner) 71 | w2 := NewAnsiColorWriter(w1) 72 | if w1 != w2 { 73 | t.Errorf("Get %#v, want %#v", w1, w2) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /internal/logs/multifile.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "encoding/json" 19 | "time" 20 | ) 21 | 22 | // A filesLogWriter manages several fileLogWriter 23 | // filesLogWriter will write logs to the file in json configuration and write the same level log to correspond file 24 | // means if the file name in configuration is project.log filesLogWriter will create project.error.log/project.debug.log 25 | // and write the error-level logs to project.error.log and write the debug-level logs to project.debug.log 26 | // the rotate attribute also acts like fileLogWriter 27 | type multiFileLogWriter struct { 28 | writers [LevelDebug + 1 + 1]*fileLogWriter // the last one for fullLogWriter 29 | fullLogWriter *fileLogWriter 30 | Separate []string `json:"separate"` 31 | } 32 | 33 | var levelNames = [...]string{"emergency", "alert", "critical", "error", "warning", "notice", "info", "debug"} 34 | 35 | // Init file logger with json config. 36 | // jsonConfig like: 37 | // { 38 | // "filename":"logs/beego.log", 39 | // "maxLines":0, 40 | // "maxsize":0, 41 | // "daily":true, 42 | // "maxDays":15, 43 | // "rotate":true, 44 | // "perm":0600, 45 | // "separate":["emergency", "alert", "critical", "error", "warning", "notice", "info", "debug"], 46 | // } 47 | 48 | func (f *multiFileLogWriter) Init(config string) error { 49 | writer := newFileWriter().(*fileLogWriter) 50 | err := writer.Init(config) 51 | if err != nil { 52 | return err 53 | } 54 | f.fullLogWriter = writer 55 | f.writers[LevelDebug+1] = writer 56 | 57 | //unmarshal "separate" field to f.Separate 58 | json.Unmarshal([]byte(config), f) 59 | 60 | jsonMap := map[string]interface{}{} 61 | json.Unmarshal([]byte(config), &jsonMap) 62 | 63 | for i := LevelEmergency; i < LevelDebug+1; i++ { 64 | for _, v := range f.Separate { 65 | if v == levelNames[i] { 66 | jsonMap["filename"] = f.fullLogWriter.fileNameOnly + "." + levelNames[i] + f.fullLogWriter.suffix 67 | jsonMap["level"] = i 68 | bs, _ := json.Marshal(jsonMap) 69 | writer = newFileWriter().(*fileLogWriter) 70 | writer.Init(string(bs)) 71 | f.writers[i] = writer 72 | } 73 | } 74 | } 75 | 76 | return nil 77 | } 78 | 79 | func (f *multiFileLogWriter) Destroy() { 80 | for i := 0; i < len(f.writers); i++ { 81 | if f.writers[i] != nil { 82 | f.writers[i].Destroy() 83 | } 84 | } 85 | } 86 | 87 | func (f *multiFileLogWriter) WriteMsg(when time.Time, msg string, level int) error { 88 | if f.fullLogWriter != nil { 89 | f.fullLogWriter.WriteMsg(when, msg, level) 90 | } 91 | for i := 0; i < len(f.writers)-1; i++ { 92 | if f.writers[i] != nil { 93 | if level == f.writers[i].Level { 94 | f.writers[i].WriteMsg(when, msg, level) 95 | } 96 | } 97 | } 98 | return nil 99 | } 100 | 101 | func (f *multiFileLogWriter) Flush() { 102 | for i := 0; i < len(f.writers); i++ { 103 | if f.writers[i] != nil { 104 | f.writers[i].Flush() 105 | } 106 | } 107 | } 108 | 109 | // newFilesWriter create a FileLogWriter returning as LoggerInterface. 110 | func newFilesWriter() Logger { 111 | return &multiFileLogWriter{} 112 | } 113 | 114 | func init() { 115 | Register(AdapterMultiFile, newFilesWriter) 116 | } 117 | -------------------------------------------------------------------------------- /internal/logs/multifile_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "bufio" 19 | "os" 20 | "strconv" 21 | "strings" 22 | "testing" 23 | ) 24 | 25 | func TestFiles_1(t *testing.T) { 26 | log := NewLogger(10000) 27 | log.SetLogger("multifile", `{"filename":"test.log","separate":["emergency", "alert", "critical", "error", "warning", "notice", "info", "debug"]}`) 28 | log.Debug("debug") 29 | log.Informational("info") 30 | log.Notice("notice") 31 | log.Warning("warning") 32 | log.Error("error") 33 | log.Alert("alert") 34 | log.Critical("critical") 35 | log.Emergency("emergency") 36 | fns := []string{""} 37 | fns = append(fns, levelNames[0:]...) 38 | name := "test" 39 | suffix := ".log" 40 | for _, fn := range fns { 41 | 42 | file := name + suffix 43 | if fn != "" { 44 | file = name + "." + fn + suffix 45 | } 46 | f, err := os.Open(file) 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | b := bufio.NewReader(f) 51 | lineNum := 0 52 | lastLine := "" 53 | for { 54 | line, _, err := b.ReadLine() 55 | if err != nil { 56 | break 57 | } 58 | if len(line) > 0 { 59 | lastLine = string(line) 60 | lineNum++ 61 | } 62 | } 63 | var expected = 1 64 | if fn == "" { 65 | expected = LevelDebug + 1 66 | } 67 | if lineNum != expected { 68 | t.Fatal(file, "has", lineNum, "lines not "+strconv.Itoa(expected)+" lines") 69 | } 70 | if lineNum == 1 { 71 | if !strings.Contains(lastLine, fn) { 72 | t.Fatal(file + " " + lastLine + " not contains the log msg " + fn) 73 | } 74 | } 75 | os.Remove(file) 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /internal/logs/slack.go: -------------------------------------------------------------------------------- 1 | package logs 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "time" 9 | ) 10 | 11 | // SLACKWriter implements beego LoggerInterface and is used to send jiaoliao webhook 12 | type SLACKWriter struct { 13 | WebhookURL string `json:"webhookurl"` 14 | Level int `json:"level"` 15 | } 16 | 17 | // newSLACKWriter create jiaoliao writer. 18 | func newSLACKWriter() Logger { 19 | return &SLACKWriter{Level: LevelTrace} 20 | } 21 | 22 | // Init SLACKWriter with json config string 23 | func (s *SLACKWriter) Init(jsonconfig string) error { 24 | return json.Unmarshal([]byte(jsonconfig), s) 25 | } 26 | 27 | // WriteMsg write message in smtp writer. 28 | // it will send an email with subject and only this message. 29 | func (s *SLACKWriter) WriteMsg(when time.Time, msg string, level int) error { 30 | if level > s.Level { 31 | return nil 32 | } 33 | 34 | text := fmt.Sprintf("{\"text\": \"%s %s\"}", when.Format("2006-01-02 15:04:05"), msg) 35 | 36 | form := url.Values{} 37 | form.Add("payload", text) 38 | 39 | resp, err := http.PostForm(s.WebhookURL, form) 40 | if err != nil { 41 | return err 42 | } 43 | defer resp.Body.Close() 44 | if resp.StatusCode != http.StatusOK { 45 | return fmt.Errorf("Post webhook failed %s %d", resp.Status, resp.StatusCode) 46 | } 47 | return nil 48 | } 49 | 50 | // Flush implementing method. empty. 51 | func (s *SLACKWriter) Flush() { 52 | } 53 | 54 | // Destroy implementing method. empty. 55 | func (s *SLACKWriter) Destroy() { 56 | } 57 | 58 | func init() { 59 | Register(AdapterSlack, newSLACKWriter) 60 | } 61 | -------------------------------------------------------------------------------- /internal/logs/smtp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "crypto/tls" 19 | "encoding/json" 20 | "fmt" 21 | "net" 22 | "net/smtp" 23 | "strings" 24 | "time" 25 | ) 26 | 27 | // SMTPWriter implements LoggerInterface and is used to send emails via given SMTP-server. 28 | type SMTPWriter struct { 29 | Username string `json:"username"` 30 | Password string `json:"password"` 31 | Host string `json:"host"` 32 | Subject string `json:"subject"` 33 | FromAddress string `json:"fromAddress"` 34 | RecipientAddresses []string `json:"sendTos"` 35 | Level int `json:"level"` 36 | } 37 | 38 | // NewSMTPWriter create smtp writer. 39 | func newSMTPWriter() Logger { 40 | return &SMTPWriter{Level: LevelTrace} 41 | } 42 | 43 | // Init smtp writer with json config. 44 | // config like: 45 | // { 46 | // "username":"example@gmail.com", 47 | // "password:"password", 48 | // "host":"smtp.gmail.com:465", 49 | // "subject":"email title", 50 | // "fromAddress":"from@example.com", 51 | // "sendTos":["email1","email2"], 52 | // "level":LevelError 53 | // } 54 | func (s *SMTPWriter) Init(jsonconfig string) error { 55 | return json.Unmarshal([]byte(jsonconfig), s) 56 | } 57 | 58 | func (s *SMTPWriter) getSMTPAuth(host string) smtp.Auth { 59 | if len(strings.Trim(s.Username, " ")) == 0 && len(strings.Trim(s.Password, " ")) == 0 { 60 | return nil 61 | } 62 | return smtp.PlainAuth( 63 | "", 64 | s.Username, 65 | s.Password, 66 | host, 67 | ) 68 | } 69 | 70 | func (s *SMTPWriter) sendMail(hostAddressWithPort string, auth smtp.Auth, fromAddress string, recipients []string, msgContent []byte) error { 71 | client, err := smtp.Dial(hostAddressWithPort) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | host, _, _ := net.SplitHostPort(hostAddressWithPort) 77 | tlsConn := &tls.Config{ 78 | InsecureSkipVerify: true, 79 | ServerName: host, 80 | } 81 | if err = client.StartTLS(tlsConn); err != nil { 82 | return err 83 | } 84 | 85 | if auth != nil { 86 | if err = client.Auth(auth); err != nil { 87 | return err 88 | } 89 | } 90 | 91 | if err = client.Mail(fromAddress); err != nil { 92 | return err 93 | } 94 | 95 | for _, rec := range recipients { 96 | if err = client.Rcpt(rec); err != nil { 97 | return err 98 | } 99 | } 100 | 101 | w, err := client.Data() 102 | if err != nil { 103 | return err 104 | } 105 | _, err = w.Write(msgContent) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | err = w.Close() 111 | if err != nil { 112 | return err 113 | } 114 | 115 | return client.Quit() 116 | } 117 | 118 | // WriteMsg write message in smtp writer. 119 | // it will send an email with subject and only this message. 120 | func (s *SMTPWriter) WriteMsg(when time.Time, msg string, level int) error { 121 | if level > s.Level { 122 | return nil 123 | } 124 | 125 | hp := strings.Split(s.Host, ":") 126 | 127 | // Set up authentication information. 128 | auth := s.getSMTPAuth(hp[0]) 129 | 130 | // Connect to the server, authenticate, set the sender and recipient, 131 | // and send the email all in one step. 132 | contentType := "Content-Type: text/plain" + "; charset=UTF-8" 133 | mailmsg := []byte("To: " + strings.Join(s.RecipientAddresses, ";") + "\r\nFrom: " + s.FromAddress + "<" + s.FromAddress + 134 | ">\r\nSubject: " + s.Subject + "\r\n" + contentType + "\r\n\r\n" + fmt.Sprintf(".%s", when.Format("2006-01-02 15:04:05")) + msg) 135 | 136 | return s.sendMail(s.Host, auth, s.FromAddress, s.RecipientAddresses, mailmsg) 137 | } 138 | 139 | // Flush implementing method. empty. 140 | func (s *SMTPWriter) Flush() { 141 | } 142 | 143 | // Destroy implementing method. empty. 144 | func (s *SMTPWriter) Destroy() { 145 | } 146 | 147 | func init() { 148 | Register(AdapterMail, newSMTPWriter) 149 | } 150 | -------------------------------------------------------------------------------- /internal/logs/smtp_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 beego Author. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logs 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | ) 21 | 22 | func TestSmtp(t *testing.T) { 23 | log := NewLogger(10000) 24 | log.SetLogger("smtp", `{"username":"beegotest@gmail.com","password":"xxxxxxxx","host":"smtp.gmail.com:587","sendTos":["xiemengjun@gmail.com"]}`) 25 | log.Critical("sendmail critical") 26 | time.Sleep(time.Second * 30) 27 | } 28 | -------------------------------------------------------------------------------- /internal/proto/cs.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "io" 7 | "net" 8 | ) 9 | 10 | const ( 11 | CmdAuth = iota 12 | CmdHeartbeat 13 | CmdData 14 | ) 15 | 16 | type S2CHeartbeat struct { 17 | } 18 | 19 | type C2SHeartbeat struct{} 20 | 21 | type C2SAuth struct { 22 | Key string `json:"key" yaml:"key"` 23 | Domain string `json:"domain" yaml:"domain"` 24 | Forward []ForwardItem `json:"forwards" yaml:"forwards"` // request forwards, not real, it depends on opennotrd 25 | } 26 | 27 | type ForwardItem struct { 28 | // forward protocol. eg: tcp, udp, https, http 29 | Protocol string `json:"protocol" yaml:"protocol"` 30 | 31 | // forward ports 32 | // key is the port opennotrd listen 33 | // value is local port 34 | Ports map[int]string `json:"ports" yaml:"ports"` 35 | 36 | // local ip, default is 127.0.0.1 37 | // the traffic will be forward to $LocalIP:$LocalPort 38 | // for example: 127.0.0.1:8080. 192.168.31.65:8080 39 | LocalIP string `json:"localIP" yaml:"localIP"` 40 | 41 | // raw config pass to server 42 | RawConfig string `json:"rawConfig" yaml:"rawConfig"` 43 | } 44 | 45 | type S2CAuth struct { 46 | Domain string `json:"domain"` // uniq domain for opennotr 47 | Vip string `json:"vip"` // vip for opennotr 48 | ProxyInfos []*ProxyTuple `json:"proxyInfos"` // real proxy table 49 | } 50 | 51 | type ProxyTuple struct { 52 | Protocol string 53 | FromPort string 54 | ToPort string 55 | } 56 | 57 | type ProxyProtocol struct { 58 | Protocol string `json:"protocol"` 59 | SrcIP string `json:"sip"` 60 | SrcPort string `json:"sport"` 61 | DstIP string `json:"dip"` 62 | DstPort string `json:"dport"` 63 | } 64 | 65 | // 1字节版本 66 | // 1字节命令 67 | // 2字节长度 68 | type Header [4]byte 69 | 70 | func (h Header) Version() int { 71 | return int(h[0]) 72 | } 73 | 74 | func (h Header) Cmd() int { 75 | return int(h[1]) 76 | } 77 | 78 | func (h Header) Bodylen() int { 79 | return (int(h[2]) << 8) + int(h[3]) 80 | } 81 | 82 | func Read(conn net.Conn) (Header, []byte, error) { 83 | h := Header{} 84 | _, err := io.ReadFull(conn, h[:]) 85 | if err != nil { 86 | return h, nil, err 87 | } 88 | 89 | bodylen := h.Bodylen() 90 | if bodylen <= 0 { 91 | return h, nil, nil 92 | } 93 | 94 | body := make([]byte, bodylen) 95 | _, err = io.ReadFull(conn, body) 96 | if err != nil { 97 | return h, nil, err 98 | } 99 | 100 | return h, body, nil 101 | } 102 | 103 | func Write(conn net.Conn, cmd int, body []byte) error { 104 | bodylen := make([]byte, 2) 105 | binary.BigEndian.PutUint16(bodylen, uint16(len(body))) 106 | 107 | hdr := []byte{0x01, byte(cmd)} 108 | hdr = append(hdr, bodylen...) 109 | 110 | writebody := make([]byte, 0) 111 | writebody = append(writebody, hdr...) 112 | writebody = append(writebody, body...) 113 | _, err := conn.Write(writebody) 114 | return err 115 | } 116 | 117 | func WriteJSON(conn net.Conn, cmd int, obj interface{}) error { 118 | body, err := json.Marshal(obj) 119 | if err != nil { 120 | return err 121 | } 122 | return Write(conn, cmd, body) 123 | } 124 | 125 | func ReadJSON(conn net.Conn, obj interface{}) error { 126 | _, body, err := Read(conn) 127 | if err != nil { 128 | return err 129 | } 130 | 131 | return json.Unmarshal(body, obj) 132 | } 133 | -------------------------------------------------------------------------------- /opennotr/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net" 10 | "sync" 11 | "time" 12 | 13 | "github.com/ICKelin/opennotr/internal/proto" 14 | "github.com/xtaci/smux" 15 | ) 16 | 17 | type Client struct { 18 | srv string 19 | key string 20 | domain string 21 | forwards []proto.ForwardItem 22 | udppool sync.Pool 23 | tcppool sync.Pool 24 | } 25 | 26 | func NewClient(cfg *Config) *Client { 27 | return &Client{ 28 | srv: cfg.ServerAddr, 29 | key: cfg.Key, 30 | domain: cfg.Domain, 31 | forwards: cfg.Forwards, 32 | tcppool: sync.Pool{ 33 | New: func() interface{} { 34 | return make([]byte, 4096) 35 | }, 36 | }, 37 | udppool: sync.Pool{ 38 | New: func() interface{} { 39 | return make([]byte, 64*1024) 40 | }, 41 | }, 42 | } 43 | } 44 | 45 | func (c *Client) Run() { 46 | for { 47 | conn, err := net.Dial("tcp", c.srv) 48 | if err != nil { 49 | log.Println(err) 50 | time.Sleep(time.Second * 3) 51 | continue 52 | } 53 | 54 | c2sauth := &proto.C2SAuth{ 55 | Key: c.key, 56 | Domain: c.domain, 57 | Forward: c.forwards, 58 | } 59 | 60 | err = proto.WriteJSON(conn, proto.CmdAuth, c2sauth) 61 | if err != nil { 62 | log.Println(err) 63 | time.Sleep(time.Second * 3) 64 | continue 65 | } 66 | 67 | auth := proto.S2CAuth{} 68 | err = proto.ReadJSON(conn, &auth) 69 | if err != nil { 70 | log.Println(err) 71 | time.Sleep(time.Second * 3) 72 | continue 73 | } 74 | 75 | log.Println("connect success") 76 | log.Println("vhost:", auth.Vip) 77 | log.Println("domain:", auth.Domain) 78 | for _, item := range auth.ProxyInfos { 79 | fromaddr := fmt.Sprintf("%s:%s", auth.Domain, item.FromPort) 80 | if len(item.FromPort) == 0 { 81 | fromaddr = auth.Domain 82 | } 83 | log.Printf("%s://%s => 127.0.0.1:%s\n", item.Protocol, fromaddr, item.ToPort) 84 | } 85 | 86 | mux, err := smux.Client(conn, nil) 87 | if err != nil { 88 | log.Println(err) 89 | time.Sleep(time.Second * 3) 90 | continue 91 | } 92 | 93 | for { 94 | stream, err := mux.AcceptStream() 95 | if err != nil { 96 | log.Println(err) 97 | break 98 | } 99 | 100 | go c.handleStream(stream) 101 | } 102 | 103 | log.Println("reconnecting") 104 | time.Sleep(time.Second * 3) 105 | } 106 | } 107 | 108 | func (c *Client) handleStream(stream *smux.Stream) { 109 | lenbuf := make([]byte, 2) 110 | _, err := stream.Read(lenbuf) 111 | if err != nil { 112 | log.Println(err) 113 | stream.Close() 114 | return 115 | } 116 | 117 | bodylen := binary.BigEndian.Uint16(lenbuf) 118 | buf := make([]byte, bodylen) 119 | nr, err := io.ReadFull(stream, buf) 120 | if err != nil { 121 | log.Println(err) 122 | stream.Close() 123 | return 124 | } 125 | 126 | proxyProtocol := proto.ProxyProtocol{} 127 | err = json.Unmarshal(buf[:nr], &proxyProtocol) 128 | if err != nil { 129 | log.Println("unmarshal fail: ", err) 130 | return 131 | } 132 | switch proxyProtocol.Protocol { 133 | case "tcp": 134 | c.tcpProxy(stream, &proxyProtocol) 135 | case "udp": 136 | c.udpProxy(stream, &proxyProtocol) 137 | } 138 | } 139 | 140 | func (c *Client) tcpProxy(stream *smux.Stream, p *proto.ProxyProtocol) { 141 | addr := fmt.Sprintf("%s:%s", p.DstIP, p.DstPort) 142 | remoteConn, err := net.DialTimeout("tcp", addr, time.Second*10) 143 | if err != nil { 144 | log.Println(err) 145 | stream.Close() 146 | return 147 | } 148 | 149 | go func() { 150 | defer remoteConn.Close() 151 | defer stream.Close() 152 | obj := c.tcppool.Get() 153 | buf := obj.([]byte) 154 | defer c.tcppool.Put(buf) 155 | 156 | io.CopyBuffer(remoteConn, stream, buf) 157 | }() 158 | 159 | defer remoteConn.Close() 160 | defer stream.Close() 161 | obj := c.tcppool.Get() 162 | buf := obj.([]byte) 163 | defer c.tcppool.Put(buf) 164 | io.CopyBuffer(stream, remoteConn, buf) 165 | } 166 | 167 | func (c *Client) udpProxy(stream *smux.Stream, p *proto.ProxyProtocol) { 168 | addr := fmt.Sprintf("%s:%s", p.DstIP, p.DstPort) 169 | raddr, err := net.ResolveUDPAddr("udp", addr) 170 | if err != nil { 171 | log.Println(err) 172 | stream.Close() 173 | return 174 | } 175 | 176 | remoteConn, err := net.DialUDP("udp", nil, raddr) 177 | if err != nil { 178 | log.Println(err) 179 | return 180 | } 181 | 182 | go func() { 183 | defer remoteConn.Close() 184 | defer stream.Close() 185 | hdr := make([]byte, 2) 186 | for { 187 | 188 | _, err := io.ReadFull(stream, hdr) 189 | if err != nil { 190 | log.Println("read stream fail: ", err) 191 | break 192 | } 193 | nlen := binary.BigEndian.Uint16(hdr) 194 | buf := make([]byte, nlen) 195 | nr, err := io.ReadFull(stream, buf) 196 | if err != nil { 197 | log.Println("read stream body fail: ", err) 198 | break 199 | } 200 | 201 | remoteConn.Write(buf[:nr]) 202 | } 203 | }() 204 | 205 | defer remoteConn.Close() 206 | defer stream.Close() 207 | obj := c.udppool.Get() 208 | buf := obj.([]byte) 209 | defer c.udppool.Put(buf) 210 | for { 211 | nr, err := remoteConn.Read(buf) 212 | if err != nil { 213 | log.Println(err) 214 | break 215 | } 216 | 217 | bytes := encode(buf[:nr]) 218 | _, err = stream.Write(bytes) 219 | if err != nil { 220 | log.Println(err) 221 | break 222 | } 223 | } 224 | } 225 | 226 | func encode(raw []byte) []byte { 227 | buf := make([]byte, 2) 228 | binary.BigEndian.PutUint16(buf, uint16(len(raw))) 229 | buf = append(buf, raw...) 230 | return buf 231 | } 232 | -------------------------------------------------------------------------------- /opennotr/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | 7 | "github.com/ICKelin/opennotr/internal/proto" 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | type Config struct { 12 | ServerAddr string `yaml:"serverAddr"` 13 | Key string `yaml:"key"` 14 | Domain string `yaml:"domain"` 15 | Forwards []proto.ForwardItem `yaml:"forwards"` 16 | } 17 | 18 | func ParseConfig(path string) (*Config, error) { 19 | cnt, err := ioutil.ReadFile(path) 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | var cfg Config 25 | err = yaml.Unmarshal(cnt, &cfg) 26 | return &cfg, err 27 | } 28 | 29 | func (c *Config) String() string { 30 | cnt, _ := json.Marshal(c) 31 | return string(cnt) 32 | } 33 | -------------------------------------------------------------------------------- /opennotr/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | ) 7 | 8 | func main() { 9 | confpath := flag.String("conf", "", "config file path") 10 | flag.Parse() 11 | 12 | cfg, err := ParseConfig(*confpath) 13 | if err != nil { 14 | log.Println(err) 15 | return 16 | } 17 | 18 | cli := NewClient(cfg) 19 | cli.Run() 20 | } 21 | -------------------------------------------------------------------------------- /opennotrd/core/config.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | 7 | "gopkg.in/yaml.v2" 8 | ) 9 | 10 | type Config struct { 11 | ServerConfig ServerConfig `yaml:"server"` 12 | DHCPConfig DHCPConfig `yaml:"dhcp"` 13 | ResolverConfig ResolverConfig `yaml:"resolver"` 14 | TCPForwardConfig TCPForwardConfig `yaml:"tcpforward"` 15 | UDPForwardConfig UDPForwardConfig `yaml:"udpforward"` 16 | Plugins map[string]string `yaml:"plugin"` 17 | } 18 | 19 | type ServerConfig struct { 20 | ListenAddr string `yaml:"listen"` 21 | AuthKey string `yaml:"authKey"` 22 | Domain string `yaml:"domain"` 23 | } 24 | 25 | type TCPForwardConfig struct { 26 | ListenAddr string `yaml:"listen"` 27 | ReadTimeout int `yaml:"readTimeout"` 28 | WriteTimeout int `yaml:"writeTimeout"` 29 | } 30 | 31 | type UDPForwardConfig struct { 32 | ListenAddr string `yaml:"listen"` 33 | ReadTimeout int `yaml:"readTimeout"` 34 | WriteTimeout int `yaml:"writeTimeout"` 35 | SessionTimeout int `yaml:"sessionTimeout"` 36 | } 37 | 38 | type DHCPConfig struct { 39 | Cidr string `yaml:"cidr"` 40 | IP string `yaml:"ip"` 41 | } 42 | 43 | type ResolverConfig struct { 44 | EtcdEndpoints []string `yaml:"etcdEndpoints"` 45 | } 46 | 47 | func ParseConfig(path string) (*Config, error) { 48 | cnt, err := ioutil.ReadFile(path) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | var cfg Config 54 | err = yaml.Unmarshal(cnt, &cfg) 55 | return &cfg, err 56 | } 57 | 58 | func (c *Config) String() string { 59 | cnt, _ := json.Marshal(c) 60 | return string(cnt) 61 | } 62 | -------------------------------------------------------------------------------- /opennotrd/core/dhcp.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "sync" 7 | ) 8 | 9 | type DHCP struct { 10 | rw sync.Mutex 11 | cidr string 12 | localIP string 13 | free map[string]struct{} 14 | inuse map[string]struct{} 15 | } 16 | 17 | func NewDHCP(cidr string) (*DHCP, error) { 18 | begin, end, err := getIPRange(cidr) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | free := make(map[string]struct{}) 24 | inused := make(map[string]struct{}) 25 | for i := begin + 1; i < end; i++ { 26 | free[toIP(i)] = struct{}{} 27 | } 28 | 29 | return &DHCP{ 30 | free: free, 31 | inuse: inused, 32 | cidr: cidr, 33 | localIP: toIP(begin), 34 | }, nil 35 | } 36 | 37 | func (g *DHCP) GetCIDR() string { 38 | return g.cidr 39 | } 40 | 41 | func (g *DHCP) SelectIP() (string, error) { 42 | g.rw.Lock() 43 | defer g.rw.Unlock() 44 | 45 | for ip := range g.free { 46 | delete(g.free, ip) 47 | g.inuse[ip] = struct{}{} 48 | return ip, nil 49 | } 50 | 51 | return "", fmt.Errorf("no available ip") 52 | } 53 | 54 | func (g *DHCP) ReleaseIP(ip string) { 55 | g.rw.Lock() 56 | defer g.rw.Unlock() 57 | 58 | g.free[ip] = struct{}{} 59 | delete(g.inuse, ip) 60 | } 61 | 62 | func getIPRange(cidr string) (int32, int32, error) { 63 | ip, mask, err := net.ParseCIDR(cidr) 64 | if err != nil { 65 | return -1, -1, err 66 | } 67 | 68 | ipv4 := ip.To4() 69 | if ipv4 == nil { 70 | return -1, -1, fmt.Errorf("parse cidr fail") 71 | } 72 | 73 | one, _ := mask.Mask.Size() 74 | begin := (int32(ipv4[0]) << 24) + (int32(ipv4[1]) << 16) + (int32(ipv4[2]) << 8) + int32(ipv4[3]) 75 | end := begin | (1<<(32-one) - 1) 76 | 77 | return begin, end, nil 78 | } 79 | 80 | // int32 ip地址转换为string 81 | func toIP(iip int32) string { 82 | return fmt.Sprintf("%d.%d.%d.%d", byte(iip>>24), byte(iip>>16), byte(iip>>8), byte(iip)) 83 | } 84 | -------------------------------------------------------------------------------- /opennotrd/core/net.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "fmt" 7 | "net" 8 | "syscall" 9 | 10 | "github.com/ICKelin/opennotr/internal/proto" 11 | ) 12 | 13 | func checksumAdd(buf []byte, seed uint32) uint32 { 14 | sum := seed 15 | for i, l := 0, len(buf); i < l; i += 2 { 16 | j := i + 1 17 | if j == l { 18 | sum += uint32(buf[i]) << 8 19 | break 20 | } 21 | sum += uint32(buf[i])<<8 | uint32(buf[j]) 22 | } 23 | return sum 24 | } 25 | 26 | func checksumWrapper(seed uint32) uint16 { 27 | sum := seed 28 | for sum > 0xffff { 29 | sum = (sum >> 16) + (sum & 0xffff) 30 | } 31 | csum := ^uint16(sum) 32 | 33 | // RFC 768 34 | if csum == 0 { 35 | csum = 0xffff 36 | } 37 | return csum 38 | } 39 | 40 | func CheckSum(buf []byte) uint16 { 41 | return checksumWrapper(checksumAdd(buf, 0)) 42 | } 43 | 44 | func sendUDPViaRaw(fd int, src, dst *net.UDPAddr, payload []byte) error { 45 | iplen, ulen := uint16(28+len(payload)), uint16(8+len(payload)) 46 | if iplen > 65535 { 47 | return fmt.Errorf("too big packet") 48 | } 49 | 50 | // UDP checksum: sip + dip + udp-head + payload + PROTO + ulen 51 | data := make([]byte, iplen) 52 | data[9] = syscall.IPPROTO_UDP 53 | copy(data[12:16], src.IP.To4()) 54 | copy(data[16:20], dst.IP.To4()) 55 | data[20] = byte(src.Port >> 8) 56 | data[21] = byte(src.Port) 57 | data[22] = byte(dst.Port >> 8) 58 | data[23] = byte(dst.Port) 59 | data[24] = byte(ulen >> 8) 60 | data[25] = byte(ulen) 61 | copy(data[28:], payload) 62 | 63 | uc := checksumWrapper(checksumAdd(data, uint32(ulen))) 64 | data[26] = byte(uc >> 8) 65 | data[27] = byte(uc) 66 | 67 | data[0] = 0x45 68 | data[2] = byte(iplen >> 8) 69 | data[3] = byte(iplen) 70 | data[6] = 0x40 71 | data[8] = 64 72 | ipc := CheckSum(data[:20]) 73 | data[10] = byte(ipc >> 8) 74 | data[11] = byte(ipc) 75 | 76 | addr := syscall.SockaddrInet4{Port: dst.Port} 77 | copy(addr.Addr[:], data[16:20]) 78 | return syscall.Sendto(fd, data, 0, &addr) 79 | } 80 | 81 | func encode(raw []byte) []byte { 82 | buf := make([]byte, 2) 83 | binary.BigEndian.PutUint16(buf, uint16(len(raw))) 84 | buf = append(buf, raw...) 85 | return buf 86 | } 87 | 88 | func encodeProxyProtocol(protocol, sip, sport, dip, dport string) []byte { 89 | proxyProtocol := &proto.ProxyProtocol{ 90 | Protocol: protocol, 91 | SrcIP: sip, 92 | SrcPort: sport, 93 | DstIP: dip, 94 | DstPort: dport, 95 | } 96 | 97 | body, _ := json.Marshal(proxyProtocol) 98 | return encode(body) 99 | } 100 | -------------------------------------------------------------------------------- /opennotrd/core/resolver.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "strings" 8 | "time" 9 | 10 | "github.com/coreos/etcd/clientv3" 11 | ) 12 | 13 | type record struct { 14 | Host string `json:"host"` 15 | } 16 | 17 | type Resolver struct { 18 | endpoints []string 19 | cli *clientv3.Client 20 | } 21 | 22 | func NewResolve(endpoints []string) (*Resolver, error) { 23 | cli, err := clientv3.New(clientv3.Config{ 24 | Endpoints: endpoints, 25 | DialTimeout: time.Minute * 1, 26 | }) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | return &Resolver{ 32 | endpoints: endpoints, 33 | cli: cli, 34 | }, nil 35 | } 36 | 37 | func (r *Resolver) ApplyDomain(domain, ip string) error { 38 | sp := strings.Split(domain, ".") 39 | if len(sp) == 0 { 40 | return fmt.Errorf("invalid domain: %s", domain) 41 | } 42 | 43 | key := "/skydns" 44 | for i := len(sp) - 1; i >= 0; i-- { 45 | key = fmt.Sprintf("%s/%s", key, sp[i]) 46 | } 47 | 48 | value := &record{Host: ip} 49 | b, err := json.Marshal(value) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | _, err = r.cli.Put(context.Background(), key, string(b)) 55 | return err 56 | } 57 | -------------------------------------------------------------------------------- /opennotrd/core/server.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net" 7 | "net/http" 8 | "strings" 9 | "time" 10 | 11 | "github.com/ICKelin/opennotr/internal/logs" 12 | "github.com/ICKelin/opennotr/internal/proto" 13 | "github.com/ICKelin/opennotr/opennotrd/plugin" 14 | "github.com/xtaci/smux" 15 | ) 16 | 17 | type Server struct { 18 | cfg ServerConfig 19 | addr string 20 | authKey string 21 | domain string 22 | publicIP string 23 | 24 | // dhcp manager select/release ip for client 25 | dhcp *DHCP 26 | 27 | // call stream proxy for dynamic add/del tcp/udp proxy 28 | pluginMgr *plugin.PluginManager 29 | 30 | // resolver writes domains to etcd and it will be used by coredns 31 | resolver *Resolver 32 | 33 | // sess manager is the model of client session 34 | sessMgr *SessionManager 35 | } 36 | 37 | func NewServer(cfg ServerConfig, 38 | dhcp *DHCP, 39 | resolver *Resolver) *Server { 40 | return &Server{ 41 | cfg: cfg, 42 | addr: cfg.ListenAddr, 43 | authKey: cfg.AuthKey, 44 | domain: cfg.Domain, 45 | publicIP: publicIP(), 46 | dhcp: dhcp, 47 | pluginMgr: plugin.DefaultPluginManager(), 48 | resolver: resolver, 49 | sessMgr: GetSessionManager(), 50 | } 51 | } 52 | 53 | func (s *Server) ListenAndServe() error { 54 | listener, err := net.Listen("tcp", s.addr) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | for { 60 | conn, err := listener.Accept() 61 | if err != nil { 62 | return err 63 | } 64 | 65 | go s.onConn(conn) 66 | } 67 | } 68 | 69 | func (s *Server) onConn(conn net.Conn) { 70 | defer conn.Close() 71 | 72 | // auth key verify 73 | // currently we use auth key which configured in notrd.yaml 74 | auth := proto.C2SAuth{} 75 | err := proto.ReadJSON(conn, &auth) 76 | if err != nil { 77 | logs.Error("bad request, authorize fail: %v %v", err, auth) 78 | return 79 | } 80 | 81 | if auth.Key != s.authKey { 82 | logs.Error("verify key fail") 83 | return 84 | } 85 | 86 | // if client without domain 87 | // generate random domain base on time nano 88 | if len(auth.Domain) <= 0 { 89 | auth.Domain = fmt.Sprintf("%s.%s", randomDomain(time.Now().UnixNano()), s.domain) 90 | } 91 | 92 | // select a virtual ip for client. 93 | // a virtual ip is the ip address which can be use in our system 94 | // but cannot be used by other networks 95 | vip, err := s.dhcp.SelectIP() 96 | if err != nil { 97 | logs.Error("dhcp select ip fail: %v", err) 98 | return 99 | } 100 | 101 | // dynamic dns, write domain=>ip map to etcd 102 | // coredns will read records from etcd and reply to dns client 103 | if s.resolver != nil { 104 | err = s.resolver.ApplyDomain(auth.Domain, publicIP()) 105 | if err != nil { 106 | logs.Error("resolve domain fail: %v", err) 107 | return 108 | } 109 | } 110 | 111 | logs.Info("select vip: %s", vip) 112 | logs.Info("select domain: %s", auth.Domain) 113 | 114 | // create forward 115 | // 0.0.0.0:$publicPort => $vip:$localPort 116 | // 1. for from address, we listen 0.0.0.0:$publicPort 117 | // 2. for to address, we use $vip:$localPort 118 | // the vip is the virtual lan ip address 119 | // Domain is only use for restyproxy 120 | proxyInfos := make([]*proto.ProxyTuple, 0) 121 | for _, forward := range auth.Forward { 122 | for publicPort, localPort := range forward.Ports { 123 | item := &plugin.PluginMeta{ 124 | Protocol: forward.Protocol, 125 | From: fmt.Sprintf("0.0.0.0:%d", publicPort), 126 | To: fmt.Sprintf("%s:%s", vip, localPort), 127 | Domain: auth.Domain, 128 | RecycleSignal: make(chan struct{}), 129 | Ctx: forward.RawConfig, 130 | } 131 | 132 | p, err := s.pluginMgr.AddProxy(item) 133 | if err != nil { 134 | logs.Error("add proxy fail: %v", err) 135 | return 136 | } 137 | proxyInfos = append(proxyInfos, &proto.ProxyTuple{ 138 | Protocol: forward.Protocol, 139 | FromPort: p.FromPort, 140 | ToPort: p.ToPort, 141 | }) 142 | defer s.pluginMgr.DelProxy(item) 143 | } 144 | } 145 | 146 | reply := &proto.S2CAuth{ 147 | Vip: vip, 148 | Domain: auth.Domain, 149 | ProxyInfos: proxyInfos, 150 | } 151 | 152 | err = proto.WriteJSON(conn, proto.CmdAuth, reply) 153 | if err != nil { 154 | logs.Error("write json fail: %v", err) 155 | return 156 | } 157 | 158 | mux, err := smux.Server(conn, nil) 159 | if err != nil { 160 | logs.Error("smux server fail:%v", err) 161 | return 162 | } 163 | 164 | sess := newSession(mux, vip) 165 | s.sessMgr.AddSession(vip, sess) 166 | defer s.sessMgr.DeleteSession(vip) 167 | 168 | rttInterval := time.NewTicker(time.Millisecond * 500) 169 | for range rttInterval.C { 170 | if mux.IsClosed() { 171 | logs.Info("session %v close", sess.conn.RemoteAddr().String()) 172 | return 173 | } 174 | 175 | } 176 | } 177 | 178 | // randomDomain generate random domain for client 179 | func randomDomain(num int64) string { 180 | const ALPHABET = "123456789abcdefghijklmnopqrstuvwxyz" 181 | const BASE = int64(len(ALPHABET)) 182 | rs := "" 183 | for num > 0 { 184 | rs += string(ALPHABET[num%BASE]) 185 | num = num / BASE 186 | } 187 | 188 | return rs 189 | } 190 | 191 | // get public 192 | func publicIP() string { 193 | resp, err := http.Get("http://ipv4.icanhazip.com") 194 | if err != nil { 195 | logs.Error("get public ip fail: %v", err) 196 | panic(err) 197 | } 198 | 199 | defer resp.Body.Close() 200 | content, err := ioutil.ReadAll(resp.Body) 201 | if err != nil { 202 | panic(err) 203 | } 204 | 205 | str := string(content) 206 | idx := strings.LastIndex(str, "\n") 207 | return str[:idx] 208 | } 209 | -------------------------------------------------------------------------------- /opennotrd/core/session.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/xtaci/smux" 7 | ) 8 | 9 | var sessionMgr = &SessionManager{} 10 | 11 | // SessionManager defines the session info add/delete/get actions 12 | type SessionManager struct { 13 | sessions sync.Map 14 | } 15 | 16 | // GetSessionManager returs the singleton of session manager 17 | func GetSessionManager() *SessionManager { 18 | return sessionMgr 19 | } 20 | 21 | // Session defines each opennotr_client to opennotr_server connection 22 | type Session struct { 23 | conn *smux.Session 24 | rxbytes uint64 25 | txbytes uint64 26 | } 27 | 28 | func newSession(conn *smux.Session, vip string) *Session { 29 | return &Session{ 30 | conn: conn, 31 | } 32 | } 33 | 34 | func (mgr *SessionManager) AddSession(vip string, sess *Session) { 35 | mgr.sessions.Store(vip, sess) 36 | } 37 | 38 | func (mgr *SessionManager) GetSession(vip string) *Session { 39 | val, ok := mgr.sessions.Load(vip) 40 | if !ok { 41 | return nil 42 | } 43 | return val.(*Session) 44 | } 45 | 46 | func (mgr *SessionManager) DeleteSession(vip string) { 47 | mgr.sessions.Delete(vip) 48 | } 49 | -------------------------------------------------------------------------------- /opennotrd/core/tcpforward.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "io" 5 | "net" 6 | "sync" 7 | "syscall" 8 | "time" 9 | 10 | "github.com/ICKelin/opennotr/internal/logs" 11 | ) 12 | 13 | var ( 14 | // default tcp timeout(read, write), 10 seconds 15 | defaultTCPTimeout = 10 16 | ) 17 | 18 | type TCPForward struct { 19 | listenAddr string 20 | // writeTimeout defines the tcp connection write timeout in second 21 | // default value set to 10 seconds 22 | writeTimeout time.Duration 23 | 24 | // readTimeout defines the tcp connection write timeout in second 25 | // default value set to 10 seconds 26 | readTimeout time.Duration 27 | 28 | // the session manager is the global session manager 29 | // it stores opennotr_client to opennotr_server connection 30 | sessMgr *SessionManager 31 | } 32 | 33 | func NewTCPForward(cfg TCPForwardConfig) *TCPForward { 34 | tcpReadTimeout := cfg.ReadTimeout 35 | if tcpReadTimeout <= 0 { 36 | tcpReadTimeout = defaultTCPTimeout 37 | } 38 | 39 | tcpWriteTimeout := cfg.WriteTimeout 40 | if tcpWriteTimeout <= 0 { 41 | tcpWriteTimeout = int(defaultTCPTimeout) 42 | } 43 | return &TCPForward{ 44 | listenAddr: cfg.ListenAddr, 45 | writeTimeout: time.Duration(tcpWriteTimeout) * time.Second, 46 | readTimeout: time.Duration(tcpReadTimeout) * time.Second, 47 | sessMgr: GetSessionManager(), 48 | } 49 | } 50 | 51 | func (f *TCPForward) Listen() (net.Listener, error) { 52 | listener, err := net.Listen("tcp", f.listenAddr) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | // set socket with ip transparent option 58 | file, err := listener.(*net.TCPListener).File() 59 | if err != nil { 60 | return nil, err 61 | } 62 | defer file.Close() 63 | 64 | err = syscall.SetsockoptInt(int(file.Fd()), syscall.SOL_IP, syscall.IP_TRANSPARENT, 1) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | return listener, nil 70 | } 71 | 72 | func (f *TCPForward) Serve(listener net.Listener) error { 73 | for { 74 | conn, err := listener.Accept() 75 | if err != nil { 76 | logs.Error("accept fail: %v", err) 77 | break 78 | } 79 | 80 | go f.forwardTCP(conn) 81 | } 82 | 83 | return nil 84 | } 85 | 86 | func (f *TCPForward) forwardTCP(conn net.Conn) { 87 | defer conn.Close() 88 | 89 | dip, dport, _ := net.SplitHostPort(conn.LocalAddr().String()) 90 | sip, sport, _ := net.SplitHostPort(conn.RemoteAddr().String()) 91 | 92 | sess := f.sessMgr.GetSession(dip) 93 | if sess == nil { 94 | logs.Error("no route to host: %s", dip) 95 | return 96 | } 97 | 98 | stream, err := sess.conn.OpenStream() 99 | if err != nil { 100 | logs.Error("open stream fail: %v", err) 101 | return 102 | } 103 | defer stream.Close() 104 | 105 | // todo rewrite to client configuration 106 | targetIP := "127.0.0.1" 107 | bytes := encodeProxyProtocol("tcp", sip, sport, targetIP, dport) 108 | stream.SetWriteDeadline(time.Now().Add(f.writeTimeout)) 109 | _, err = stream.Write(bytes) 110 | stream.SetWriteDeadline(time.Time{}) 111 | if err != nil { 112 | logs.Error("stream write fail: %v", err) 113 | return 114 | } 115 | 116 | wg := &sync.WaitGroup{} 117 | wg.Add(1) 118 | defer wg.Wait() 119 | 120 | go func() { 121 | defer wg.Done() 122 | defer stream.Close() 123 | defer conn.Close() 124 | buf := make([]byte, 4096) 125 | io.CopyBuffer(stream, conn, buf) 126 | }() 127 | 128 | // todo: optimize mem alloc 129 | // one session will cause 4KB + 4KB buffer for io copy 130 | // and two goroutine 4KB mem used 131 | buf := make([]byte, 4096) 132 | io.CopyBuffer(conn, stream, buf) 133 | } 134 | -------------------------------------------------------------------------------- /opennotrd/core/tcpforward_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net" 7 | "net/http" 8 | _ "net/http/pprof" 9 | "os" 10 | "sync" 11 | "testing" 12 | "time" 13 | 14 | "github.com/xtaci/smux" 15 | ) 16 | 17 | func init() { 18 | go http.ListenAndServe("127.0.0.1:6060", nil) 19 | } 20 | 21 | // client -----> tproxy | opennotr server <------ opennotr client 22 | 23 | var backendAddr = "127.0.0.1:8522" 24 | var serverAddr = "127.0.0.1:8521" 25 | var tproxyAddr = "127.0.0.1:8520" 26 | var vip = "100.64.240.10" 27 | 28 | type mockConn struct { 29 | net.Conn 30 | addr mockAddr 31 | } 32 | 33 | type mockAddr struct{} 34 | 35 | func (addr mockAddr) Network() string { 36 | return "tcp" 37 | } 38 | 39 | func (addr mockAddr) String() string { 40 | return "100.64.240.10:8522" 41 | } 42 | 43 | func (c *mockConn) LocalAddr() net.Addr { 44 | return c.addr 45 | } 46 | 47 | func runBackend() { 48 | conn, err := net.Dial("tcp", serverAddr) 49 | if err != nil { 50 | panic(err) 51 | } 52 | defer conn.Close() 53 | sess, err := smux.Client(conn, nil) 54 | if err != nil { 55 | panic(err) 56 | } 57 | defer sess.Close() 58 | 59 | for { 60 | stream, err := sess.AcceptStream() 61 | if err != nil { 62 | panic(err) 63 | } 64 | 65 | go func() { 66 | defer stream.Close() 67 | buf := make([]byte, len("ping\n")) 68 | for { 69 | nr, err := stream.Read(buf) 70 | if err != nil { 71 | break 72 | } 73 | stream.Write(buf[:nr]) 74 | } 75 | }() 76 | } 77 | } 78 | 79 | func runserver(listener net.Listener) { 80 | for { 81 | conn, err := listener.Accept() 82 | if err != nil { 83 | break 84 | } 85 | 86 | go func() { 87 | sess, err := smux.Server(conn, nil) 88 | if err != nil { 89 | panic(err) 90 | } 91 | 92 | sessMgr := GetSessionManager() 93 | sessMgr.AddSession(vip, &Session{conn: sess}) 94 | fmt.Println("add session: ", vip) 95 | }() 96 | } 97 | } 98 | 99 | func runtproxy(tcpfw *TCPForward, listener net.Listener) { 100 | for { 101 | conn, err := listener.Accept() 102 | if err != nil { 103 | break 104 | } 105 | 106 | go func() { 107 | // forward test 108 | mConn := &mockConn{} 109 | mConn.Conn = conn 110 | tcpfw.forwardTCP(mConn) 111 | }() 112 | } 113 | } 114 | 115 | func TestTCPForward(t *testing.T) { 116 | // listen tproxy 117 | tcpfw := NewTCPForward(TCPForwardConfig{ 118 | ListenAddr: tproxyAddr, 119 | }) 120 | listener, err := tcpfw.Listen() 121 | if err != nil { 122 | t.Error(err) 123 | return 124 | } 125 | // defer listener.Close() 126 | 127 | srvlistener, err := net.Listen("tcp", serverAddr) 128 | if err != nil { 129 | t.Error(err) 130 | return 131 | } 132 | defer srvlistener.Close() 133 | 134 | go runBackend() 135 | go runserver(srvlistener) 136 | go runtproxy(tcpfw, listener) 137 | // wait for session created 138 | time.Sleep(time.Second * 1) 139 | conn, err := net.Dial("tcp", tproxyAddr) 140 | if err != nil { 141 | t.FailNow() 142 | } 143 | defer conn.Close() 144 | 145 | go func() { 146 | defer conn.Close() 147 | for i := 0; i < 10; i++ { 148 | conn.Write([]byte("ping\n")) 149 | time.Sleep(time.Second * 1) 150 | } 151 | fmt.Println("connection close") 152 | }() 153 | 154 | buf := make([]byte, 128) 155 | c := 0 156 | for { 157 | nr, err := conn.Read(buf) 158 | if err != nil { 159 | break 160 | } 161 | fmt.Printf("receive %d %s\n", c+1, string(buf[:nr])) 162 | c += 1 163 | } 164 | } 165 | 166 | func benchmark(t *testing.B, nconn int) { 167 | // listen tproxy 168 | tcpfw := NewTCPForward(TCPForwardConfig{ 169 | ListenAddr: tproxyAddr, 170 | }) 171 | listener, err := tcpfw.Listen() 172 | if err != nil { 173 | t.Error(err) 174 | return 175 | } 176 | // defer listener.Close() 177 | 178 | srvlistener, err := net.Listen("tcp", serverAddr) 179 | if err != nil { 180 | t.Error(err) 181 | return 182 | } 183 | defer srvlistener.Close() 184 | 185 | go runBackend() 186 | go runserver(srvlistener) 187 | go runtproxy(tcpfw, listener) 188 | 189 | // wait for session created 190 | time.Sleep(time.Second * 1) 191 | wg := sync.WaitGroup{} 192 | wg.Add(nconn) 193 | defer wg.Wait() 194 | for i := 0; i < nconn; i++ { 195 | go func() { 196 | defer wg.Done() 197 | conn, err := net.Dial("tcp", tproxyAddr) 198 | if err != nil { 199 | t.FailNow() 200 | } 201 | defer conn.Close() 202 | 203 | go func() { 204 | defer conn.Close() 205 | for i := 0; i < 10; i++ { 206 | conn.Write([]byte("ping\n")) 207 | time.Sleep(time.Second * 1) 208 | } 209 | }() 210 | fp, _ := os.Open(os.DevNull) 211 | defer fp.Close() 212 | io.Copy(fp, conn) 213 | }() 214 | } 215 | } 216 | 217 | func Benchmark1K(b *testing.B) { 218 | benchmark(b, 1024) 219 | } 220 | 221 | func Benchmark2K(b *testing.B) { 222 | benchmark(b, 1024*2) 223 | } 224 | 225 | func Benchmark4K(b *testing.B) { 226 | benchmark(b, 1024*4) 227 | } 228 | 229 | func Benchmark8K(b *testing.B) { 230 | benchmark(b, 1024*8) 231 | } 232 | 233 | func Benchmark10K(b *testing.B) { 234 | benchmark(b, 1024*10) 235 | } 236 | 237 | func Benchmark14K(b *testing.B) { 238 | benchmark(b, 1024*14) 239 | } 240 | -------------------------------------------------------------------------------- /opennotrd/core/udpforward.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "io" 8 | "net" 9 | "sync" 10 | "syscall" 11 | "time" 12 | "unsafe" 13 | 14 | "github.com/ICKelin/opennotr/internal/logs" 15 | "github.com/xtaci/smux" 16 | ) 17 | 18 | var ( 19 | // default udp timeout(read, write)(seconds) 20 | defaultUDPTimeout = 10 21 | 22 | // default udp session timeout(seconds) 23 | defaultUDPSessionTimeout = 30 24 | ) 25 | 26 | type udpSession struct { 27 | stream *smux.Stream 28 | lastActive time.Time 29 | } 30 | 31 | type UDPForward struct { 32 | listenAddr string 33 | sessionTimeout int 34 | readTimeout time.Duration 35 | writeTimeout time.Duration 36 | rawfd int 37 | 38 | // the session manager is the global session manager 39 | // it stores opennotr_client to opennotr_server connection 40 | sessMgr *SessionManager 41 | 42 | // udpSessions stores each client forward stream 43 | // the purpose of udpSession is to reuse stream 44 | udpSessions map[string]*udpSession 45 | udpsessLock sync.Mutex 46 | } 47 | 48 | func NewUDPForward(cfg UDPForwardConfig) *UDPForward { 49 | readTimeout := cfg.ReadTimeout 50 | if readTimeout <= 0 { 51 | readTimeout = defaultUDPTimeout 52 | } 53 | 54 | writeTimeout := cfg.WriteTimeout 55 | if writeTimeout <= 0 { 56 | writeTimeout = defaultUDPTimeout 57 | } 58 | 59 | sessionTimeout := cfg.SessionTimeout 60 | if sessionTimeout <= 0 { 61 | sessionTimeout = defaultUDPSessionTimeout 62 | } 63 | 64 | return &UDPForward{ 65 | listenAddr: cfg.ListenAddr, 66 | readTimeout: time.Duration(readTimeout) * time.Second, 67 | writeTimeout: time.Duration(writeTimeout) * time.Second, 68 | sessionTimeout: sessionTimeout, 69 | sessMgr: GetSessionManager(), 70 | udpSessions: make(map[string]*udpSession), 71 | } 72 | } 73 | 74 | // Listen listens a udp port, since that we use tproxy to 75 | // redirect traffic to this listened udp port 76 | // so the socket should set to ip transparent option 77 | func (f *UDPForward) Listen() (*net.UDPConn, error) { 78 | laddr, err := net.ResolveUDPAddr("udp", f.listenAddr) 79 | if err != nil { 80 | logs.Error("resolve udp fail: %v", err) 81 | return nil, err 82 | } 83 | 84 | lconn, err := net.ListenUDP("udp", laddr) 85 | if err != nil { 86 | return nil, err 87 | } 88 | 89 | // set socket with ip transparent option 90 | file, err := lconn.File() 91 | if err != nil { 92 | lconn.Close() 93 | return nil, err 94 | } 95 | defer file.Close() 96 | 97 | err = syscall.SetsockoptInt(int(file.Fd()), syscall.SOL_IP, syscall.IP_TRANSPARENT, 1) 98 | if err != nil { 99 | lconn.Close() 100 | return nil, err 101 | } 102 | 103 | // set socket with recv origin dst option 104 | err = syscall.SetsockoptInt(int(file.Fd()), syscall.SOL_IP, syscall.IP_RECVORIGDSTADDR, 1) 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | // create raw socket fd 110 | // we use rawsocket to send udp packet back to client. 111 | rawfd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW) 112 | if err != nil || rawfd < 0 { 113 | logs.Error("call socket fail: %v", err) 114 | return nil, err 115 | } 116 | 117 | err = syscall.SetsockoptInt(rawfd, syscall.IPPROTO_IP, syscall.IP_HDRINCL, 1) 118 | if err != nil { 119 | return nil, err 120 | } 121 | 122 | f.rawfd = rawfd 123 | return lconn, nil 124 | } 125 | 126 | func (f *UDPForward) Serve(lconn *net.UDPConn) error { 127 | go f.recyeleSession() 128 | buf := make([]byte, 64*1024) 129 | oob := make([]byte, 1024) 130 | for { 131 | // udp is not connect oriented, it should use read message 132 | // and read the origin dst ip and port from msghdr 133 | nr, oobn, _, raddr, err := lconn.ReadMsgUDP(buf, oob) 134 | if err != nil { 135 | logs.Error("read from udp fail: %v", err) 136 | break 137 | } 138 | 139 | origindst, err := f.getOriginDst(oob[:oobn]) 140 | if err != nil { 141 | logs.Error("get origin dst fail: %v", err) 142 | continue 143 | } 144 | 145 | dip, dport, _ := net.SplitHostPort(origindst.String()) 146 | sip, sport, _ := net.SplitHostPort(raddr.String()) 147 | 148 | key := fmt.Sprintf("%s:%s:%s:%s", sip, sport, dip, dport) 149 | 150 | f.udpsessLock.Lock() 151 | udpsess := f.udpSessions[key] 152 | if udpsess != nil { 153 | udpsess.lastActive = time.Now() 154 | f.udpsessLock.Unlock() 155 | } else { 156 | f.udpsessLock.Unlock() 157 | sess := f.sessMgr.GetSession(dip) 158 | if sess == nil { 159 | logs.Error("no route to host: %s", dip) 160 | continue 161 | } 162 | 163 | stream, err := sess.conn.OpenStream() 164 | if err != nil { 165 | logs.Error("open stream fail: %v", err) 166 | continue 167 | } 168 | 169 | udpsess = &udpSession{stream, time.Now()} 170 | f.udpsessLock.Lock() 171 | f.udpSessions[key] = udpsess 172 | f.udpsessLock.Unlock() 173 | 174 | targetIP := "127.0.0.1" 175 | bytes := encodeProxyProtocol("udp", sip, sport, targetIP, dport) 176 | stream.SetWriteDeadline(time.Now().Add(f.writeTimeout)) 177 | _, err = stream.Write(bytes) 178 | stream.SetWriteDeadline(time.Time{}) 179 | if err != nil { 180 | logs.Error("stream write fail: %v", err) 181 | continue 182 | } 183 | 184 | go f.forwardUDP(stream, key, origindst, raddr) 185 | } 186 | 187 | stream := udpsess.stream 188 | 189 | bytes := encode(buf[:nr]) 190 | stream.SetWriteDeadline(time.Now().Add(f.writeTimeout)) 191 | _, err = stream.Write(bytes) 192 | stream.SetWriteDeadline(time.Time{}) 193 | if err != nil { 194 | logs.Error("stream write fail: %v", err) 195 | } 196 | } 197 | return nil 198 | } 199 | 200 | // forwardUDP reads from stream and write to tofd via rawsocket 201 | func (f *UDPForward) forwardUDP(stream *smux.Stream, sessionKey string, fromaddr, toaddr *net.UDPAddr) { 202 | defer stream.Close() 203 | defer func() { 204 | f.udpsessLock.Lock() 205 | delete(f.udpSessions, sessionKey) 206 | f.udpsessLock.Unlock() 207 | }() 208 | 209 | hdr := make([]byte, 2) 210 | for { 211 | nr, err := stream.Read(hdr) 212 | if err != nil { 213 | if err != io.EOF { 214 | logs.Error("read stream fail %v", err) 215 | } 216 | break 217 | } 218 | if nr != 2 { 219 | logs.Error("invalid bodylen: %d", nr) 220 | continue 221 | } 222 | 223 | nlen := binary.BigEndian.Uint16(hdr) 224 | buf := make([]byte, nlen) 225 | stream.SetReadDeadline(time.Now().Add(f.readTimeout)) 226 | _, err = io.ReadFull(stream, buf) 227 | stream.SetReadDeadline(time.Time{}) 228 | if err != nil { 229 | logs.Error("read stream body fail: %v", err) 230 | break 231 | } 232 | 233 | err = sendUDPViaRaw(f.rawfd, fromaddr, toaddr, buf) 234 | if err != nil { 235 | logs.Error("send via raw socket fail: %v", err) 236 | } 237 | 238 | f.udpsessLock.Lock() 239 | udpsess := f.udpSessions[sessionKey] 240 | if udpsess != nil { 241 | udpsess.lastActive = time.Now() 242 | } 243 | f.udpsessLock.Unlock() 244 | } 245 | } 246 | 247 | func (f *UDPForward) recyeleSession() { 248 | tick := time.NewTicker(time.Second * 5) 249 | for range tick.C { 250 | f.udpsessLock.Lock() 251 | for k, s := range f.udpSessions { 252 | if time.Now().Sub(s.lastActive).Seconds() > float64(f.sessionTimeout) { 253 | logs.Warn("remove udp %v session, lastActive: %v", k, s.lastActive) 254 | delete(f.udpSessions, k) 255 | } 256 | } 257 | f.udpsessLock.Unlock() 258 | } 259 | } 260 | 261 | func (f *UDPForward) getOriginDst(hdr []byte) (*net.UDPAddr, error) { 262 | msgs, err := syscall.ParseSocketControlMessage(hdr) 263 | if err != nil { 264 | return nil, err 265 | } 266 | 267 | var origindst *net.UDPAddr 268 | for _, msg := range msgs { 269 | if msg.Header.Level == syscall.SOL_IP && 270 | msg.Header.Type == syscall.IP_RECVORIGDSTADDR { 271 | originDstRaw := &syscall.RawSockaddrInet4{} 272 | err := binary.Read(bytes.NewReader(msg.Data), binary.LittleEndian, originDstRaw) 273 | if err != nil { 274 | logs.Error("read origin dst fail: %v", err) 275 | continue 276 | } 277 | 278 | // only support for ipv4 279 | if originDstRaw.Family == syscall.AF_INET { 280 | pp := (*syscall.RawSockaddrInet4)(unsafe.Pointer(originDstRaw)) 281 | p := (*[2]byte)(unsafe.Pointer(&pp.Port)) 282 | origindst = &net.UDPAddr{ 283 | IP: net.IPv4(pp.Addr[0], pp.Addr[1], pp.Addr[2], pp.Addr[3]), 284 | Port: int(p[0])<<8 + int(p[1]), 285 | } 286 | } 287 | } 288 | } 289 | 290 | if origindst == nil { 291 | return nil, fmt.Errorf("get origin dst fail") 292 | } 293 | 294 | return origindst, nil 295 | } 296 | -------------------------------------------------------------------------------- /opennotrd/plugin/dummy/dummy.go: -------------------------------------------------------------------------------- 1 | package dummy 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/ICKelin/opennotr/internal/logs" 7 | "github.com/ICKelin/opennotr/opennotrd/plugin" 8 | ) 9 | 10 | func init() { 11 | plugin.Register("dummy", &DummyPlugin{}) 12 | } 13 | 14 | type DummyPlugin struct{} 15 | 16 | func (d *DummyPlugin) Setup(cfg json.RawMessage) error { 17 | return nil 18 | } 19 | 20 | func (d *DummyPlugin) RunProxy(meta *plugin.PluginMeta) (*plugin.ProxyTuple, error) { 21 | logs.Info("dummy plugin client config: %v", meta.Ctx) 22 | return &plugin.ProxyTuple{}, nil 23 | } 24 | 25 | func (d *DummyPlugin) StopProxy(meta *plugin.PluginMeta) {} 26 | -------------------------------------------------------------------------------- /opennotrd/plugin/plugin.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/ICKelin/opennotr/internal/logs" 9 | ) 10 | 11 | var pluginMgr = &PluginManager{ 12 | routes: make(map[string]*PluginMeta), 13 | plugins: make(map[string]IPlugin), 14 | } 15 | 16 | // ProxyTuple defineds plugins real proxy address 17 | type ProxyTuple struct { 18 | Protocol string 19 | FromPort string 20 | ToPort string 21 | } 22 | 23 | // PluginMeta defineds data that the plugins needs 24 | // these members are filled by server.go 25 | type PluginMeta struct { 26 | // plugin register protocol 27 | // eg: tcp, udp, http, http2, h2c 28 | Protocol string 29 | 30 | // From specific local listener address of plugin 31 | // browser or other clients will connect to this address 32 | // it's no use for restyproxy plugin. 33 | From string 34 | 35 | // To specific VIP:port of our VPN peer node. 36 | // For example: 37 | // our VPN virtual lan cidr is 100.64.100.1/24 38 | // the connected VPN client's VPN lan ip is 100.64.100.10/24 39 | // and it wants to export 8080 as http port, so the $To is 40 | // 100.64.100.10:8080 41 | To string 42 | 43 | // Domain specific the domain of our VPN peer node. 44 | // It could be empty 45 | Domain string 46 | 47 | // Data you want to passto plugin 48 | // Reserve 49 | Ctx interface{} 50 | RecycleSignal chan struct{} 51 | } 52 | 53 | func (item *PluginMeta) identify() string { 54 | return fmt.Sprintf("%s:%s:%s", item.Protocol, item.From, item.Domain) 55 | } 56 | 57 | // IPlugin defines plugin interface 58 | // Plugin should implements the IPlugin 59 | type IPlugin interface { 60 | // Setup calls at the begin of plugin system initialize 61 | // plugin system will pass the raw message to plugin's Setup function 62 | Setup(json.RawMessage) error 63 | 64 | // Close a proxy, it may be called by client's connection close 65 | StopProxy(item *PluginMeta) 66 | 67 | // Run a proxy, it may be called by client's connection established 68 | RunProxy(item *PluginMeta) (*ProxyTuple, error) 69 | } 70 | 71 | type PluginManager struct { 72 | mu sync.Mutex 73 | 74 | // routes stores proxier of localAddress 75 | // key: pluginMeta.identify() 76 | // value: pluginMeta 77 | routes map[string]*PluginMeta 78 | 79 | // plugins store plugin information 80 | // by call plugin.Register function. 81 | // key: protocol, eg: tcp, udp 82 | // value: plugin implement 83 | plugins map[string]IPlugin 84 | } 85 | 86 | func DefaultPluginManager() *PluginManager { 87 | return pluginMgr 88 | } 89 | 90 | func Register(protocol string, p IPlugin) { 91 | pluginMgr.plugins[protocol] = p 92 | } 93 | 94 | func Setup(plugins map[string]string) error { 95 | for protocol, cfg := range plugins { 96 | logs.Info("setup for %s with configuration:\n%s", protocol, cfg) 97 | plug, ok := pluginMgr.plugins[protocol] 98 | if !ok { 99 | logs.Error("protocol %s not register", protocol) 100 | return fmt.Errorf("protocol %s not register", protocol) 101 | } 102 | 103 | err := plug.Setup([]byte(cfg)) 104 | if err != nil { 105 | logs.Error("setup protocol %s fail: %v", protocol, err) 106 | return err 107 | } 108 | } 109 | 110 | return nil 111 | } 112 | 113 | func (p *PluginManager) AddProxy(item *PluginMeta) (*ProxyTuple, error) { 114 | p.mu.Lock() 115 | defer p.mu.Unlock() 116 | key := item.identify() 117 | if _, ok := p.routes[key]; ok { 118 | return nil, fmt.Errorf("port %s is in used", key) 119 | } 120 | 121 | plug, ok := p.plugins[item.Protocol] 122 | if !ok { 123 | return nil, fmt.Errorf("proxy %s not register", item.Protocol) 124 | } 125 | 126 | tuple, err := plug.RunProxy(item) 127 | if err != nil { 128 | logs.Error("run proxy fail: %v", err) 129 | return nil, err 130 | } 131 | p.routes[key] = item 132 | return tuple, nil 133 | } 134 | 135 | func (p *PluginManager) DelProxy(item *PluginMeta) { 136 | p.mu.Lock() 137 | defer p.mu.Unlock() 138 | key := item.identify() 139 | 140 | plug, ok := p.plugins[item.Protocol] 141 | if ok { 142 | plug.StopProxy(item) 143 | } 144 | 145 | delete(p.routes, key) 146 | } 147 | -------------------------------------------------------------------------------- /opennotrd/plugin/restyproxy/restyproxy.go: -------------------------------------------------------------------------------- 1 | package restyproxy 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net" 9 | "net/http" 10 | "time" 11 | 12 | "github.com/ICKelin/opennotr/internal/logs" 13 | "github.com/ICKelin/opennotr/opennotrd/plugin" 14 | ) 15 | 16 | var restyAdminUrl string 17 | 18 | func init() { 19 | plugin.Register("http", &RestyProxy{}) 20 | plugin.Register("https", &RestyProxy{}) 21 | plugin.Register("h2c", &RestyProxy{}) 22 | } 23 | 24 | type AddUpstreamBody struct { 25 | Scheme string `json:"scheme"` 26 | Host string `json:"host"` 27 | IP string `json:"ip"` 28 | Port string `json:"port"` 29 | } 30 | 31 | type RestyConfig struct { 32 | RestyAdminUrl string `json:"adminUrl"` 33 | } 34 | 35 | type RestyProxy struct { 36 | cfg RestyConfig 37 | } 38 | 39 | func (p *RestyProxy) Setup(config json.RawMessage) error { 40 | var cfg = RestyConfig{} 41 | err := json.Unmarshal([]byte(config), &cfg) 42 | if err != nil { 43 | return err 44 | } 45 | p.cfg = cfg 46 | return nil 47 | } 48 | 49 | func (p *RestyProxy) StopProxy(item *plugin.PluginMeta) { 50 | p.sendDeleteReq(item.Domain, item.Protocol) 51 | } 52 | 53 | func (p *RestyProxy) RunProxy(item *plugin.PluginMeta) (*plugin.ProxyTuple, error) { 54 | vip, port, err := net.SplitHostPort(item.To) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | req := &AddUpstreamBody{ 60 | Scheme: item.Protocol, 61 | Host: item.Domain, 62 | IP: vip, 63 | Port: port, 64 | } 65 | 66 | go p.sendPostReq(req) 67 | 68 | _, toPort, _ := net.SplitHostPort(item.To) 69 | return &plugin.ProxyTuple{ 70 | Protocol: item.Protocol, 71 | ToPort: toPort, 72 | }, nil 73 | } 74 | 75 | func (p *RestyProxy) sendPostReq(body interface{}) { 76 | cli := http.Client{ 77 | Timeout: time.Second * 10, 78 | } 79 | 80 | buf, _ := json.Marshal(body) 81 | br := bytes.NewBuffer(buf) 82 | 83 | req, err := http.NewRequest("POST", p.cfg.RestyAdminUrl, br) 84 | if err != nil { 85 | logs.Error("request %v fail: %v", body, err) 86 | return 87 | } 88 | 89 | resp, err := cli.Do(req) 90 | if err != nil { 91 | logs.Error("request %v fail: %v", body, err) 92 | return 93 | } 94 | defer resp.Body.Close() 95 | 96 | cnt, err := ioutil.ReadAll(resp.Body) 97 | if err != nil { 98 | logs.Error("request %v fail: %v", body, err) 99 | return 100 | } 101 | logs.Info("set upstream %v reply: %s", body, string(cnt)) 102 | } 103 | 104 | func (p *RestyProxy) sendDeleteReq(host, scheme string) { 105 | cli := http.Client{ 106 | Timeout: time.Second * 10, 107 | } 108 | url := fmt.Sprintf("%s?host=%s&scheme=%s", p.cfg.RestyAdminUrl, host, scheme) 109 | req, err := http.NewRequest("DELETE", url, nil) 110 | if err != nil { 111 | logs.Error("delete host %s fail: %v", host, err) 112 | return 113 | } 114 | 115 | resp, err := cli.Do(req) 116 | if err != nil { 117 | logs.Error("delete host %s fail: %v", host, err) 118 | return 119 | } 120 | defer resp.Body.Close() 121 | 122 | cnt, err := ioutil.ReadAll(resp.Body) 123 | if err != nil { 124 | logs.Error("delete host %s fail: %v", host, err) 125 | return 126 | } 127 | 128 | logs.Info("delete upstream reply: %s", string(cnt)) 129 | } 130 | -------------------------------------------------------------------------------- /opennotrd/plugin/tcpproxy/tcpproxy.go: -------------------------------------------------------------------------------- 1 | package tcpproxy 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "net" 7 | "sync" 8 | "time" 9 | 10 | "github.com/ICKelin/opennotr/internal/logs" 11 | "github.com/ICKelin/opennotr/opennotrd/plugin" 12 | ) 13 | 14 | func init() { 15 | plugin.Register("tcp", &TCPProxy{}) 16 | } 17 | 18 | type TCPProxy struct{} 19 | 20 | func (t *TCPProxy) Setup(config json.RawMessage) error { return nil } 21 | 22 | func (t *TCPProxy) StopProxy(item *plugin.PluginMeta) { 23 | select { 24 | case item.RecycleSignal <- struct{}{}: 25 | default: 26 | } 27 | } 28 | 29 | // RunProxy runs a tcp server and proxy to item.To 30 | // RunProxy may change item.From address to the real listenner address 31 | func (t *TCPProxy) RunProxy(item *plugin.PluginMeta) (*plugin.ProxyTuple, error) { 32 | from, to := item.From, item.To 33 | lis, err := net.Listen("tcp", from) 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | fin := make(chan struct{}) 39 | go func() { 40 | select { 41 | case <-item.RecycleSignal: 42 | logs.Info("receive recycle signal for %s", from) 43 | lis.Close() 44 | case <-fin: 45 | return 46 | } 47 | }() 48 | 49 | go func() { 50 | defer lis.Close() 51 | defer close(fin) 52 | 53 | sess := &sync.Map{} 54 | defer func() { 55 | sess.Range(func(k, v interface{}) bool { 56 | if conn, ok := v.(net.Conn); ok { 57 | conn.Close() 58 | } 59 | return true 60 | }) 61 | }() 62 | 63 | for { 64 | conn, err := lis.Accept() 65 | if err != nil { 66 | logs.Error("accept fail: %v", err) 67 | break 68 | } 69 | 70 | go func() { 71 | sess.Store(conn.RemoteAddr().String(), conn) 72 | defer sess.Delete(conn.RemoteAddr().String()) 73 | t.doProxy(conn, to) 74 | }() 75 | } 76 | }() 77 | 78 | _, fromPort, _ := net.SplitHostPort(lis.Addr().String()) 79 | _, toPort, _ := net.SplitHostPort(item.To) 80 | 81 | return &plugin.ProxyTuple{ 82 | Protocol: item.Protocol, 83 | FromPort: fromPort, 84 | ToPort: toPort, 85 | }, nil 86 | } 87 | 88 | func (t *TCPProxy) doProxy(conn net.Conn, to string) { 89 | defer conn.Close() 90 | 91 | toconn, err := net.DialTimeout("tcp", to, time.Second*10) 92 | if err != nil { 93 | logs.Error("dial fail: %v", err) 94 | return 95 | } 96 | defer toconn.Close() 97 | 98 | wg := &sync.WaitGroup{} 99 | wg.Add(2) 100 | 101 | go func() { 102 | defer wg.Done() 103 | buf := make([]byte, 1500) 104 | io.CopyBuffer(toconn, conn, buf) 105 | }() 106 | 107 | go func() { 108 | defer wg.Done() 109 | buf := make([]byte, 1500) 110 | io.CopyBuffer(conn, toconn, buf) 111 | }() 112 | wg.Wait() 113 | } 114 | -------------------------------------------------------------------------------- /opennotrd/plugin/udpproxy/udpproxy.go: -------------------------------------------------------------------------------- 1 | package udpproxy 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "net" 7 | "sync" 8 | "time" 9 | 10 | "github.com/ICKelin/opennotr/internal/logs" 11 | "github.com/ICKelin/opennotr/opennotrd/plugin" 12 | ) 13 | 14 | // default timeout for udp session 15 | var defaultTimeout = 30 16 | 17 | func init() { 18 | plugin.Register("udp", &UDPProxy{}) 19 | } 20 | 21 | type config struct { 22 | // session timeout(second) 23 | SessionTimeout int `json:"sessionTimeout"` 24 | } 25 | 26 | type UDPProxy struct { 27 | cfg config 28 | } 29 | 30 | func (p *UDPProxy) Setup(rawMessage json.RawMessage) error { 31 | var cfg config 32 | err := json.Unmarshal(rawMessage, &cfg) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | if cfg.SessionTimeout <= 0 { 38 | cfg.SessionTimeout = defaultTimeout 39 | } 40 | p.cfg = cfg 41 | return nil 42 | } 43 | 44 | func (p *UDPProxy) StopProxy(item *plugin.PluginMeta) { 45 | select { 46 | case item.RecycleSignal <- struct{}{}: 47 | default: 48 | } 49 | } 50 | 51 | func (p *UDPProxy) RunProxy(item *plugin.PluginMeta) (*plugin.ProxyTuple, error) { 52 | from := item.From 53 | laddr, err := net.ResolveUDPAddr("udp", from) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | lis, err := net.ListenUDP("udp", laddr) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | go p.doProxy(lis, item) 64 | 65 | _, fromPort, _ := net.SplitHostPort(lis.LocalAddr().String()) 66 | _, toPort, _ := net.SplitHostPort(item.To) 67 | 68 | return &plugin.ProxyTuple{ 69 | Protocol: item.Protocol, 70 | FromPort: fromPort, 71 | ToPort: toPort, 72 | }, nil 73 | } 74 | 75 | func (p *UDPProxy) doProxy(lis *net.UDPConn, item *plugin.PluginMeta) { 76 | defer lis.Close() 77 | 78 | from := item.From 79 | ctx, cancel := context.WithCancel(context.Background()) 80 | defer cancel() 81 | 82 | // receive this signal and close the listener 83 | // the listener close will force lis.ReadFromUDP loop break 84 | // then close all the client socket and end udpCopy 85 | go func() { 86 | select { 87 | case <-item.RecycleSignal: 88 | logs.Info("receive recycle signal for %s", from) 89 | lis.Close() 90 | case <-ctx.Done(): 91 | return 92 | } 93 | }() 94 | 95 | // sess store all backend connection 96 | // key: client address 97 | // value: *net.UDPConn 98 | sess := sync.Map{} 99 | 100 | // sessionTimeout store all session key active time 101 | // the purpose of this is to avoid too session without expired 102 | sessionTimeout := sync.Map{} 103 | 104 | // close all backend sockets 105 | // this action may end udpCopy 106 | defer func() { 107 | sess.Range(func(k, v interface{}) bool { 108 | if conn, ok := v.(*net.UDPConn); ok { 109 | conn.Close() 110 | } 111 | return true 112 | }) 113 | }() 114 | 115 | go func() { 116 | timeout := p.cfg.SessionTimeout 117 | interval := timeout / 2 118 | if interval <= 0 { 119 | interval = timeout 120 | } 121 | tick := time.NewTicker(time.Second * time.Duration(interval)) 122 | for range tick.C { 123 | sessionTimeout.Range(func(k, v interface{}) bool { 124 | lastActiveAt, ok := v.(time.Time) 125 | if !ok { 126 | return true 127 | } 128 | 129 | if time.Now().Sub(lastActiveAt).Seconds() > float64(timeout) { 130 | sess.Delete(k) 131 | } 132 | return true 133 | }) 134 | } 135 | }() 136 | 137 | var buf = make([]byte, 64*1024) 138 | for { 139 | nr, raddr, err := lis.ReadFromUDP(buf) 140 | if err != nil { 141 | logs.Error("read from udp fail: %v", err) 142 | break 143 | } 144 | 145 | key := raddr.String() 146 | val, ok := sess.Load(key) 147 | if !ok { 148 | backendAddr, err := net.ResolveUDPAddr("udp", item.To) 149 | if err != nil { 150 | logs.Error("resolve udp fail: %v", err) 151 | break 152 | } 153 | 154 | backendConn, err := net.DialUDP("udp", nil, backendAddr) 155 | if err != nil { 156 | logs.Error("dial udp fail: %v", err) 157 | break 158 | } 159 | sess.Store(key, backendConn) 160 | sessionTimeout.Store(key, time.Now()) 161 | 162 | // read from $to address and write to $from address 163 | go p.udpCopy(lis, backendConn, raddr) 164 | } 165 | 166 | val, ok = sess.Load(key) 167 | if !ok { 168 | continue 169 | } 170 | 171 | sessionTimeout.Store(key, time.Now()) 172 | // read from $from address and write to $to address 173 | val.(*net.UDPConn).Write(buf[:nr]) 174 | } 175 | } 176 | 177 | func (p *UDPProxy) udpCopy(dst, src *net.UDPConn, toaddr *net.UDPAddr) { 178 | defer src.Close() 179 | buf := make([]byte, 64*1024) 180 | for { 181 | nr, _, err := src.ReadFromUDP(buf) 182 | if err != nil { 183 | logs.Error("read from udp fail: %v", err) 184 | break 185 | } 186 | 187 | _, err = dst.WriteToUDP(buf[:nr], toaddr) 188 | if err != nil { 189 | logs.Error("write to udp fail: %v", err) 190 | break 191 | } 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /opennotrd/plugins.go: -------------------------------------------------------------------------------- 1 | package opennotrd 2 | 3 | import ( 4 | // plugin import 5 | _ "github.com/ICKelin/opennotr/opennotrd/plugin/dummy" 6 | _ "github.com/ICKelin/opennotr/opennotrd/plugin/restyproxy" 7 | _ "github.com/ICKelin/opennotr/opennotrd/plugin/tcpproxy" 8 | _ "github.com/ICKelin/opennotr/opennotrd/plugin/udpproxy" 9 | ) 10 | -------------------------------------------------------------------------------- /opennotrd/run.go: -------------------------------------------------------------------------------- 1 | package opennotrd 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | 7 | "github.com/ICKelin/opennotr/internal/logs" 8 | "github.com/ICKelin/opennotr/opennotrd/core" 9 | "github.com/ICKelin/opennotr/opennotrd/plugin" 10 | ) 11 | 12 | func Run() { 13 | confpath := flag.String("conf", "", "config file path") 14 | flag.Parse() 15 | 16 | cfg, err := core.ParseConfig(*confpath) 17 | if err != nil { 18 | fmt.Println(err) 19 | return 20 | } 21 | 22 | logs.Init("opennotrd.log", "info", 10) 23 | logs.Info("config: %v", cfg) 24 | 25 | // create dhcp manager 26 | // dhcp Select/Release ip for opennotr client 27 | dhcp, err := core.NewDHCP(cfg.DHCPConfig.Cidr) 28 | if err != nil { 29 | logs.Error("new dhcp module fail: %v", err) 30 | return 31 | } 32 | 33 | // setup all plugin base on plugin json configuration 34 | err = plugin.Setup(cfg.Plugins) 35 | if err != nil { 36 | logs.Error("setup plugin fail: %v", err) 37 | return 38 | } 39 | 40 | // initial resolver 41 | // currently resolver use coredns and etcd 42 | // our resolver just write DOMAIN => VIP record to etcd 43 | var resolver *core.Resolver 44 | if len(cfg.ResolverConfig.EtcdEndpoints) > 0 { 45 | resolver, err = core.NewResolve(cfg.ResolverConfig.EtcdEndpoints) 46 | if err != nil { 47 | logs.Error("new resolve fail: %v", err) 48 | return 49 | } 50 | } 51 | 52 | // up local tcp,udp service 53 | // we use tproxy to route traffic to the tcp port and udp port here. 54 | tcpfw := core.NewTCPForward(cfg.TCPForwardConfig) 55 | listener, err := tcpfw.Listen() 56 | if err != nil { 57 | logs.Error("listen tproxy tcp fail: %v", err) 58 | return 59 | } 60 | 61 | go tcpfw.Serve(listener) 62 | 63 | udpfw := core.NewUDPForward(cfg.UDPForwardConfig) 64 | lconn, err := udpfw.Listen() 65 | if err != nil { 66 | logs.Error("listen tproxy udp fail: %v", err) 67 | return 68 | } 69 | go udpfw.Serve(lconn) 70 | 71 | // server provides tcp server for opennotr client 72 | s := core.NewServer(cfg.ServerConfig, dhcp, resolver) 73 | fmt.Println(s.ListenAndServe()) 74 | } 75 | -------------------------------------------------------------------------------- /opennotrd/test/httpecho/httpclient.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | func main() { 12 | remoteAddr := flag.String("r", "", "remote address") 13 | flag.Parse() 14 | 15 | url := fmt.Sprintf("http://%s/echo", *remoteAddr) 16 | for i := 0; i < 100; i++ { 17 | beg := time.Now() 18 | rsp, err := http.Get(url) 19 | if err != nil { 20 | break 21 | } 22 | cnt, err := ioutil.ReadAll(rsp.Body) 23 | if err != nil { 24 | fmt.Println(err) 25 | break 26 | } 27 | 28 | rsp.Body.Close() 29 | fmt.Printf("echo http packet %d %s rtt %dms\n", i+1, string(cnt), time.Now().Sub(beg).Milliseconds()) 30 | time.Sleep(time.Second * 1) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /opennotrd/test/httpecho/httpserver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "net/http" 4 | 5 | func main() { 6 | http.HandleFunc("/echo", func(w http.ResponseWriter, r *http.Request) { 7 | w.Write([]byte(r.RemoteAddr)) 8 | }) 9 | http.ListenAndServe(":8080", nil) 10 | } 11 | -------------------------------------------------------------------------------- /opennotrd/test/plugin_test.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | "time" 7 | 8 | "github.com/ICKelin/opennotr/opennotrd/plugin" 9 | _ "github.com/ICKelin/opennotr/opennotrd/plugin/tcpproxy" 10 | ) 11 | 12 | var listener net.Listener 13 | 14 | func init() { 15 | 16 | } 17 | 18 | func runTCPServer(bufsize int) net.Listener { 19 | lis, err := net.Listen("tcp", "127.0.0.1:2345") 20 | if err != nil { 21 | panic(err) 22 | } 23 | 24 | go func() { 25 | defer lis.Close() 26 | for { 27 | conn, err := lis.Accept() 28 | if err != nil { 29 | break 30 | } 31 | 32 | go onconn(conn, bufsize) 33 | } 34 | }() 35 | return lis 36 | } 37 | 38 | func onconn(conn net.Conn, bufsize int) { 39 | defer conn.Close() 40 | buf := make([]byte, bufsize) 41 | nr, _ := conn.Read(buf) 42 | conn.Write(buf[:nr]) 43 | } 44 | 45 | func runEcho(t *testing.T, bufsize, numconn int) { 46 | item := &plugin.PluginMeta{ 47 | Protocol: "tcp", 48 | From: "127.0.0.1:1234", 49 | To: "127.0.0.1:2345", 50 | RecycleSignal: make(chan struct{}), 51 | } 52 | err := plugin.DefaultPluginManager().AddProxy(item) 53 | if err != nil { 54 | t.Error(err) 55 | return 56 | } 57 | defer plugin.DefaultPluginManager().DelProxy(item) 58 | 59 | lis := runTCPServer(bufsize) 60 | 61 | // client 62 | for i := 0; i < numconn; i++ { 63 | go func() { 64 | buf := make([]byte, bufsize) 65 | conn, err := net.Dial("tcp", "127.0.0.1:1234") 66 | if err != nil { 67 | t.Error(err) 68 | return 69 | } 70 | defer conn.Close() 71 | for { 72 | conn.Write(buf) 73 | conn.Read(buf) 74 | } 75 | }() 76 | } 77 | tick := time.NewTicker(time.Second * 60) 78 | <-tick.C 79 | lis.Close() 80 | time.Sleep(time.Second) 81 | } 82 | 83 | func TestTCPEcho128B(t *testing.T) { 84 | numconn := 128 85 | bufsize := 1024 86 | runEcho(t, bufsize, numconn) 87 | } 88 | -------------------------------------------------------------------------------- /opennotrd/test/tcpecho/tcpclient.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "net" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | remoteAddr := flag.String("r", "", "remote address") 12 | flag.Parse() 13 | 14 | buf := make([]byte, 1024) 15 | for i := 0; i < 100; i++ { 16 | beg := time.Now() 17 | 18 | conn, err := net.Dial("tcp", *remoteAddr) 19 | if err != nil { 20 | fmt.Println(err) 21 | break 22 | } 23 | 24 | conn.Write(buf) 25 | conn.Read(buf) 26 | conn.Close() 27 | fmt.Printf("echo tcp packet %d rtt %dms\n", i+1, time.Now().Sub(beg).Milliseconds()) 28 | time.Sleep(time.Second * 1) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /opennotrd/test/tcpecho/tcpserver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "net" 7 | ) 8 | 9 | func main() { 10 | localAddress := flag.String("l", "", "local address") 11 | flag.Parse() 12 | 13 | listener, err := net.Listen("tcp", *localAddress) 14 | if err != nil { 15 | fmt.Println(err) 16 | return 17 | } 18 | defer listener.Close() 19 | 20 | for { 21 | conn, err := listener.Accept() 22 | if err != nil { 23 | break 24 | } 25 | 26 | go func() { 27 | defer conn.Close() 28 | buf := make([]byte, 1024) 29 | nr, err := conn.Read(buf) 30 | if err != nil { 31 | fmt.Println(err) 32 | return 33 | } 34 | 35 | conn.Write(buf[:nr]) 36 | }() 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /opennotrd/test/udpecho/client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "net" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | remoteAddr := flag.String("r", "", "remote address") 12 | flag.Parse() 13 | 14 | rconn, err := net.Dial("udp", *remoteAddr) 15 | if err != nil { 16 | fmt.Println(err) 17 | return 18 | } 19 | 20 | buf := make([]byte, 1024) 21 | for i := 0; i < 100; i++ { 22 | beg := time.Now() 23 | _, err := rconn.Write(buf) 24 | if err != nil { 25 | fmt.Println(err) 26 | break 27 | } 28 | 29 | rconn.Read(buf) 30 | fmt.Printf("echo udp packet %d rtt %dms\n", i+1, time.Now().Sub(beg).Milliseconds()) 31 | time.Sleep(time.Second * 1) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /opennotrd/test/udpecho/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "net" 7 | ) 8 | 9 | func main() { 10 | localAddress := flag.String("l", "", "local address") 11 | flag.Parse() 12 | 13 | laddr, err := net.ResolveUDPAddr("udp", *localAddress) 14 | if err != nil { 15 | fmt.Println(err) 16 | return 17 | } 18 | 19 | lconn, err := net.ListenUDP("udp", laddr) 20 | if err != nil { 21 | fmt.Println(err) 22 | return 23 | } 24 | defer lconn.Close() 25 | 26 | buf := make([]byte, 64*1024) 27 | for { 28 | nr, raddr, err := lconn.ReadFromUDP(buf) 29 | if err != nil { 30 | fmt.Println(err) 31 | break 32 | } 33 | lconn.WriteToUDP(buf[:nr], raddr) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /opennotrd/test/websocket/wsclient.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/gorilla/websocket" 9 | ) 10 | 11 | func main() { 12 | raddr := flag.String("r", "", "remote address") 13 | flag.Parse() 14 | buf := make([]byte, 1024) 15 | for i := 0; i < 100; i++ { 16 | beg := time.Now() 17 | conn, _, err := websocket.DefaultDialer.Dial(*raddr, nil) 18 | if err != nil { 19 | fmt.Println(err) 20 | return 21 | } 22 | 23 | defer conn.Close() 24 | conn.WriteMessage(websocket.BinaryMessage, buf) 25 | conn.ReadMessage() 26 | fmt.Printf("echo websocket packet %d rtt %dms\n", i+1, time.Now().Sub(beg).Milliseconds()) 27 | time.Sleep(time.Second * 1) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /opennotrd/test/websocket/wsserver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/siddontang/go/websocket" 9 | ) 10 | 11 | func main() { 12 | localAddress := flag.String("l", "", "local address") 13 | flag.Parse() 14 | 15 | http.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) { 16 | conn, err := websocket.Upgrade(w, r, r.Header) 17 | if err != nil { 18 | fmt.Println(err) 19 | return 20 | } 21 | defer conn.Close() 22 | 23 | msgType, msg, err := conn.Read() 24 | if err != nil { 25 | fmt.Println(err) 26 | return 27 | } 28 | conn.WriteMessage(msgType, msg) 29 | }) 30 | http.ListenAndServe(*localAddress, nil) 31 | } 32 | --------------------------------------------------------------------------------