11 |
12 |
13 |
35 |
36 |
37 |
38 |
39 |
43 |
44 |
56 |
57 |
58 |
--------------------------------------------------------------------------------
/goroutine/chart/chat_server/model/mgr.go:
--------------------------------------------------------------------------------
1 | package model
2 |
3 | import (
4 | "fmt"
5 | "time"
6 |
7 | "encoding/json"
8 |
9 | "github.com/garyburd/redigo/redis"
10 | )
11 |
12 | var (
13 | UserTable = "users"
14 | )
15 |
16 | type UserMgr struct {
17 | pool *redis.Pool
18 | }
19 |
20 | func NewUserMgr(pool *redis.Pool) (mgr *UserMgr) {
21 |
22 | mgr = &UserMgr{
23 | pool: pool,
24 | }
25 | return
26 | }
27 |
28 | func (p *UserMgr) getUser(conn redis.Conn, id int) (user *User, err error) {
29 |
30 | result, err := redis.String(conn.Do("HGet", UserTable, fmt.Sprintf("%d", id)))
31 | if err != nil {
32 | if err == redis.ErrNil {
33 | err = ErrUserNotExist
34 | }
35 | return
36 | }
37 |
38 | user = &User{}
39 | err = json.Unmarshal([]byte(result), user)
40 | if err != nil {
41 | return
42 | }
43 | return
44 | }
45 |
46 | func (p *UserMgr) Login(id int, passwd string) (user *User, err error) {
47 |
48 | conn := p.pool.Get()
49 | defer conn.Close()
50 |
51 | user, err = p.getUser(conn, id)
52 | if err != nil {
53 | return
54 | }
55 |
56 | if user.UserId != id || user.Passwd != passwd {
57 | err = ErrInvalidPasswd
58 | }
59 |
60 | user.Status = UserStatusOnline
61 | user.LastLogin = fmt.Sprintf("%v", time.Now())
62 |
63 | return
64 | }
65 |
66 | func (p *UserMgr) Register(user *User) (err error) {
67 |
68 | conn := p.pool.Get()
69 | defer conn.Close()
70 |
71 | if user == nil {
72 | fmt.Println("invalid user")
73 | err = ErrInvalidParams
74 | return
75 | }
76 |
77 | _, err = p.getUser(conn, user.UserId)
78 | if err == nil {
79 | err = ErrUserExist
80 | return
81 | }
82 |
83 | if err != ErrUserNotExist {
84 | return
85 | }
86 |
87 | data, err := json.Marshal(user)
88 | if err != nil {
89 | return
90 | }
91 |
92 | _, err = conn.Do("HSet", UserTable, fmt.Sprintf("%d", user.UserId), string(data))
93 | if err != nil {
94 | return
95 | }
96 | return
97 | }
98 |
--------------------------------------------------------------------------------
/logcollection/web_admin/views/log/apply.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | | 15 | | 日志ID | 16 |项目名称 | 17 |日志收集路径 | 18 |创建时间 | 19 |TOPIC | 20 |
|---|---|---|---|---|---|
| 26 | | {{.LogId}} | 27 |{{.AppName}} | 28 |{{.LogPath}} | 29 |{{.CreateTime}} | 30 |{{.Topic}} | 31 |
16 |
45 |
46 |
47 |
50 |
51 |
52 |
--------------------------------------------------------------------------------
/crawler-distributed/distributed.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "flag"
5 | "golang-awesome/crawler-distributed/config"
6 | "golang-awesome/crawler-distributed/persist/client"
7 | "golang-awesome/crawler-distributed/rpcsupport"
8 | worker "golang-awesome/crawler-distributed/worker/client"
9 | "golang-awesome/crawler/engine"
10 | "golang-awesome/crawler/scheduler"
11 | "golang-awesome/crawler/zhenai/parser"
12 | "log"
13 | "net/rpc"
14 | "strings"
15 | )
16 |
17 | func createClientPool(hosts []string) chan *rpc.Client{
18 | var clients []*rpc.Client
19 | for _, h := range hosts{
20 | client, err := rpcsupport.NewClient(h)
21 | if err != nil {
22 | log.Printf("error connection to %s : %s",h,err)
23 | }else {
24 | clients = append(clients, client)
25 | log.Printf("connected to %s",h)
26 | }
27 | }
28 | out := make(chan *rpc.Client)
29 | //持续分发客户端
30 | go func() {
31 | for{
32 | for _, c := range clients {
33 | out <- c
34 | }
35 | }
36 | }()
37 | return out
38 | }
39 |
40 | var hosts = flag.String("hosts", ":9002","多个工作节点的端口，以都好分隔，如:9002,:9003")
41 |
42 | func main(){
43 | flag.Parse()
44 | itemChan, err := client.ItemSaver(config.ItemSaverPort)
45 | if err != nil{
46 | panic(err)
47 | }
48 | log.Printf("connected item saver at: %v",config.ItemSaverPort)
49 |
50 | pool := createClientPool(strings.Split(*hosts,","))
51 |
52 | processor:= worker.CreateProcessor(pool)
53 |
54 | var seed []engine.Request
55 | seed = []engine.Request{
56 | {
57 | Url:"http://www.zhenai.com/zhenghun",
58 | Parse: engine.NewFuncParser(parser.ParseCityList,"ParseCityList"),
59 | },
60 | //{
61 | // Url: "http://www.zhenai.com/zhenghun/henan",
62 | // Parse: engine.NewFuncParser(parser.ParseCity, "ParseCity"),
63 | //},
64 | //{
65 | // Url: "http://www.zhenai.com/zhenghun/beijing",
66 | // Parse: engine.NewFuncParser(parser.ParseCity, "ParseCity"),
67 | //},
68 | }
69 |
70 | e := engine.ConcurrentEngine{
71 | MaxWorkerCount:100,
72 | Scheduler:&scheduler.QueuedScheduler{},
73 | ItemChan:itemChan,
74 | RequestWorker:processor,
75 | }
76 | e.Run(seed...)
77 | }
78 |
--------------------------------------------------------------------------------
/goroutine/chart/chat_client/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/binary"
5 | "encoding/json"
6 | "errors"
7 | "net"
8 | "fmt"
9 | "golang-awesome/goroutine/chart/proto"
10 | )
11 |
12 | func readPackage(conn net.Conn) (msg proto.Message, err error) {
13 | var buf [8192]byte
14 | n, err := conn.Read(buf[0:4])
15 | if n != 4 {
16 | err = errors.New("read header failed")
17 | return
18 | }
19 | fmt.Println("read package:", buf[0:4])
20 |
21 | var packLen uint32
22 | packLen = binary.BigEndian.Uint32(buf[0:4])
23 |
24 | fmt.Printf("receive len:%d", packLen)
25 | n, err = conn.Read(buf[0:packLen])
26 | if n != int(packLen) {
27 | err = errors.New("read body failed")
28 | return
29 | }
30 |
31 | fmt.Printf("receive data:%s\n", string(buf[0:packLen]))
32 | err = json.Unmarshal(buf[0:packLen], &msg)
33 | if err != nil {
34 | fmt.Println("unmarshal failed, err:", err)
35 | }
36 | return
37 | }
38 |
39 | func login(conn net.Conn) (err error) {
40 | var msg proto.Message
41 | msg.Cmd = proto.UserLogin
42 |
43 | var loginCmd proto.LoginCmd
44 | loginCmd.Id = 1
45 | loginCmd.Passwd = "123456789"
46 |
47 | data, err := json.Marshal(loginCmd)
48 | if err != nil {
49 | return
50 | }
51 |
52 | msg.Data = string(data)
53 | data, err = json.Marshal(msg)
54 | if err != nil {
55 | return
56 | }
57 |
58 | var buf [4]byte
59 | packLen := uint32(len(data))
60 |
61 | fmt.Println("packlen:", packLen)
62 | binary.BigEndian.PutUint32(buf[0:4], packLen)
63 |
64 | n, err := conn.Write(buf[:])
65 | if err != nil || n != 4 {
66 | fmt.Println("write data failed")
67 | return
68 | }
69 |
70 | _, err = conn.Write([]byte(data))
71 | if err != nil {
72 | return
73 | }
74 |
75 | msg, err = readPackage(conn)
76 | if err != nil {
77 | fmt.Println("read package failed, err:", err)
78 | }
79 |
80 | fmt.Println(msg)
81 | return
82 | }
83 |
84 | func main() {
85 | conn, err := net.Dial("tcp", "localhost:10000")
86 | if err != nil {
87 | fmt.Println("Error dialing", err.Error())
88 | return
89 | }
90 |
91 | err = login(conn)
92 | if err != nil {
93 | fmt.Println("login failed, err:", err)
94 | return
95 | }
96 |
97 | }
98 |
--------------------------------------------------------------------------------
/seckill/SecProxy/service/antispam.go:
--------------------------------------------------------------------------------
1 | package service
2 |
3 | import (
4 | "fmt"
5 | "github.com/astaxie/beego/logs"
6 | "sync"
7 | )
8 |
9 | type SecLimitMgr struct {
10 | UserLimitMap map[int]*Limit
11 | IpLimitMap map[string]*Limit
12 | lock sync.Mutex
13 | }
14 |
15 | func antiSpam(req *SecRequest)( err error){
16 | _,ok := secKillConf.idBlackMap[req.UserId]
17 | if ok {
18 | err = fmt.Errorf("invalid request")
19 | logs.Error("userId[%d] is block by id black",req.UserId)
20 | return
21 | }
22 |
23 | _,ok = secKillConf.ipBlackMap[req.ClientAddr]
24 | if ok {
25 | err = fmt.Errorf("invalid request")
26 | logs.Error("userId[%d] ip[%s] id block by ip black",req.UserId,req.ClientAddr)
27 | return
28 | }
29 |
30 | secKillConf.secLimitMgr.lock.Lock()
31 | //uid 频率控制，防刷
32 | limit,ok := secKillConf.secLimitMgr.UserLimitMap[req.UserId]
33 | if !ok {
34 | limit = &Limit{
35 | secLimit: &SecLimit{},
36 | minLimit: &MinLimit{},
37 | }
38 |
39 | secKillConf.secLimitMgr.UserLimitMap[req.UserId] = limit
40 | }
41 | secIdCount := limit.secLimit.Count(req.AccessTime.Unix())
42 | minIdCount := limit.minLimit.Count(req.AccessTime.Unix())
43 |
44 | //ip 频率控制
45 | limit, ok = secKillConf.secLimitMgr.IpLimitMap[req.ClientAddr]
46 | if !ok {
47 | limit = &Limit{
48 | secLimit:&SecLimit{},
49 | minLimit:&MinLimit{},
50 | }
51 | secKillConf.secLimitMgr.IpLimitMap[req.ClientAddr] = limit
52 | }
53 |
54 | secIpCount := limit.secLimit.Count(req.AccessTime.Unix())
55 | minIpCount := limit.minLimit.Count(req.AccessTime.Unix())
56 |
57 | secKillConf.secLimitMgr.lock.Unlock()
58 |
59 | if secIpCount > secKillConf.AccessLimitConf.IPSecAccessLimit {
60 | err = fmt.Errorf("invalid request")
61 | return
62 | }
63 |
64 | if minIpCount > secKillConf.AccessLimitConf.IPMinAccessLimit {
65 | err = fmt.Errorf("invalid request")
66 | return
67 | }
68 |
69 | if secIdCount > secKillConf.AccessLimitConf.UserSecAccessLimit {
70 | err = fmt.Errorf("invalid request")
71 | return
72 | }
73 |
74 | if minIdCount > secKillConf.AccessLimitConf.UserMinAccessLimit {
75 | err = fmt.Errorf("invalid request")
76 | return
77 | }
78 |
79 | return
80 | }
81 |
--------------------------------------------------------------------------------
/logcollection/web_admin/controllers/AppController/app.go:
--------------------------------------------------------------------------------
1 | package AppController
2 |
3 | import (
4 | "fmt"
5 | "github.com/astaxie/beego"
6 | "github.com/astaxie/beego/logs"
7 | "golang-awesome/logcollection/web_admin/models"
8 | "strings"
9 | )
10 |
11 | type AppController struct {
12 | beego.Controller
13 | }
14 |
15 | func (a *AppController) AppList(){
16 | logs.Debug("enter index controller")
17 |
18 | a.Layout = "layout/layout.html"
19 | appList, err := models.GetAllAppInfo()
20 | if err != nil {
21 | a.Data["Error"] = fmt.Sprintf("服务器繁忙")
22 | a.TplName = "app/error.html"
23 |
24 | logs.Warn("get app list failed, err:%v", err)
25 | return
26 | }
27 |
28 | logs.Debug("get app list succ, data:%v", appList)
29 | a.Data["applist"] = appList
30 |
31 | a.TplName = "app/index.html"
32 | }
33 |
34 | func (a *AppController) AppApply(){
35 | logs.Debug("enter apply controller")
36 | a.Layout = "layout/layout.html"
37 | a.TplName = "app/apply.html"
38 | }
39 |
40 | func (a *AppController) AppCreate(){
41 | logs.Debug("enter appCreate controller")
42 | appName := a.GetString("appName")
43 | appType := a.GetString("appType")
44 | developPath := a.GetString("devPath")
45 | ipstr := a.GetString("iplist")
46 |
47 | a.Layout = "layout/layout.html"
48 |
49 | if len(appName) == 0 || len(appType) == 0 || len(developPath) == 0 || len(ipstr) == 0 {
50 | a.Data["Error"] = fmt.Sprintf("非法参数")
51 | a.TplName = "app/error.html"
52 |
53 | logs.Warn("invalid parameter")
54 | return
55 | }
56 |
57 | appInfo := &models.AppInfo{}
58 | appInfo.AppName = appName
59 | appInfo.AppType = appType
60 | appInfo.DevelopPath = developPath
61 | appInfo.IP = strings.Split(ipstr, ",")
62 |
63 | err := models.CreateApp(appInfo)
64 | if err != nil {
65 | a.Data["Error"] = fmt.Errorf(err.Error())
66 | a.TplName = "app/error.html"
67 | return
68 | }
69 |
70 | a.Redirect("/app/list", 302)
71 | }
72 |
73 |
74 | func (a *AppController) TestRestful(){
75 | appList, err := models.GetAllAppInfo()
76 | if err != nil{
77 | logs.Warn("get appInfo failed ,err:",err)
78 | return
79 | }
80 | a.Data["json"] = map[string]interface{}{"success": 200, "data": appList}
81 | a.ServeJSON()
82 | }
83 |
--------------------------------------------------------------------------------
/goroutine/maze/maze.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | )
7 |
8 | func readMaze(fileName string)[][]int{
9 | file, err := os.Open(fileName)
10 | if err != nil{
11 | panic(err)
12 | }
13 |
14 | var row, col int
15 | fmt.Fscanf(file,"%d %d", &row, &col)
16 |
17 | maze := make([][]int,row)
18 | for i := range maze{
19 | maze[i] = make([]int, col)
20 | for j := range maze[i]{
21 | fmt.Fscanf(file,"%d",&maze[i][j])
22 | }
23 | }
24 | return maze
25 | }
26 |
27 | type point struct {
28 | i, j int
29 | }
30 |
31 | //directions, up -> left -> down -> right
32 | var dirs = [4]point{
33 | {-1,0},{0,-1},{1,0},{0,1}}
34 |
35 |
36 | func (p point) add(r point) point{
37 | return point{p.i +r.i,p.j+r.j}
38 | }
39 |
40 | func (p point) at(gird [][]int) (int, bool){
41 | if p.i < 0 || p.i >= len(gird){
42 | return 0,false
43 | }
44 |
45 | if p.j < 0 || p.j >= len(gird[p.i]){
46 | return 0,false
47 | }
48 | return gird[p.i][p.j],true
49 | }
50 |
51 | func walk(maze [][]int,start,end point) [][]int{
52 | steps := make([][]int,len(maze))
53 | for i:= range steps{
54 | steps[i] = make([]int,len(maze[i]))
55 | }
56 |
57 | Q := []point{start}
58 | for len(Q) > 0{
59 | cur := Q[0]
60 | Q = Q[1:]
61 |
62 | if cur == end{
63 | break
64 | }
65 | for _, dir := range dirs{
66 | next := cur.add(dir)
67 | // maze at next is zero
68 | val, ok := next.at(maze)
69 | if !ok || val == 1 {
70 | continue
71 | }
72 | // and step at next is zero
73 | val, ok = next.at(steps)
74 | if !ok || val !=0 {
75 | continue
76 | }
77 | // and next != start
78 | if next == start {
79 | continue
80 | }
81 | curSteps,_ := cur.at(steps)
82 | steps[next.i][next.j] = curSteps + 1
83 | Q = append(Q,next)
84 | }
85 | }
86 | return steps
87 | }
88 |
89 |
90 |
91 | func main(){
92 | maze := readMaze( "goroutine/maze/maze.dat")
93 | steps := walk(maze,point{0,0},point{len(maze) - 1,len(maze[0]) - 1})
94 |
95 | for _,row := range steps{
96 | for _, val := range row {
97 | fmt.Printf("%3d",val)
98 | }
99 | fmt.Println()
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/crawler-distributed/worker/types.go:
--------------------------------------------------------------------------------
1 | package worker
2 |
3 | import (
4 | "github.com/pkg/errors"
5 | "golang-awesome/crawler/engine"
6 | "golang-awesome/crawler/zhenai/parser"
7 | "log"
8 | )
9 |
10 | type SerializedParser struct{
11 | Name string
12 | Args interface{}
13 | }
14 |
15 | type Request struct {
16 | Url string
17 | Parser SerializedParser
18 | }
19 |
20 | type ParseResult struct {
21 | Items []engine.Item
22 | Requests []Request
23 | }
24 |
25 | func SerializeRequest(r engine.Request) Request{
26 | name, args := r.Parse.Serialize()
27 | return Request{
28 | Url:r.Url,
29 | Parser:SerializedParser{
30 | Name:name,
31 | Args:args,
32 | },
33 | }
34 | }
35 |
36 | func SerializeResult(r engine.ParserResult) (p ParseResult){
37 | p.Items = r.Items
38 | for _, req := range r.Requests{
39 | p.Requests = append(p.Requests, SerializeRequest(req))
40 | }
41 |
42 | return p
43 | }
44 |
45 |
46 | func deserializeParser(p SerializedParser)(engine.Parser, error){
47 | switch p.Name {
48 | case "ParseCity":
49 | return engine.NewFuncParser(parser.ParseCity,p.Name),nil
50 | case "ParseCityList":
51 | return engine.NewFuncParser(parser.ParseCityList,p.Name),nil
52 | case "ProfileParser":
53 | if userName, ok := p.Args.(string);ok{
54 | return parser.NewProfileParser(userName),nil
55 | }else {
56 | return nil,errors.New("invalid args for profileParser")
57 | }
58 | case "NilParser":
59 | return engine.NilParse{},nil
60 | default:
61 | return nil, errors.New("unknown parser name")
62 | }
63 |
64 | }
65 |
66 | func DeserializeRequest(r Request)(engine.Request,error){
67 | parse,err := deserializeParser(r.Parser)
68 | if err != nil {
69 | return engine.Request{}, err
70 | }
71 | req := engine.Request{
72 | Url: r.Url,
73 | Parse:parse,
74 | }
75 | return req, nil
76 | }
77 |
78 | func DeserializeResult(r ParseResult) engine.ParserResult{
79 | result := engine.ParserResult{
80 | Items: r.Items,
81 | }
82 |
83 | for _, req := range r.Requests{
84 | ereq, err := DeserializeRequest(req)
85 | if err != nil {
86 | log.Printf("error deserialize request: %v",err)
87 | continue
88 | }
89 | result.Requests = append(result.Requests,ereq)
90 | }
91 | return result
92 | }
93 |
--------------------------------------------------------------------------------
/logcollection/logagent/main/config.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "errors"
5 | "fmt"
6 | "github.com/astaxie/beego/config"
7 | "golang-awesome/logcollection/logagent/tail"
8 | )
9 |
10 | var (
11 | appConfig *Config
12 | )
13 |
14 | type Config struct {
15 | logLevel string
16 | logPath string
17 | chanSize int
18 | kafkaAddr string
19 | etcdAddr string
20 | etcdKey string
21 | collectConf []tail.CollectConf
22 | }
23 |
24 |
25 | func loadCollectConf(conf config.Configer) (err error) {
26 | var cc tail.CollectConf
27 | cc.LogPath = conf.String("collect::log_path")
28 | if len(cc.LogPath) == 0 {
29 | err = errors.New("invalid collect::log_path")
30 | return
31 | }
32 |
33 | cc.Topic = conf.String("collect::topic")
34 | if len(cc.Topic) == 0 {
35 | err = errors.New("invalid collect::topic")
36 | return
37 | }
38 |
39 | appConfig.collectConf = append(appConfig.collectConf, cc)
40 | return
41 | }
42 |
43 | func loadConf(confType string, filename string) (err error) {
44 | conf, err := config.NewConfig(confType, filename)
45 | if err != nil {
46 | fmt.Println("new config failed ,err :", err)
47 | return
48 | }
49 |
50 | appConfig = &Config{}
51 | appConfig.logLevel = conf.String("logs::log_level")
52 | if len(appConfig.logLevel) == 0 {
53 | appConfig.logLevel = "debug"
54 | }
55 |
56 | appConfig.logPath = conf.String("logs::log_path")
57 | if len(appConfig.logPath) == 0 {
58 | appConfig.logPath = "./logs"
59 | }
60 |
61 | appConfig.chanSize, err = conf.Int("collect::chan_size")
62 | if err != nil {
63 | appConfig.chanSize = 100
64 | }
65 |
66 | appConfig.kafkaAddr= conf.String("kafka::server_addr")
67 | if len(appConfig.kafkaAddr) == 0 {
68 | err = fmt.Errorf("invalid kafka address")
69 | return
70 | }
71 |
72 | appConfig.etcdAddr= conf.String("etcd::addr")
73 | if len(appConfig.etcdAddr) == 0 {
74 | err = fmt.Errorf("invalid etcd addr")
75 | return
76 | }
77 |
78 | appConfig.etcdKey= conf.String("etcd::configKey")
79 | if len(appConfig.etcdKey) == 0 {
80 | err = fmt.Errorf("invalid etcd key")
81 | return
82 | }
83 |
84 | err = loadCollectConf(conf)
85 | if err != nil {
86 | fmt.Printf("load collect conf failed, err:%v\n", err)
87 | return
88 | }
89 | return
90 | }
91 |
--------------------------------------------------------------------------------
/video_server/api/dbops/iternal.go:
--------------------------------------------------------------------------------
1 | package dbops
2 |
3 | import (
4 | "database/sql"
5 | "golang-awesome/video_server/api/defs"
6 | "log"
7 | "strconv"
8 | "sync"
9 | )
10 |
11 | func InsertSession(sid string, ttl int64, uname string)(err error ){
12 | ttlstr := strconv.FormatInt(ttl, 10)
13 | stmtIns, err := dbConn.Prepare("insert into sessions (session_id, TTL, login_name) values (?,?,?)")
14 | if err != nil {
15 | return
16 | }
17 | _, err = stmtIns.Exec(sid, ttlstr, uname)
18 | defer stmtIns.Close()
19 | return
20 | }
21 |
22 | func RetrieveSession(sid string)(session *defs.SimpleSession, err error){
23 | stmtOut, err := dbConn.Prepare("select TTL, login_name from sessions where session_id = ?")
24 | if err != nil {
25 | return
26 | }
27 | var ttl, uname string
28 | err = stmtOut.QueryRow(sid).Scan(&ttl, &uname)
29 | if err != nil && err != sql.ErrNoRows{
30 | return
31 | }
32 | if res, err := strconv.ParseInt(ttl,10,64); err == nil {
33 | session.TTL = res
34 | session.UserName = uname
35 | }else {
36 | return
37 | }
38 |
39 | defer stmtOut.Close()
40 | return
41 | }
42 |
43 | func RetrieveAllSessions()(m *sync.Map,err error){
44 | stmtOut, err := dbConn.Prepare("select * from sessions")
45 | if err != nil {
46 | log.Printf("%s",err)
47 | return
48 | }
49 | rows, err := stmtOut.Query()
50 | if err != nil {
51 | log.Printf("%s",err)
52 | return
53 | }
54 |
55 | for rows.Next() {
56 | var id, ttlstr, login_name string
57 | if err := rows.Scan(&id, &ttlstr, &login_name); err != nil {
58 | log.Printf("retrieve sessions error: %s",err)
59 | break
60 | }
61 | if ttl, err := strconv.ParseInt(ttlstr, 10, 64); err != nil {
62 | ss := &defs.SimpleSession{
63 | UserName:login_name,
64 | TTL: ttl,
65 | }
66 | m.Store(id, ss)
67 | log.Printf("session id: %s, ttl: %d",id,ss.TTL)
68 | }else{
69 | log.Printf("parse TTL error: %s",err)
70 | break
71 | }
72 | }
73 | return
74 | }
75 |
76 | func DeleteSession(sid string)(err error){
77 | stmtOut, err := dbConn.Prepare("delete from sessions where session_id =?")
78 | if err != nil {
79 | return
80 | }
81 | if _, err := stmtOut.Query(sid); err != nil {
82 | return
83 | }
84 |
85 | defer stmtOut.Close()
86 | return
87 | }
88 |
--------------------------------------------------------------------------------
/logcollection/web_admin/views/app/apply.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
17 |
43 |
44 |
16 |
51 |
52 |
53 |
56 |
57 |
58 |
--------------------------------------------------------------------------------
/video_server/web/handlers.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "github.com/julienschmidt/httprouter"
6 | "html/template"
7 | "io"
8 | "io/ioutil"
9 | "log"
10 | "net/http"
11 | "net/http/httputil"
12 | "net/url"
13 | )
14 |
15 | type HomePage struct {
16 | Name string
17 | }
18 |
19 | type UserPage struct {
20 | Name string
21 | }
22 |
23 | func homeHandler(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
24 | cname, err := r.Cookie("username")
25 | sid, err := r.Cookie("session")
26 | if err != nil {
27 | p := &HomePage{Name: "JAYCE"}
28 | t, e := template.ParseFiles("./templates/home.html")
29 | if e != nil {
30 | log.Printf("Parsing template home.html error: %s", e)
31 | return
32 | }
33 | t.Execute(w, p)
34 | return
35 | }
36 |
37 | if len(cname.Value) != 0 && len(sid.Value) != 0 {
38 | http.Redirect(w, r, "/userhome", http.StatusFound)
39 | return
40 | }
41 | }
42 |
43 | func userHomeHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
44 | cname, err := r.Cookie("username")
45 | _, err = r.Cookie("session")
46 | if err != nil {
47 | http.Redirect(w, r, "/", http.StatusFound)
48 | return
49 | }
50 |
51 | fname := r.FormValue("username")
52 | var p *UserPage
53 | if len(cname.Value) != 0 {
54 | p = &UserPage{Name: cname.Value}
55 | } else if len(fname) != 0 {
56 | p = &UserPage{Name: fname}
57 | }
58 | files, err := template.ParseFiles("./templates/userhome.html")
59 | if err != nil {
60 | log.Printf("parsing userhome.html error: %s", err)
61 | return
62 | }
63 |
64 | files.Execute(w, p)
65 | }
66 |
67 | func apiHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
68 | if r.Method != http.MethodPost {
69 | re, _ := json.Marshal(ErrorRequestNotRecognized)
70 | io.WriteString(w, string(re))
71 | return
72 | }
73 |
74 | res, _ := ioutil.ReadAll(r.Body)
75 | apiBody := &ApiBody{}
76 | if err := json.Unmarshal(res, apiBody); err != nil {
77 | re, _ := json.Marshal(ErrorRequestBodyParseFailed)
78 | io.WriteString(w, string(re))
79 | return
80 | }
81 | request(apiBody, w, r)
82 | defer r.Body.Close()
83 | }
84 |
85 | func proxyHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
86 | parse, _ := url.Parse("http://127.0.0.1:9000/")
87 | proxy := httputil.NewSingleHostReverseProxy(parse)
88 | proxy.ServeHTTP(w, r)
89 | }
90 |
--------------------------------------------------------------------------------
/seckill/SecProxy/service/config.go:
--------------------------------------------------------------------------------
1 | package service
2 |
3 | import (
4 | "github.com/garyburd/redigo/redis"
5 | "sync"
6 | "time"
7 | )
8 |
9 | const (
10 | ProductStatusNormal = 0
11 | ProductStatusSaleOut = 1
12 | ProductStatusForceSaleOut = 2
13 | )
14 |
15 | type RedisConf struct {
16 | RedisAddr string
17 | RedisPwd string
18 | RedisMaxIdle int
19 | RedisMaxActive int
20 | RedisIdleTimeout int
21 | }
22 |
23 | type EtcdConf struct {
24 | EtcdAddr string
25 | Timeout int
26 | EtcdSecKeyPrefix string
27 | EtcdSecProductKey string
28 | }
29 |
30 | type AccessLimitConf struct {
31 | IPSecAccessLimit int
32 | UserSecAccessLimit int
33 | IPMinAccessLimit int
34 | UserMinAccessLimit int
35 | }
36 |
37 | type SecProductInfoConf struct {
38 | ProductId int
39 | StartTime int64
40 | EndTime int64
41 | Status int
42 | Total int
43 | Left int
44 | }
45 |
46 | type SecKillConf struct {
47 | RedisBlackConf RedisConf
48 | RedisProxy2LayerConf RedisConf
49 | RedisLayer2ProxyConf RedisConf
50 |
51 | EtcdConf EtcdConf
52 | LogPath string
53 | LogLevel string
54 | SecProductInfoMap map[int]*SecProductInfoConf
55 | RWSecProductLock sync.RWMutex
56 | CookieSecretKey string
57 |
58 | ReferWhiteList []string
59 |
60 | ipBlackMap map[string]bool
61 | idBlackMap map[int]bool
62 |
63 | AccessLimitConf AccessLimitConf
64 | blackRedisPool *redis.Pool
65 | proxy2LayerRedisPool *redis.Pool
66 | layer2ProxyRedisPool *redis.Pool
67 |
68 | secLimitMgr *SecLimitMgr
69 |
70 | RWBlackLock sync.RWMutex
71 | WriteProxy2LayerGoroutineNum int
72 | ReadProxy2LayerGoroutineNum int
73 |
74 | SecReqChan chan*SecRequest
75 | SecReqChanSize int
76 |
77 | UserConnMap map[string]chan *SecResult
78 | UserConnMapLock sync.Mutex
79 | }
80 |
81 | type SecRequest struct {
82 | ProductId int
83 | Source string
84 | AuthCode string
85 | SecTime string
86 | Nance string
87 | UserId int
88 | UserAuthSign string
89 | AccessTime time.Time
90 | ClientAddr string
91 | ClientReferer string
92 | CloseNotify <-chan bool `json:"-"`
93 |
94 | ResultChan chan *SecResult `json:"-"`
95 | }
96 |
97 | type SecResult struct{
98 | ProductId int
99 | UserId int
100 | Code int
101 | Token string
102 | }
--------------------------------------------------------------------------------
/logcollection/web_admin/models/app.go:
--------------------------------------------------------------------------------
1 | package models
2 |
3 | import (
4 | "github.com/astaxie/beego/logs"
5 | "github.com/jmoiron/sqlx"
6 | )
7 |
8 | type AppInfo struct {
9 | AppId int `db:"app_id" json:"appId""`
10 | AppName string `db:"app_name" json:"appName"`
11 | AppType string `db:"app_type" json:"appType"`
12 | CreateTime string `db:"create_time" json:"createTime"`
13 | DevelopPath string `db:"dev_path" json:"developPath"`
14 | IP []string
15 | }
16 |
17 | var (
18 | Db *sqlx.DB
19 | )
20 |
21 | func InitDb(db *sqlx.DB) {
22 | Db = db
23 | }
24 |
25 | func GetAllAppInfo() (appList []AppInfo, err error) {
26 | err = Db.Select(&appList, "select app_id, app_name, app_type, create_time, dev_path from tbl_app_info")
27 | if err != nil {
28 | logs.Warn("Get All App Info failed, err:%v", err)
29 | return
30 | }
31 | return
32 | }
33 |
34 | func CreateApp(info *AppInfo)(err error){
35 |
36 | tx, err := Db.Begin()
37 | if err != nil{
38 | logs.Warn("createApp failed, Db.Begin error: %v",err)
39 | return
40 | }
41 |
42 | //事务提交或者回滚
43 | defer func(){
44 | if err != nil{
45 | tx.Rollback()
46 | return
47 | }
48 | tx.Commit()
49 | }()
50 |
51 | result, err := tx.Exec("insert into tbl_app_info(app_name,app_type,dev_path) values (?,?,?)",
52 | info.AppName, info.AppType, info.DevelopPath)
53 | if err != nil{
54 | logs.Warn("insert into tbl_app_info failed ,err:%v",err)
55 | return
56 | }
57 |
58 | id, err := result.LastInsertId()
59 | if err != nil{
60 | logs.Warn("get lastInsertId failed, err:%v",err)
61 | return
62 | }
63 | for _,ip := range info.IP{
64 | _, err = tx.Exec("insert into tbl_app_ip (app_id,ip) values (?,?)", id, ip)
65 | if err != nil {
66 | logs.Warn("insert into app_tbl_ip failed, err%v",err)
67 | }
68 | return
69 | }
70 |
71 | return
72 | }
73 |
74 |
75 | func GetIPInfoByName(appName string) (iplist []string, err error) {
76 |
77 | var appId []int
78 | err = Db.Select(&appId, "select app_id from tbl_app_info where app_name=?", appName)
79 | if err != nil || len(appId) == 0 {
80 | logs.Warn("select app_id failed, Db.Exec error:%v", err)
81 | return
82 | }
83 |
84 | err = Db.Select(&iplist, "select ip from tbl_app_ip where app_id=?", appId[0])
85 | if err != nil {
86 | logs.Warn("Get All App Info failed, err:%v", err)
87 | return
88 | }
89 |
90 | logs.Warn("get appId ",iplist)
91 | return
92 | }
--------------------------------------------------------------------------------
/logcollection/web_admin/controllers/LogController/log.go:
--------------------------------------------------------------------------------
1 | package LogController
2 |
3 | import (
4 | "fmt"
5 | "github.com/astaxie/beego"
6 | "github.com/astaxie/beego/logs"
7 | "golang-awesome/logcollection/web_admin/models"
8 | )
9 |
10 | type LogController struct {
11 | beego.Controller
12 | }
13 |
14 | func (a *LogController) LogList() {
15 | logs.Debug("enter logList controller")
16 | a.Layout = "layout/layout.html"
17 | a.Layout = "layout/layout.html"
18 | logList, err := models.GetAllLogInfo()
19 | if err != nil {
20 | a.Data["Error"] = fmt.Sprintf("服务器繁忙")
21 | a.TplName = "app/error.html"
22 |
23 | logs.Warn("get app list failed, err:%v", err)
24 | return
25 | }
26 |
27 | logs.Debug("get log list succ, data:%v", logList)
28 | a.Data["loglist"] = logList
29 |
30 | a.TplName = "log/index.html"
31 | }
32 |
33 | func (a *LogController) LogApply() {
34 | logs.Debug(" enter logApply controller")
35 | a.Layout = "layout/layout.html"
36 | a.TplName = "log/apply.html"
37 |
38 | }
39 |
40 | func (a *LogController) LogCreate() {
41 | logs.Debug("enter logCreate controller")
42 | appName := a.GetString("appName")
43 | logPath := a.GetString("logPath")
44 | topic := a.GetString("topic")
45 |
46 | a.Layout = "layout/layout.html"
47 | if len(appName) == 0 || len(logPath) == 0 || len(topic) == 0 {
48 | a.Data["Error"] = fmt.Sprintf("非法参数")
49 | a.TplName = "log/error.html"
50 |
51 | logs.Warn("invalid parameter")
52 | return
53 | }
54 |
55 | logInfo := &models.LogInfo{
56 | AppName: appName,
57 | LogPath: logPath,
58 | Topic: topic,
59 | }
60 |
61 | err := models.CreateLog(logInfo)
62 | logs.Debug(err)
63 | if err != nil {
64 | a.Data["Error"] = fmt.Errorf(err.Error())
65 | logs.Warn("create logInfo failed, err%v", err)
66 | a.TplName = "app/error.html"
67 | return
68 | }
69 |
70 | iplist, err := models.GetIPInfoByName(appName)
71 | if err != nil || len(iplist) == 0{
72 | a.Data["Error"] = fmt.Sprintf("获取项目ip失败，数据库繁忙")
73 | a.TplName = "log/error.html"
74 |
75 | logs.Warn("invalid parameter")
76 | return
77 | }
78 | keyFormat := "/net/badme/logagent/config/%s"
79 |
80 | logs.Debug(" get iplist:",iplist)
81 |
82 | for _, ip := range iplist {
83 | key := fmt.Sprintf(keyFormat, ip)
84 | err = models.SetLogConfToEtcd(key, logInfo)
85 | if err != nil {
86 | logs.Warn("Set log conf to etcd failed, err:%v", err)
87 | continue
88 | }
89 | }
90 | a.Redirect("/log/list", 302)
91 |
92 | }
93 |
--------------------------------------------------------------------------------
/seckill/SecProxy/tools/setconf/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "fmt"
7 | etcdClient "go.etcd.io/etcd/clientv3"
8 | "time"
9 |
10 |
11 | )
12 |
13 | const (
14 | EtcdKey = "/net/badme/backend/seckill/product"
15 | )
16 |
17 | type SecProductInfoConf = struct {
18 | ProductId int
19 | StartTime int64
20 | EndTime int64
21 | Status int
22 | Total int
23 | Left int
24 | OnePersonBuyLimit int
25 | BuyRate float64
26 | //每秒组多能卖多少个
27 | SoldMaxLimit int
28 | }
29 |
30 |
31 | func SetLogConfToEtcd() {
32 | cli, err := etcdClient.New(etcdClient.Config{
33 | Endpoints: []string{"localhost:2379", "localhost:22379", "localhost:32379"},
34 | DialTimeout: 5 * time.Second,
35 | })
36 | if err != nil {
37 | fmt.Println("connect failed, err:", err)
38 | return
39 | }
40 |
41 | fmt.Println("connect succ")
42 | defer cli.Close()
43 |
44 | now := time.Now().Unix()
45 | var SecProductInfoConfArr []SecProductInfoConf
46 | SecProductInfoConfArr = append(
47 | SecProductInfoConfArr,
48 | SecProductInfoConf{
49 | ProductId: 1001,
50 | StartTime: now - 60,
51 | EndTime: now + 3600,
52 | Status: 0,
53 | Total: 10,
54 | OnePersonBuyLimit:1,
55 | SoldMaxLimit:10000,
56 | BuyRate: 0.5,
57 | },
58 | )
59 | SecProductInfoConfArr = append(
60 | SecProductInfoConfArr,
61 | SecProductInfoConf{
62 | ProductId: 1002,
63 | StartTime: now - 60,
64 | EndTime: now + 3600,
65 | Status: 0,
66 | Total: 10,
67 | OnePersonBuyLimit:1,
68 | SoldMaxLimit:10000,
69 | BuyRate: 0.5,
70 | },
71 | )
72 |
73 | data, err := json.Marshal(SecProductInfoConfArr)
74 | if err != nil {
75 | fmt.Println("json failed, ", err)
76 | return
77 | }
78 |
79 | ctx, cancel := context.WithTimeout(context.Background(), time.Second)
80 | //cli.Delete(ctx, EtcdKey)
81 | //return
82 | _, err = cli.Put(ctx, EtcdKey, string(data))
83 | cancel()
84 | if err != nil {
85 | fmt.Println("put failed, err:", err)
86 | return
87 | }
88 |
89 | ctx, cancel = context.WithTimeout(context.Background(), time.Second)
90 | resp, err := cli.Get(ctx, EtcdKey)
91 | cancel()
92 | if err != nil {
93 | fmt.Println("get failed, err:", err)
94 | return
95 | }
96 | for _, ev := range resp.Kvs {
97 | fmt.Printf("%s : %s\n", ev.Key, ev.Value)
98 | }
99 | }
100 |
101 | func main() {
102 | SetLogConfToEtcd()
103 | }
104 |
--------------------------------------------------------------------------------
/logcollection/web_admin/views/layout/layout.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
17 |
49 |
50 |
11 |
30 |
31 |
63 |
64 |
74 |
75 |
76 |
--------------------------------------------------------------------------------
/logcollection/web_admin/views/app/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
32 |
50 |
51 |
52 |
53 |
54 |
58 |
59 |
62 |
55 | {{.LayoutContent}}
56 |
57 |
11 |
12 |
13 |
37 |
38 |
39 |
40 |
41 |
45 |
46 |
81 |
82 |
83 |
--------------------------------------------------------------------------------
/seckill/SecLayer/service/product.go:
--------------------------------------------------------------------------------
1 | package service
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "github.com/astaxie/beego/logs"
7 | "go.etcd.io/etcd/mvcc/mvccpb"
8 | "time"
9 | )
10 |
11 | func loadProductFromEtcd(conf *SecLayerConf)(err error){
12 |
13 | logs.Debug("start get product from etcd")
14 | ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*10)
15 | defer cancelFunc()
16 |
17 | resp,err := secLayerContext.etcdClient.Get(ctx,conf.EtcdConfig.EtcdSecProductKey)
18 | if err!= nil{
19 | logs.Debug("get [%s] from etcd failed, err: %v",conf.EtcdConfig.EtcdSecProductKey,err)
20 | return
21 | }
22 |
23 | logs.Debug("get from etcd success, resp: %v",resp)
24 | var secProductInfo []SecProductInfoConf
25 | for k,v := range resp.Kvs{
26 | logs.Debug("key[%v] value[%v]",k,v)
27 | err = json.Unmarshal(v.Value,&secProductInfo)
28 | if err!= nil {
29 | logs.Error("Unmarshal sec product info failed, err: %v",err)
30 | return
31 | }
32 |
33 | logs.Debug("sec info conf is [%v]",secProductInfo)
34 | }
35 |
36 | updateSecProductInfo(conf,secProductInfo)
37 | logs.Debug("update product info success, data:%v",err)
38 |
39 | initSecProductWatcher(conf)
40 | logs.Debug("init etcd watcher success")
41 | return
42 | }
43 |
44 |
45 | func updateSecProductInfo(conf *SecLayerConf,secProductIndo []SecProductInfoConf){
46 | tmp := make(map[int]*SecProductInfoConf,1024)
47 | for _,v :=range secProductIndo{
48 | productInfo := v
49 | productInfo.secLimit = &SecLimit{}
50 | tmp[v.ProductId] = &productInfo
51 | }
52 |
53 | secLayerContext.RWSecProductLock.Lock()
54 | conf.SecProductInfoMap = tmp
55 | secLayerContext.RWSecProductLock.Unlock()
56 | }
57 |
58 | func initSecProductWatcher(conf *SecLayerConf){
59 | go watchSecProductKey(conf)
60 | }
61 |
62 | func watchSecProductKey(conf *SecLayerConf){
63 |
64 | key := conf.EtcdConfig.EtcdSecProductKey
65 | logs.Debug("begin watch key: %s",key)
66 | var err error
67 | for{
68 | rch := secLayerContext.etcdClient.Watch(context.Background(),key)
69 | var secProductInfo []SecProductInfoConf
70 | var getConfSucc = true
71 |
72 | for wresp := range rch{
73 | for _,ev := range wresp.Events{
74 | if ev.Type == mvccpb.DELETE{
75 | logs.Warn("key[%s] 's config deleted ",key)
76 | continue
77 | }
78 |
79 | if ev.Type ==mvccpb.PUT && string(ev.Kv.Key) == key{
80 | err = json.Unmarshal(ev.Kv.Value,&secProductInfo)
81 | if err !=nil{
82 | logs.Error("key [%s] ,Unmarshal [%s] ,err: %v",err)
83 | getConfSucc = false
84 | continue
85 | }
86 | }
87 |
88 | logs.Debug("get config from etcd, %s %q: %q\n",ev.Type,ev.Kv.Key,ev.Kv.Value)
89 | }
90 |
91 | if getConfSucc {
92 | logs.Debug("get config from etcd success, %v",secProductInfo)
93 | updateSecProductInfo(conf,secProductInfo)
94 | }
95 | }
96 | }
97 | }
--------------------------------------------------------------------------------
/seckill/SecProxy/controller/skill.go:
--------------------------------------------------------------------------------
1 | package controller
2 |
3 | import (
4 | "fmt"
5 | "github.com/astaxie/beego"
6 | "github.com/astaxie/beego/logs"
7 | "golang-awesome/seckill/SecProxy/service"
8 | "strings"
9 | "time"
10 | )
11 |
12 | type SkillController struct {
13 | beego.Controller
14 | }
15 |
16 | func (p *SkillController) SecKill(){
17 |
18 | productId,err := p.GetInt("product_id")
19 | result := make(map[string]interface{})
20 | result["code"] = 0
21 | result["message"] = "success"
22 |
23 | defer func() {
24 | p.Data["json"] =result
25 | p.ServeJSON()
26 | }()
27 |
28 | if err !=nil{
29 | result["code"] =service.ErrInvalidRequest
30 | result["message"] = "invalid product_id"
31 | return
32 | }
33 |
34 | source :=p.GetString("src")
35 | authCode := p.GetString("authcode")
36 | secTime :=p.GetString("time")
37 | nance := p.GetString("nance")
38 |
39 | secRequest := service.NewSecRequest()
40 | secRequest.AuthCode =authCode
41 | secRequest.Nance = nance
42 | secRequest.ProductId = productId
43 | secRequest.SecTime = secTime
44 | secRequest.Source =source
45 | secRequest.UserAuthSign = p.Ctx.GetCookie("userAuthSign")
46 | secRequest.UserId,_ = p.GetInt("user_id")
47 | secRequest.AccessTime = time.Now()
48 | if len(p.Ctx.Request.RemoteAddr) >0{
49 | secRequest.ClientAddr = strings.Split(p.Ctx.Request.RemoteAddr,":")[0]
50 | }
51 | secRequest.ClientReferer = p.Ctx.Request.Referer()
52 | secRequest.CloseNotify = p.Ctx.ResponseWriter.CloseNotify()
53 |
54 | logs.Debug("client request: [%v]",secRequest)
55 | if err !=nil{
56 | result["code"] = service.ErrInvalidRequest
57 | result["message"] = fmt.Sprintf("invalid cookokie : userid")
58 | return
59 | }
60 |
61 | data,code,err := service.SecKill(secRequest)
62 | if err != nil {
63 | result["code"] = code
64 | result["message"] = err.Error()
65 | return
66 | }
67 | result["data"] = data
68 | result["code"] = code
69 | return
70 | }
71 |
72 | func (p *SkillController) SecInfo(){
73 |
74 | productId ,err := p.GetInt("product_id")
75 | result := make(map[string]interface{})
76 | logs.Debug("product_id:%d",productId)
77 |
78 | result["code"] = 0
79 | result["message"] = "success"
80 |
81 | defer func(){
82 | p.Data["json"] = result
83 | p.ServeJSON()
84 | }()
85 |
86 | if err != nil {
87 | data, code, err := service.SecInfoList()
88 | if err != nil {
89 | result["code"] = code
90 | result["message"] = err.Error()
91 |
92 | logs.Error("invalid request, get product_id failed, err:%v",err)
93 | return
94 | }
95 | result["code"] = code
96 | result["data"] = data
97 | }else{
98 | data, code, err := service.SecInfo(productId)
99 | if err != nil {
100 | result["code"] = code
101 | result["message"] = err.Error()
102 |
103 | logs.Error("invalid request,get product_id falied, err :%v",err)
104 | return
105 | }
106 |
107 | result["data"] = data
108 | result["code"] = code
109 | }
110 | }
--------------------------------------------------------------------------------
/logcollection/web_admin/models/log.go:
--------------------------------------------------------------------------------
1 | package models
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "github.com/astaxie/beego/logs"
7 | etcdclient "go.etcd.io/etcd/clientv3"
8 | "golang-awesome/logcollection/logagent/tail"
9 | "time"
10 | )
11 |
12 | var(
13 | etcdClient *etcdclient.Client
14 | )
15 | type LogInfo struct {
16 | AppId int `db:"app_id" json:"app_id"`
17 | AppName string `db:"app_name"`
18 | LogId int `db:"log_id"`
19 | Status int `db:"status"`
20 | CreateTime string `db:"create_time"`
21 | LogPath string `db:"log_path"`
22 | Topic string `db:"topic"`
23 | }
24 |
25 | func InitEtcd(client *etcdclient.Client) {
26 | etcdClient = client
27 | }
28 |
29 |
30 | func GetAllLogInfo() (loglist []LogInfo, err error) {
31 | err = Db.Select(&loglist,
32 | "select a.app_id, b.app_name, a.create_time, a.topic, a.log_id, a.status, a.log_path from tbl_log_info a, tbl_app_info b where a.app_id=b.app_id")
33 | if err != nil {
34 | logs.Warn("Get All App Info failed, err:%v", err)
35 | return
36 | }
37 | return
38 | }
39 |
40 | func CreateLog(info *LogInfo) (err error){
41 | tx, err := Db.Begin()
42 |
43 | if err != nil{
44 | logs.Warn("create log failed,err %v",err)
45 | return
46 | }
47 |
48 | defer func(){
49 | if err != nil {
50 | tx.Rollback()
51 | return
52 | }
53 | tx.Commit()
54 | }()
55 |
56 | var appList []AppInfo
57 | err = Db.Select(&appList, "select app_id,app_name from tbl_app_info where app_name = ?", info.AppName)
58 |
59 | if err != nil || len(appList) == 0{
60 | logs.Warn("select appInfo failed, err:%v",err)
61 | return
62 | }
63 | info.AppId = appList[0].AppId
64 | info.AppName = appList[0].AppName
65 | logs.Debug(info.AppId)
66 | result, err := tx.Exec("insert into tbl_log_info (app_id,app_name,log_path,topic) values (?,?,?,?)",
67 | info.AppId, info.AppName,info.LogPath, info.Topic)
68 |
69 | if err != nil {
70 | logs.Warn("insert into tbl_log_info falied, err %v",err)
71 | return
72 | }
73 |
74 | _, err = result.LastInsertId()
75 |
76 | if err != nil{
77 | logs.Warn("get last insert id failes, err %v",err)
78 | return
79 | }
80 |
81 | return
82 | }
83 |
84 | func SetLogConfToEtcd(etcdKey string,info *LogInfo)(err error){
85 |
86 | var logConfArr []tail.CollectConf
87 |
88 | logConfArr = append(
89 | logConfArr,
90 | tail.CollectConf{
91 | LogPath:info.LogPath,
92 | Topic:info.Topic,
93 | },
94 | )
95 | logs.Warn("etcdctl:",etcdKey)
96 | bytes, err := json.Marshal(logConfArr)
97 | if err != nil {
98 | logs.Warn("marshal log conf failed, err%v",err)
99 | return
100 | }
101 |
102 | ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second)
103 | _, err = etcdClient.Put(ctx, etcdKey, string(bytes))
104 | cancelFunc()
105 |
106 | if err != nil {
107 | logs.Warn("put log conf to etcd failed, err%v",err)
108 | return
109 | }
110 |
111 | logs.Debug("put etcd success, data:%v",string(bytes))
112 |
113 | return
114 | }
--------------------------------------------------------------------------------
/logcollection/logagent/main/etcd.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "github.com/astaxie/beego/logs"
7 | "go.etcd.io/etcd/clientv3"
8 | "go.etcd.io/etcd/mvcc/mvccpb"
9 | "golang-awesome/logcollection/logagent/tail"
10 | "time"
11 | )
12 |
13 | type EtcdClient struct {
14 | client *clientv3.Client
15 | keys []string
16 | }
17 |
18 | var (
19 | etcdClient *EtcdClient
20 | )
21 |
22 | func initEtcd(addr,key string)(collectConf tail.CollectConf,err error){
23 | client, err := clientv3.New(clientv3.Config{
24 | Endpoints: []string{"localhost:2379", "localhost:22379", "localhost:32379"},
25 | DialTimeout: 5 * time.Second,
26 | })
27 |
28 | if err != nil {
29 | logs.Error("connect etcd failed, err:",err)
30 |
31 | }
32 |
33 | etcdClient =&EtcdClient{
34 | client:client,
35 | }
36 |
37 | //if strings.HasSuffix(key,"/") == false {
38 | // key = key + "/"
39 | //}
40 |
41 | //for _,ip := range localIPArray{
42 | //etcdKey := fmt.Sprintf("%s%s", key, ip)
43 | //etcdKey := fmt.Sprintf("%s", key)
44 | etcdClient.keys = append(etcdClient.keys, key)
45 | ctx, cancel := context.WithTimeout(context.Background(), time.Second)
46 | resp, err := client.Get(ctx, key)
47 | if err != nil {
48 | logs.Error("client get from etcd failed, err:%v", err)
49 | }
50 | cancel()
51 | logs.Debug("resps from etcd:%v", resp.Kvs)
52 | for _, v := range resp.Kvs {
53 | if string(v.Key) == key {
54 | logs.Debug("v.Key:",v.Key)
55 | logs.Debug("v.Value:",v.Value)
56 | err = json.Unmarshal(v.Value, &collectConf)
57 | if err != nil {
58 | logs.Error("unmarshal failed, err:%v", err)
59 | continue
60 | }
61 |
62 | logs.Debug("log config is %v", collectConf)
63 | }
64 | }
65 | //}
66 |
67 | initEtcdWatcher()
68 | return
69 | }
70 |
71 | func initEtcdWatcher() {
72 |
73 | for _, key := range etcdClient.keys {
74 | logs.Debug("etcd key :" ,key)
75 | go watchKey(key)
76 | }
77 | }
78 |
79 | func watchKey(key string) {
80 |
81 | cli, err := clientv3.New(clientv3.Config{
82 | Endpoints: []string{"localhost:2379", "localhost:22379", "localhost:32379"},
83 | DialTimeout: 5 * time.Second,
84 | })
85 | if err != nil {
86 | logs.Error("connect etcd failed, err:", err)
87 | return
88 | }
89 |
90 | logs.Debug("begin watch key:%s", key)
91 | for {
92 | rch := cli.Watch(context.Background(), key)
93 | var collectConf []tail.CollectConf
94 | var getConfSucc = true
95 | for wresp := range rch {
96 | for _, ev := range wresp.Events {
97 | if ev.Type == mvccpb.DELETE {
98 | logs.Warn("key[%s] 's config deleted", key)
99 | continue
100 | }
101 |
102 | if ev.Type == mvccpb.PUT && string(ev.Kv.Key) == key {
103 | err = json.Unmarshal(ev.Kv.Value, &collectConf)
104 | if err != nil {
105 | logs.Error("key [%s], Unmarshal[%s], err:%v ", err)
106 | getConfSucc = false
107 | continue
108 | }
109 | }
110 | logs.Debug("get config from etcd, %s %q : %q\n", ev.Type, ev.Kv.Key, ev.Kv.Value)
111 | }
112 |
113 | if getConfSucc {
114 | logs.Debug("get config from etcd succ, %v", collectConf)
115 | tail.UpdateConfig(collectConf)
116 | }
117 | }
118 |
119 | }
120 | }
121 |
--------------------------------------------------------------------------------
/crawler/zhenai/parser/profile.go:
--------------------------------------------------------------------------------
1 | package parser
2 |
3 | import (
4 | "golang-awesome/crawler/engine"
5 | "golang-awesome/crawler/model"
6 | "regexp"
7 | )
8 | //| 15 | | 项目ID | 16 |项目名称 | 17 |项目类型 | 18 |创建时间 | 19 |部署路径 | 20 |操作 | 21 |
|---|---|---|---|---|---|---|
| 27 | | {{.AppId}} | 28 |{{.AppName}} | 29 |{{.AppType}} | 30 |{{.CreateTime}} | 31 |{{.DevelopPath}} | 32 |33 | |
离异
9 | //var ageRe = regexp.MustCompile(`ID：([^<]+)
`)
22 | var Re = regexp.MustCompile(`([^<]+)
`)
23 |
24 |
25 | func ParseProfile(contents []byte, url, name string) engine.ParserResult{
26 | profile := &model.Profile{}
27 | profile.Name = name
28 | //idsubmatch := IdRe.FindSubmatch(contents)
29 | //profile.Id = string(idsubmatch[1])
30 | submatch := Re.FindAllSubmatch(contents, -1)
31 | if len(submatch) > 0{
32 | profile.Marriage = string(submatch[0][1])
33 | profile.Age = string(submatch[1][1])
34 | profile.Xingzuo = string(submatch[2][1])
35 | profile.Height = string(submatch[3][1])
36 | profile.Weight = string(submatch[4][1])
37 | profile.Hukou = string(submatch[5][1])
38 | profile.Income = string(submatch[6][1])
39 | profile.Education = string(submatch[len(submatch)-1][1])
40 | }
41 |
42 |
43 | item := engine.Item{
44 | Url:url,
45 | Payload:profile,
46 | Type:"zhenai",
47 | Id:profile.Id,
48 | }
49 | rs := engine.ParserResult{}
50 | rs.Items = []engine.Item{item}
51 | // 取本页面内，猜你喜欢的的
52 | //var guessRe = regexp.MustCompile(`href="(http://album.zhenai.com/u/\w+)"[^>]*>([^<]+)`)
53 | //ms := guessRe.FindAllSubmatch(contents, -1)
54 | //for _, m := range ms {
55 | // rs.Requests = append(rs.Requests, engine.Request{
56 | // Url: string(m[1]),
57 | // Parse: NewProfileParser(string(m[2])),
58 | // })
59 | //}
60 | return rs
61 | }
62 |
63 | //func extractString(contents []byte, re *regexp.Regexp) string{
64 | // match := re.FindSubmatch(contents)
65 | // if len(match) >= 2 {
66 | // return string(match[1])
67 | // }else {
68 | // return ""
69 | //
70 | // }
71 | //}
72 |
73 | type ProfileParser struct{
74 | userName string
75 | }
76 |
77 | func NewProfileParser(userName string) *ProfileParser {
78 | return &ProfileParser{userName:
79 | userName}
80 | }
81 |
82 | func (p *ProfileParser) Parse(contents []byte, url string) engine.ParserResult {
83 | return ParseProfile(contents, url, p.userName)
84 | }
85 |
86 | func (p *ProfileParser) Serialize() (name string, args interface{}) {
87 | return "ProfileParser", p.userName
88 | }
89 |
90 |
--------------------------------------------------------------------------------
/logcollection/logagent/tail/tail.go:
--------------------------------------------------------------------------------
1 | package tail
2 |
3 | import (
4 | "fmt"
5 | "github.com/astaxie/beego/logs"
6 | "github.com/hpcloud/tail"
7 | "sync"
8 | "time"
9 | )
10 |
11 | const (
12 | StatusNormal = 1
13 | StatusDelete = 2
14 | )
15 |
16 | type CollectConf struct {
17 | LogPath string `json:"logpath"`
18 | Topic string `json:"topic"`
19 | }
20 |
21 | type TailObj struct {
22 | tail *tail.Tail
23 | conf CollectConf
24 | status int
25 | exitChan chan int
26 | }
27 |
28 | type TextMsg struct {
29 | Msg string
30 | Topic string
31 | }
32 | type TailObjMgr struct {
33 | tailObjs []*TailObj
34 | msgChan chan *TextMsg
35 | lock sync.Mutex
36 | }
37 |
38 | var (
39 | tailObjMgr *TailObjMgr
40 | )
41 |
42 | func GetOneLine() (msg *TextMsg) {
43 | msg = <-tailObjMgr.msgChan
44 | return
45 | }
46 |
47 | func UpdateConfig(confs []CollectConf) (err error) {
48 | tailObjMgr.lock.Lock()
49 | defer tailObjMgr.lock.Unlock()
50 |
51 | for _, oneConf := range confs {
52 | var isRunning = false
53 | for _, obj := range tailObjMgr.tailObjs {
54 | if oneConf.LogPath == obj.conf.LogPath {
55 | isRunning = true
56 | break
57 | }
58 | }
59 | if isRunning {
60 | continue
61 | }
62 |
63 | createNewTask(oneConf)
64 | }
65 |
66 | var tailObjs []*TailObj
67 | for _, obj := range tailObjMgr.tailObjs {
68 | obj.status = StatusDelete
69 | for _, oneConf := range confs {
70 | if oneConf.LogPath == obj.conf.LogPath {
71 | obj.status = StatusNormal
72 | break
73 | }
74 | }
75 | if obj.status == StatusDelete {
76 | obj.exitChan <- 1
77 | continue
78 | }
79 | tailObjs = append(tailObjs, obj)
80 | }
81 | tailObjMgr.tailObjs = tailObjs
82 | return
83 | }
84 |
85 | func createNewTask(conf CollectConf) {
86 |
87 | obj := &TailObj{
88 | conf: conf,
89 | exitChan: make(chan int, 1),
90 | }
91 |
92 | file, e := tail.TailFile(conf.LogPath, tail.Config{
93 | ReOpen: true,
94 | Follow: true,
95 | //Location: &tail.SeekInfo{Offset:0,Whence:2,},
96 | MustExist: false,
97 | Poll: true,
98 | })
99 |
100 | if e != nil {
101 | logs.Error("collect filename[%s] failed ,err:%v", e)
102 | return
103 | }
104 | obj.tail = file
105 | tailObjMgr.tailObjs = append(tailObjMgr.tailObjs, obj)
106 |
107 | go readFromTail(obj)
108 | }
109 |
110 | func InitTail(conf []CollectConf, chanSize int) (err error) {
111 |
112 | if len(conf) == 0 {
113 | err = fmt.Errorf("invalid config for log collect: conf:%v", conf)
114 | return
115 | }
116 |
117 | tailObjMgr = &TailObjMgr{
118 | msgChan: make(chan *TextMsg, chanSize),
119 | }
120 |
121 | for _, v := range conf {
122 | obj := &TailObj{
123 | conf: v,
124 | }
125 |
126 | tails, errTail := tail.TailFile(v.LogPath, tail.Config{
127 | ReOpen: true,
128 | Follow: true,
129 | //Location: &tail.SeekInfo{Offset:0,Whence:2,},
130 | MustExist: false,
131 | Poll: true,
132 | })
133 |
134 | if errTail != nil {
135 | err = errTail
136 | return
137 | }
138 |
139 | obj.tail = tails
140 |
141 | tailObjMgr.tailObjs = append(tailObjMgr.tailObjs, obj)
142 |
143 | go readFromTail(obj)
144 | }
145 | return
146 | }
147 |
148 | func readFromTail(tailObj *TailObj) {
149 | for {
150 | lines, ok := <-tailObj.tail.Lines
151 | if !ok {
152 | logs.Warn("tail file close reopen ,filename:%s\n", tailObj.tail.Filename)
153 | time.Sleep(100 * time.Millisecond)
154 | continue
155 | }
156 |
157 | textMsg := &TextMsg{
158 | Msg: lines.Text,
159 | Topic: tailObj.conf.Topic,
160 | }
161 |
162 | tailObjMgr.msgChan <- textMsg
163 | }
164 | }
165 |
--------------------------------------------------------------------------------
/goroutine/chart/chat_server/main/client.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/binary"
5 | "encoding/json"
6 | "errors"
7 | "fmt"
8 | "net"
9 | )
10 |
11 | type Client struct {
12 | conn net.Conn
13 | buf [8192]byte
14 | }
15 |
16 | func (p *Client) readPackage() (msg Message, err error) {
17 |
18 | n, err := p.conn.Read(p.buf[0:4])
19 | if n != 4 {
20 | err = errors.New("read header failed")
21 | return
22 | }
23 | fmt.Println("read package:", p.buf[0:4])
24 |
25 | var packLen uint32
26 | packLen = binary.BigEndian.Uint32(p.buf[0:4])
27 |
28 | fmt.Printf("receive len:%d", packLen)
29 | n, err = p.conn.Read(p.buf[0:packLen])
30 | if n != int(packLen) {
31 | err = errors.New("read body failed")
32 | return
33 | }
34 |
35 | fmt.Printf("receive data:%s\n", string(p.buf[0:packLen]))
36 | err = json.Unmarshal(p.buf[0:packLen], &msg)
37 | if err != nil {
38 | fmt.Println("unmarshal failed, err:", err)
39 | }
40 | return
41 | }
42 |
43 | func (p *Client) writePackage(data []byte) (err error) {
44 |
45 | packLen := uint32(len(data))
46 |
47 | binary.BigEndian.PutUint32(p.buf[0:4], packLen)
48 | n, err := p.conn.Write(p.buf[0:4])
49 | if err != nil {
50 | fmt.Println("write data failed")
51 | return
52 | }
53 |
54 | n, err = p.conn.Write(data)
55 | if err != nil {
56 | fmt.Println("write data failed")
57 | return
58 | }
59 |
60 | if n != int(packLen) {
61 | fmt.Println("write data not finished")
62 | err = errors.New("write data not fninshed")
63 | return
64 | }
65 |
66 | return
67 | }
68 |
69 | func (p *Client) Process() (err error) {
70 |
71 | for {
72 | var msg Message
73 | msg, err = p.readPackage()
74 | if err != nil {
75 | return err
76 | }
77 |
78 | err = p.processMsg(msg)
79 | if err != nil {
80 | return
81 | }
82 | }
83 | }
84 |
85 | func (p *Client) processMsg(msg Message) (err error) {
86 |
87 | switch msg.Cmd {
88 | case UserLogin:
89 | err = p.login(msg)
90 | case UserRegister:
91 | err = p.register(msg)
92 | default:
93 | err = errors.New("unsupport message")
94 | return
95 | }
96 | return
97 | }
98 |
99 | func (p *Client) loginResp(err error) {
100 | var respMsg Message
101 | respMsg.Cmd = UserLoginRes
102 |
103 | var loginRes LoginCmdRes
104 | loginRes.Code = 200
105 |
106 | if err != nil {
107 | loginRes.Code = 500
108 | loginRes.Error = fmt.Sprintf("%v", err)
109 | }
110 |
111 | data, err := json.Marshal(loginRes)
112 | if err != nil {
113 | fmt.Println("marshal failed, ", err)
114 | return
115 | }
116 |
117 | respMsg.Data = string(data)
118 | data, err = json.Marshal(respMsg)
119 | if err != nil {
120 | fmt.Println("marshal failed, ", err)
121 | return
122 | }
123 | err = p.writePackage(data)
124 | if err != nil {
125 | fmt.Println("send failed, ", err)
126 | return
127 | }
128 | }
129 |
130 | func (p *Client) login(msg Message) (err error) {
131 | defer func() {
132 | p.loginResp(err)
133 | }()
134 |
135 | fmt.Printf("recv user login request, data:%v", msg)
136 | var cmd LoginCmd
137 | err = json.Unmarshal([]byte(msg.Data), &cmd)
138 | if err != nil {
139 | return
140 | }
141 |
142 | _, err = mgr.Login(cmd.Id, cmd.Passwd)
143 | if err != nil {
144 | return
145 | }
146 |
147 | return
148 | }
149 |
150 | func (p *Client) register(msg Message) (err error) {
151 | var cmd RegisterCmd
152 | err = json.Unmarshal([]byte(msg.Data), &cmd)
153 | if err != nil {
154 | return
155 | }
156 |
157 | err = mgr.Register(&cmd.User)
158 | if err != nil {
159 | return
160 | }
161 |
162 | return
163 | }
164 |
--------------------------------------------------------------------------------
/seckill/SecProxy/service/service.go:
--------------------------------------------------------------------------------
1 | package service
2 |
3 | import (
4 | "fmt"
5 | "github.com/astaxie/beego/logs"
6 | "time"
7 | )
8 |
9 | var (
10 | secKillConf *SecKillConf
11 | )
12 |
13 | func NewSecRequest() (secRequest *SecRequest) {
14 | secRequest = &SecRequest{
15 | ResultChan: make(chan *SecResult, 1),
16 | }
17 | return
18 | }
19 |
20 | func SecInfoList() (data []map[string]interface{}, code int, err error) {
21 | secKillConf.RWBlackLock.RLock()
22 | defer secKillConf.RWBlackLock.RUnlock()
23 |
24 | for _, v := range secKillConf.SecProductInfoMap {
25 | item, _, err := SecInfoById(v.ProductId)
26 | if err != nil {
27 | logs.Error("get product_id[%d] failed, err:%v", err)
28 | continue
29 | }
30 |
31 | logs.Debug("get product[%d], result[%v], all[%v] v[%v]", v.ProductId, item, secKillConf.SecProductInfoMap, v)
32 | data = append(data, item)
33 | }
34 |
35 | return
36 | }
37 |
38 | func SecInfoById(productId int) (data map[string]interface{}, code int, err error) {
39 | secKillConf.RWBlackLock.RLock()
40 | defer secKillConf.RWBlackLock.RUnlock()
41 |
42 | v, ok := secKillConf.SecProductInfoMap[productId]
43 | if !ok {
44 | code = ErrNotFoundProductId
45 | err = fmt.Errorf("not found product id: %d", productId)
46 | return
47 | }
48 |
49 | start := false
50 | end := false
51 | status := "success"
52 |
53 | now := time.Now().Unix()
54 | if now-v.StartTime < 0 {
55 | start = false
56 | end = false
57 | status = "sec kill is not start"
58 | code = ErrActiveNotStart
59 | }
60 |
61 | if now-v.StartTime > 0 {
62 | start = true
63 | }
64 |
65 | if now-v.EndTime > 0 {
66 | start = false
67 | end = true
68 | status = "sec kill is already end"
69 | code = ErrActiveAlreadyEnd
70 | }
71 |
72 | if v.Status == ProductStatusForceSaleOut || v.Status == ProductStatusSaleOut {
73 | start = false
74 | end = true
75 | status = "product is sale out"
76 | code = ErrActiveSaleOut
77 | }
78 |
79 | data = make(map[string]interface{})
80 | data["product_id"] = productId
81 | data["start"] = start
82 | data["end"] = end
83 | data["status"] = status
84 |
85 | return
86 | }
87 |
88 | func SecInfo(productId int) (data []map[string]interface{}, code int, err error) {
89 | secKillConf.RWSecProductLock.RLock()
90 | defer secKillConf.RWSecProductLock.RUnlock()
91 |
92 | item, code, err := SecInfoById(productId)
93 |
94 | if err != nil {
95 | return
96 | }
97 | data = append(data, item)
98 | return
99 | }
100 |
101 | func SecKill(req *SecRequest) (data map[string]interface{}, code int, err error) {
102 |
103 | //secKillConf.RWSecProductLock.RLock()
104 | //defer secKillConf.RWSecProductLock.RUnlock()
105 |
106 | //err = antiSpam(req)
107 | //if err != nil {
108 | // code = ErrUserServiceBusy
109 | // logs.Warn("userId[%d] invalid, check failed, req[%v]", req.UserId, req)
110 | // return
111 | //}
112 |
113 | data, code, err = SecInfoById(req.ProductId)
114 | if code != 0 {
115 | logs.Warn("userId[%d] secInfoBy id failed, req[%v]", req.UserId, req)
116 | return
117 | }
118 |
119 | userKey := fmt.Sprintf("%s_%s", req.UserId, req.ProductId)
120 | secKillConf.RWSecProductLock.RLock()
121 | secKillConf.UserConnMap[userKey] = req.ResultChan
122 | secKillConf.RWSecProductLock.RUnlock()
123 |
124 | secKillConf.SecReqChan <- req
125 | ticker := time.NewTicker(time.Second * 10)
126 |
127 | defer func() {
128 | ticker.Stop()
129 | secKillConf.UserConnMapLock.Lock()
130 | delete(secKillConf.UserConnMap, userKey)
131 | secKillConf.UserConnMapLock.Unlock()
132 | }()
133 |
134 | select {
135 | case <-ticker.C:
136 | code = ErrProcessTimeout
137 | err = fmt.Errorf("request timeout")
138 | return
139 |
140 | case <-req.CloseNotify:
141 | code = ErrClientClosed
142 | err = fmt.Errorf("client already closed")
143 | return
144 | case result := <-req.ResultChan:
145 | code = result.Code
146 | data["product_id"] = result.ProductId
147 | data["token"] = result.Token
148 | data["user_id"] = result.UserId
149 | return
150 | }
151 | return
152 | }
153 |
--------------------------------------------------------------------------------
/video_server/templates/home.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
105 |
110 |
111 |
112 |
113 |
114 |
119 |
120 |
121 |
123 |
124 |
145 |
146 |
167 |
168 |
169 |
170 |
--------------------------------------------------------------------------------
/seckill/SecProxy/load/init.go:
--------------------------------------------------------------------------------
1 | package load
2 |
3 | import (
4 | "context"
5 | "encoding/json"
6 | "fmt"
7 | "github.com/astaxie/beego/logs"
8 |
9 | etcd "go.etcd.io/etcd/clientv3"
10 | "go.etcd.io/etcd/mvcc/mvccpb"
11 | "golang-awesome/seckill/SecProxy/service"
12 | "time"
13 | )
14 |
15 | var (
16 | //redisClient *redis.Client
17 | //redisPool *redis.Pool
18 | etcdClient *etcd.Client
19 | )
20 |
21 | //func initRedis()(err error){
22 | // redisClient = redis.NewClient(&redis.Options{
23 | // Addr: secKillConf.RedisConf.RedisAddr,
24 | // Password: "123456", // no password set
25 | // DB: 0, // use default DB
26 | // })
27 | //
28 | // pong, err := redisClient.Ping().Result()
29 | //
30 | // fmt.Println(pong, err)
31 | //
32 | // if err != nil {
33 | // logs.Error("ping redis failed,err :%v",err)
34 | // return
35 | // }
36 | //
37 | // return
38 | //}
39 |
40 | //func initRedis() (err error) {
41 | // redisPool = &redis.Pool{
42 | // MaxIdle: secKillConf.RedisConf.RedisMaxIdle,
43 | // MaxActive: secKillConf.RedisConf.RedisMaxActive,
44 | // IdleTimeout: time.Duration(secKillConf.RedisConf.RedisIdleTimeout) * time.Second,
45 | // Dial: func() (redis.Conn, error) {
46 | // redis.Dial("tcp", secKillConf.RedisConf.RedisAddr)
47 | // c, err := redis.Dial("tcp", secKillConf.RedisConf.RedisAddr)
48 | // if err != nil {
49 | // return nil, err
50 | // }
51 | // if _, err := c.Do("AUTH", secKillConf.RedisConf.RedisPwd); err != nil {
52 | // c.Close()
53 | // return nil, err
54 | // }
55 | // return c, err
56 | // },
57 | // }
58 | //
59 | // conn := redisPool.Get()
60 | // defer conn.Close()
61 | //
62 | // _, err = conn.Do("ping")
63 | // if err != nil {
64 | // logs.Error("ping redis failed, err:%v", err)
65 | // return
66 | // }
67 | //
68 | // return
69 | //}
70 |
71 |
72 | func initEtcd()(err error){
73 | logs.Debug("etcd timeout: %v ",time.Duration(secKillConf.EtcdConf.Timeout) * time.Second)
74 | client, err := etcd.New(etcd.Config{
75 | Endpoints: []string{secKillConf.EtcdConf.EtcdAddr},
76 | DialTimeout: time.Duration(secKillConf.EtcdConf.Timeout) * time.Millisecond,
77 | })
78 |
79 | if err != nil {
80 | logs.Error("connect etcd failed ,err:", err)
81 | return
82 | }
83 |
84 | etcdClient = client
85 | return
86 | }
87 |
88 | func convertLogLevel (level string) int {
89 | switch level{
90 | case "debug":
91 | return logs.LevelDebug
92 | case "warn":
93 | return logs.LevelWarn
94 | case "info":
95 | return logs.LevelInfo
96 | case "trace":
97 | return logs.LevelTrace
98 | default:
99 | return logs.LevelDebug
100 | }
101 | }
102 |
103 |
104 | func initLogger() (err error){
105 | config := make(map[string]interface{})
106 | config["filename"] = secKillConf.LogPath
107 | config["level"] = convertLogLevel(secKillConf.LogLevel)
108 |
109 | bytes, err := json.Marshal(config)
110 | if err != nil {
111 | fmt.Println(" marshal failed,err: ",err)
112 | return
113 | }
114 |
115 | logs.SetLogger(logs.AdapterFile,string(bytes))
116 | logs.SetLogger(logs.AdapterConsole,string(bytes))
117 |
118 | return
119 | }
120 |
121 |
122 | func loadSecConf() (err error){
123 | response, err := etcdClient.Get(context.Background(), secKillConf.EtcdConf.EtcdSecProductKey)
124 | if err != nil {
125 | logs.Debug("get [%s] from etcd failed, err:%v ",secKillConf.EtcdConf.EtcdSecProductKey,err)
126 | return
127 | }
128 | var secProductInfo []service.SecProductInfoConf
129 | for k,v := range response.Kvs{
130 | logs.Debug("key[%s] value[%s]",k,v)
131 | err = json.Unmarshal(v.Value, &secProductInfo)
132 | if err != nil {
133 | logs.Debug("Unmarshal sec producr info failed, err:%v ",err)
134 | return
135 | }
136 | logs.Debug("sec info conf is [%v]",secProductInfo)
137 | }
138 | updateSecProductInfo(secProductInfo)
139 | return
140 | }
141 |
142 | func InitSecKill()(err error){
143 | err = initLogger()
144 | if err != nil{
145 | fmt.Println("load logger failed,err: %v",err)
146 | return
147 | }
148 |
149 | //err = initRedis()
150 | //if err != nil{
151 | // fmt.Println("load redis failed,err: %v",err)
152 | // return
153 | //}
154 |
155 | err = initEtcd()
156 | if err != nil{
157 | fmt.Println("load etcd failed,err: %v",err)
158 | return
159 | }
160 |
161 | err = loadSecConf()
162 | if err != nil {
163 | logs.Error("load sec conf failed, err:%v", err)
164 | return
165 | }
166 | service.InitService(secKillConf)
167 |
168 | initSecProductWatcher()
169 |
170 | logs.Info("load sec success")
171 | return
172 | }
173 |
174 | func initSecProductWatcher(){
175 | go watchSecProductKey(secKillConf.EtcdConf.EtcdSecProductKey)
176 | }
177 |
178 | func watchSecProductKey(key string) {
179 | logs.Debug("begin watch key: %s",key)
180 |
181 | for {
182 | watchChan := etcdClient.Watch(context.Background(), key)
183 | var secProductInfo []service.SecProductInfoConf
184 | var getConfSucc = true
185 |
186 | for wresp := range watchChan {
187 | for _,ev := range wresp.Events {
188 | if ev.Type == mvccpb.DELETE{
189 | logs.Warn("key[%s]'s config deleted",key)
190 | continue
191 | }
192 | if ev.Type == mvccpb.PUT && string(ev.Kv.Key) == key {
193 | err := json.Unmarshal(ev.Kv.Value,&secProductInfo)
194 | if err != nil {
195 | logs.Error("key[%s],Unmarshal[%s] failed,err: %v",ev.Kv.Key,ev.Kv.Value,err)
196 | getConfSucc = false
197 | continue
198 | }
199 | }
200 | logs.Debug("get config from etcd,%s %q: %q\n",ev.Type,ev.Kv.Key,ev.Kv.Value)
201 | }
202 |
203 | if getConfSucc {
204 | logs.Debug("get config from etcd success, %v",secProductInfo)
205 | updateSecProductInfo(secProductInfo)
206 | }
207 | }
208 | }
209 | }
210 |
211 | func updateSecProductInfo(confs []service.SecProductInfoConf) {
212 | var tmp = make(map[int]*service.SecProductInfoConf,1024)
213 |
214 | for _,v :=range confs{
215 | product := v
216 | tmp[v.ProductId] = &product
217 | }
218 |
219 | secKillConf.RWSecProductLock.Lock()
220 | secKillConf.SecProductInfoMap = tmp
221 | secKillConf.RWSecProductLock.Unlock()
222 | }
--------------------------------------------------------------------------------
/video_server/api/dbops/api.go:
--------------------------------------------------------------------------------
1 | package dbops
2 |
3 | import (
4 | "database/sql"
5 | "golang-awesome/video_server/api/defs"
6 | "golang-awesome/video_server/api/utils"
7 | "log"
8 | "time"
9 | )
10 |
11 | func AddUserCredential(loginName string, pwd string) error {
12 | stmtIns, err := dbConn.Prepare("INSERT INTO users(login_name,pwd) values (?,?)")
13 | if err != nil {
14 | return err
15 | }
16 |
17 | _, err = stmtIns.Exec(loginName, pwd)
18 | if err != nil {
19 | return err
20 | }
21 | defer stmtIns.Close()
22 | return nil
23 | }
24 |
25 | func GetUserCredential(loginName string) (string, error) {
26 | stmtOut, err := dbConn.Prepare("select pwd from users where login_name = ?")
27 | if err != nil {
28 | log.Printf("%s", err)
29 | return "", err
30 | }
31 |
32 | var pwd string
33 | err = stmtOut.QueryRow(loginName).Scan(&pwd)
34 | if err != nil {
35 | return "", nil
36 | }
37 |
38 | defer stmtOut.Close()
39 | return pwd, nil
40 | }
41 |
42 | func DeleteUser(loginName, pwd string) (err error) {
43 | stmtDel, err := dbConn.Prepare("DELETE FROM users WHERE login_name = ? and pwd = ?")
44 | if err != nil {
45 | log.Printf("Delete user error: %s", err)
46 | return
47 | }
48 |
49 | _, err = stmtDel.Exec(loginName, pwd)
50 | if err != nil {
51 | log.Printf("Delete user error: %s", err)
52 | return
53 | }
54 |
55 | defer stmtDel.Close()
56 | return
57 | }
58 |
59 | func GetUser(loginName string) (user *defs.User, err error) {
60 | stmtOut, err := dbConn.Prepare("select id, pwd from users where login_name = ?")
61 | if err != nil {
62 | log.Printf("%s", err)
63 | return
64 | }
65 |
66 | var id int
67 | var pwd string
68 | err = stmtOut.QueryRow(loginName).Scan(&id, &pwd)
69 | if err != nil && err != sql.ErrNoRows {
70 | log.Printf("%s", err)
71 | return
72 | }
73 | if err == sql.ErrNoRows {
74 | return
75 | }
76 |
77 | user = &defs.User{Id: id, LoginName: loginName, Pwd: pwd}
78 | defer stmtOut.Close()
79 | return
80 | }
81 |
82 | func AddNewVideo(aid int, name string) (video *defs.VideoInfo, err error) {
83 | vid, err := utils.NewUUID()
84 | if err != nil {
85 | return
86 | }
87 | t := time.Now()
88 | ctime := t.Format("Jan 02 2006, 15:04:05")
89 | stmtIns, err := dbConn.Prepare(`insert into video_info
90 | (id, author_id, name, display_ctime) values (?,?,?,?)`)
91 | if err != nil {
92 | log.Printf("%s", err)
93 | return
94 | }
95 | _, err = stmtIns.Exec(vid, aid, name, ctime)
96 | if err != nil {
97 | return
98 | }
99 | video = &defs.VideoInfo{Id: vid, AuthorId: aid, Name: name, DisplayTime: ctime}
100 | defer stmtIns.Close()
101 | return
102 | }
103 |
104 | func GetVideoInfo(vid string) (video *defs.VideoInfo, err error) {
105 | stmtOut, err := dbConn.Prepare("select author_id, name, display_ctime from video_info where vid = ?")
106 | if err != nil {
107 | return
108 | }
109 |
110 | var aid int
111 | var dct, name string
112 | err = stmtOut.QueryRow(vid).Scan(&aid, &name, &dct)
113 | if err != nil && err != sql.ErrNoRows {
114 | return
115 | }
116 |
117 | if err == sql.ErrNoRows {
118 | return nil, nil
119 | }
120 | defer stmtOut.Close()
121 | video = &defs.VideoInfo{
122 | Id: vid,
123 | AuthorId: aid,
124 | Name: name,
125 | DisplayTime: dct,
126 | }
127 | return
128 | }
129 |
130 | func ListVideoInfo(uname string, from, to int)(videos []*defs.VideoInfo, err error){
131 | stmtOut, err := dbConn.Prepare(`select v.id, v.author_id, v.name,v.display_ctime from video_info v
132 | inner join users u on v.author_id = u.id where u.login_name = ? and v.create_time > FROM_UNIXTIME(?)
133 | and v.create_time <= FROM_UNIXTIME(?) order by v.create_time desc `)
134 | if err != nil {
135 | log.Printf("%s",err)
136 | return
137 | }
138 |
139 | rows, err := stmtOut.Query(uname, from, to)
140 | if err != nil {
141 | log.Printf("%s",err)
142 | return
143 | }
144 |
145 | for rows.Next() {
146 | var id, name, ctime string
147 | var aid int
148 | if err := rows.Scan(&id, &aid, &name, &ctime); err != nil {
149 | log.Printf("%s",err)
150 | return
151 | }
152 | video := &defs.VideoInfo{
153 | Id:id,
154 | AuthorId:aid,
155 | Name:name,
156 | DisplayTime:ctime,
157 | }
158 | videos = append(videos,video)
159 | }
160 | defer stmtOut.Close()
161 | return
162 | }
163 |
164 | func DeleteVideoInfo(vid string) (err error){
165 | stmtDel, err := dbConn.Prepare("delete from video_info where id = ?")
166 | if err != nil {
167 | log.Printf("%s",err)
168 | return
169 | }
170 |
171 | _, err = stmtDel.Exec(vid)
172 | if err != nil {
173 | log.Printf("%s",err)
174 | return
175 | }
176 |
177 | defer stmtDel.Close()
178 | return
179 | }
180 |
181 | func AddNewComments(vid string, aid int,content string) (err error){
182 | id, err := utils.NewUUID()
183 | if err != nil {
184 | log.Printf("%s",err)
185 | return
186 | }
187 |
188 | stmtIns, err := dbConn.Prepare("insert into comments(id, video_id, author_id, content) values (?,?,?,?)")
189 | if err != nil {
190 | log.Printf("%s",err)
191 | return
192 | }
193 |
194 | _, err = stmtIns.Exec(id,vid,aid,content)
195 | if err != nil {
196 | log.Printf("%s",err)
197 | return
198 | }
199 | defer stmtIns.Close()
200 | return
201 | }
202 |
203 | func ListComments(vid string, from ,to int)(comments []*defs.Comment, err error){
204 | stmtOut, err := dbConn.Prepare(`select c.id, u.login_name, c.content from comments c
205 | inner join users u on c.author_id = u.id where c.video_id = ? and c.time >
206 | FROM_UNIXTIME(?) and c.time <= FROM_UNIXTIME(?) order by c.time desc`)
207 |
208 | rows, err := stmtOut.Query(vid, from, to)
209 | if err != nil {
210 | log.Printf("%s",err)
211 | return
212 | }
213 |
214 | for rows.Next() {
215 | var id, name, content string
216 | if err := rows.Scan(&id, &name, &content); err != nil {
217 | return
218 | }
219 | comment := &defs.Comment{
220 | Id:id,
221 | VideoId:vid,
222 | Author:name,
223 | Content:content,
224 | }
225 | comments = append(comments,comment)
226 | }
227 | defer stmtOut.Close()
228 | return
229 | }
--------------------------------------------------------------------------------
/video_server/api/handlers.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/json"
5 | "github.com/julienschmidt/httprouter"
6 | "golang-awesome/video_server/api/dbops"
7 | "golang-awesome/video_server/api/defs"
8 | "golang-awesome/video_server/api/session"
9 | "golang-awesome/video_server/api/utils"
10 | "io/ioutil"
11 | "log"
12 | "net/http"
13 | )
14 |
15 | func CreateUser(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
16 | //首先从request里面读出create user 相关的body
17 | res, _ := ioutil.ReadAll(r.Body)
18 | ubody := &defs.UserCredential{}
19 |
20 | if err := json.Unmarshal(res, ubody); err != nil {
21 | sendErrorResponse(w, defs.ErrorRequestBodyParseFailed)
22 | return
23 | }
24 |
25 | if err := dbops.AddUserCredential(ubody.UserName, ubody.Pwd); err != nil {
26 | sendErrorResponse(w, defs.ErrorDBError)
27 | }
28 |
29 | id := session.GenerateNewSessionId(ubody.UserName)
30 | su := &defs.SignedUp{Success: true, SessionId: id}
31 |
32 | if resp, err := json.Marshal(su); err != nil {
33 | sendErrorResponse(w, defs.ErrorInternalFaults)
34 | return
35 | } else {
36 | sendNormalResponse(w, string(resp), 201)
37 | }
38 | }
39 |
40 | func Login(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
41 | res, _ := ioutil.ReadAll(r.Body)
42 | log.Printf("%s", res)
43 | ubody := &defs.UserCredential{}
44 |
45 | if err := json.Unmarshal(res, ubody); err != nil {
46 | log.Panicf("%s", err)
47 | sendErrorResponse(w, defs.ErrorRequestBodyParseFailed)
48 | return
49 | }
50 |
51 | uname := params.ByName("username")
52 | log.Panicf("login url name: %s", uname)
53 | log.Printf("login body name: %s", ubody.UserName)
54 | if uname != ubody.UserName {
55 | sendErrorResponse(w, defs.ErrorNotAuthUser)
56 | return
57 | }
58 |
59 | log.Printf("%s", ubody.UserName)
60 | pwd, err := dbops.GetUserCredential(ubody.UserName)
61 | log.Printf("login pwd: %s", pwd)
62 | log.Printf("login body pwd: %s", ubody.Pwd)
63 | if err != nil || len(pwd) == 0 || pwd != ubody.Pwd {
64 | sendErrorResponse(w, defs.ErrorNotAuthUser)
65 | return
66 | }
67 |
68 | id := session.GenerateNewSessionId(ubody.UserName)
69 | si := &defs.SignedIn{Success: true, SessionId: id}
70 | if resp, err := json.Marshal(si); err != nil {
71 | sendErrorResponse(w, defs.ErrorInternalFaults)
72 | } else {
73 | sendNormalResponse(w, string(resp), 200)
74 | }
75 | }
76 |
77 | func GetUserInfo(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
78 | if !ValidateUser(w, r) {
79 | log.Printf("unauthorized user\n")
80 | return
81 | }
82 |
83 | uname := p.ByName("username")
84 | u, err := dbops.GetUser(uname)
85 | if err != nil {
86 | log.Printf("error in GetUserInfo: %s", err)
87 | sendErrorResponse(w, defs.ErrorDBError)
88 | return
89 | }
90 |
91 | ui := defs.UserInfo{Id: u.Id}
92 | if resp, err := json.Marshal(ui); err != nil {
93 | sendErrorResponse(w, defs.ErrorInternalFaults)
94 | } else {
95 | sendNormalResponse(w, string(resp), 200)
96 | }
97 | }
98 |
99 | func AddNewVideo(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
100 | if !ValidateUser(w, r) {
101 | log.Printf("unauthorized user\n")
102 | return
103 | }
104 |
105 | res, _ := ioutil.ReadAll(r.Body)
106 | nvbody := &defs.NewVideo{}
107 | if err := json.Unmarshal(res, nvbody); err != nil {
108 | sendErrorResponse(w, defs.ErrorRequestBodyParseFailed)
109 | return
110 | }
111 |
112 | vi, err := dbops.AddNewVideo(nvbody.AuthorId, nvbody.Name)
113 | log.Printf("author id: %d, name: %s \n", nvbody.AuthorId, nvbody.Name)
114 | if err != nil {
115 | log.Printf("error in AddNewVideo: %s", err)
116 | sendErrorResponse(w, defs.ErrorDBError)
117 | return
118 | }
119 |
120 | if resp, err := json.Marshal(vi); err != nil {
121 | sendErrorResponse(w, defs.ErrorInternalFaults)
122 | } else {
123 | sendNormalResponse(w, string(resp), 201)
124 | }
125 | }
126 |
127 | func ListAllVideos(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
128 | if !ValidateUser(w, r) {
129 | log.Printf("unauthirized user\n")
130 | return
131 | }
132 |
133 | uname := p.ByName("username")
134 | vs, err := dbops.ListVideoInfo(uname, 0, utils.GetCurrentTimestampSec())
135 | if err != nil {
136 | log.Printf("error in ListAllVideos: %s", err)
137 | sendErrorResponse(w, defs.ErrorDBError)
138 | return
139 | }
140 |
141 | vsi := &defs.VideosInfo{Videos: vs}
142 | if resp, err := json.Marshal(vsi); err != nil {
143 | sendErrorResponse(w, defs.ErrorInternalFaults)
144 | } else {
145 | sendNormalResponse(w, string(resp), 200)
146 | }
147 | }
148 |
149 | func DeleteVideo(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
150 | if !ValidateUser(w, r) {
151 | return
152 | }
153 |
154 | vid := p.ByName("vid-id")
155 | err := dbops.DeleteVideoInfo(vid)
156 | if err != nil {
157 | log.Printf("error in DeleteVideo: %s", err)
158 | sendErrorResponse(w, defs.ErrorDBError)
159 | return
160 | }
161 |
162 | go utils.SendDeleteVideoRequest(vid)
163 | sendNormalResponse(w, "", 204)
164 | }
165 |
166 | func PostComment(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
167 | if !ValidateUser(w, r) {
168 | return
169 | }
170 | reqBody, _ := ioutil.ReadAll(r.Body)
171 | cbody := &defs.NewComment{}
172 | if err := json.Unmarshal(reqBody, cbody); err != nil {
173 | log.Printf("%s", err)
174 | sendErrorResponse(w, defs.ErrorRequestBodyParseFailed)
175 | return
176 | }
177 | vid := p.ByName("vid-id")
178 | if err := dbops.AddNewComments(vid, cbody.AuthorId, cbody.Content); err != nil {
179 | log.Printf("%s", err)
180 | sendErrorResponse(w, defs.ErrorDBError)
181 | } else {
182 | sendNormalResponse(w, "ok", 201)
183 | }
184 |
185 | }
186 |
187 | func ShowComments(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
188 | if !ValidateUser(w, r) {
189 | return
190 | }
191 |
192 | vid := p.ByName("vid-id")
193 | cm, err := dbops.ListComments(vid, 0, utils.GetCurrentTimestampSec())
194 | if err != nil {
195 | log.Printf("error in ShowComments: %s", err)
196 | sendErrorResponse(w, defs.ErrorDBError)
197 | return
198 | }
199 |
200 | cms := &defs.Comments{Comments: cm}
201 | if resp, err := json.Marshal(cms); err != nil {
202 | sendErrorResponse(w, defs.ErrorInternalFaults)
203 | } else {
204 | sendNormalResponse(w, string(resp), 200)
205 | }
206 | }
207 |
--------------------------------------------------------------------------------
/seckill/SecProxy/service/init.go:
--------------------------------------------------------------------------------
1 | package service
2 |
3 | import (
4 | "github.com/astaxie/beego/logs"
5 | "github.com/garyburd/redigo/redis"
6 | "strconv"
7 | "time"
8 | )
9 |
10 | func InitService(serviceConf *SecKillConf)(err error){
11 | secKillConf = serviceConf
12 |
13 | err = loadBalckList()
14 | if err != nil{
15 | logs.Error("load black list err: %v",err)
16 | return
17 | }
18 | logs.Debug("init service success, config:%v",serviceConf)
19 |
20 | err = initProxy2LayerRedis()
21 | if err != nil{
22 | logs.Error("load proxy2layer redis pool failed, err: %v",err)
23 | return
24 | }
25 |
26 | secKillConf.secLimitMgr = &SecLimitMgr{
27 | UserLimitMap:make(map[int]*Limit,10000),
28 | IpLimitMap: make(map[string]*Limit,10000),
29 | }
30 | secKillConf.SecReqChan = make(chan *SecRequest,secKillConf.SecReqChanSize)
31 | secKillConf.UserConnMap = make (map[string]chan *SecResult,10000)
32 |
33 | initRedisProcessFunc()
34 | return
35 | }
36 |
37 | func loadBalckList()(err error){
38 | secKillConf.ipBlackMap = make(map[string]bool,10000)
39 | secKillConf.idBlackMap = make(map[int]bool,10000)
40 |
41 | err = initBlackRedis()
42 |
43 | if err != nil{
44 | logs.Error("init black redis failed, err:%v",err)
45 | return
46 | }
47 |
48 | conn := secKillConf.blackRedisPool.Get()
49 | defer conn.Close()
50 |
51 | reply, err := conn.Do("hgetall", "id_black_list")
52 | idlist,err :=redis.Strings(reply,err)
53 | if err != nil {
54 | logs.Warn("hget all black id failed, err %v",err)
55 | return
56 | }
57 |
58 | for _,v :=range idlist{
59 | id, err := strconv.Atoi(v)
60 | if err != nil {
61 | logs.Warn("invalid user id [%v]",id)
62 | continue
63 | }
64 | secKillConf.idBlackMap[id] = true
65 | }
66 |
67 | reply, err = conn.Do("hgetall", "ip_black_list")
68 | iplist,err :=redis.Strings(reply,err)
69 | if err != nil {
70 | logs.Warn("hget all black ip failed, err %v",err)
71 | return
72 | }
73 |
74 | for _,v :=range iplist{
75 | secKillConf.ipBlackMap[v] = true
76 | }
77 |
78 | go syncIpBLackList()
79 | go syncIdBLackList()
80 | return
81 | }
82 |
83 | //用redis的BLPOP阻塞弹出,达到watch效果
84 | func syncIpBLackList(){
85 | var ipList []string
86 | lastTime := time.Now().Unix()
87 | for{
88 | conn := secKillConf.blackRedisPool.Get()
89 | defer conn.Close()
90 | replay,err := conn.Do("BLPOP","blackiplist",time.Second)
91 | ip,err :=redis.String(replay,err)
92 | if err != nil {
93 | continue
94 | }
95 |
96 | curTime := time.Now().Unix()
97 | ipList = append(ipList,ip)
98 |
99 | if len(ipList) >100 || curTime -lastTime >5{
100 | secKillConf.RWBlackLock.Lock()
101 | for _,v :=range ipList{
102 | secKillConf.ipBlackMap[v] = true
103 | }
104 | secKillConf.RWBlackLock.Unlock()
105 |
106 | lastTime = curTime
107 | logs.Info("sync ip list from redis success, ip[%v]",ipList)
108 | }
109 | }
110 | }
111 |
112 | func syncIdBLackList(){
113 | for{
114 | conn := secKillConf.blackRedisPool.Get()
115 | defer conn.Close()
116 | reply,err := conn.Do("BLPOP","blackidlist",time.Second)
117 | id,err := redis.Int(reply,err)
118 | if err != nil{
119 | continue
120 | }
121 | secKillConf.RWBlackLock.Lock()
122 | secKillConf.idBlackMap[id] = true
123 | secKillConf.RWBlackLock.Unlock()
124 | logs.Info("sync id list from redis success, ip[%v]",id)
125 | }
126 | }
127 |
128 | func initRedisProcessFunc(){
129 | for i :=0 ;i