├── .gitignore ├── LICENSE ├── README.md └── src ├── basic ├── cgo │ ├── cgo_demo.go │ └── lib │ │ ├── go_export.go │ │ ├── go_export_def.go │ │ ├── math.go │ │ ├── print.go │ │ └── rand.go ├── map1 │ ├── cmap.go │ ├── cmap_test.go │ ├── common.go │ ├── keys.go │ ├── keys_test.go │ ├── omap.go │ └── omap_test.go ├── pkginit │ └── initpkg_demo.go ├── prof │ ├── profiling.go │ └── profiling_test.go ├── seq.go ├── set │ ├── hash_set.go │ ├── hash_set_test.go │ ├── set.go │ └── set_test.go ├── set2.go └── set2_test.go ├── chan1 └── oneway │ └── phandler.go ├── cnet └── ctcp │ ├── base.go │ ├── tcp.go │ └── tcp_test.go ├── helper ├── ds │ └── showds.go └── pds │ └── showpds.go ├── loadgen ├── gen.go ├── gen_test.go ├── lib │ ├── base.go │ ├── caller.go │ └── tickets.go ├── log.go └── testhelper │ ├── comm.go │ └── server.go ├── logging ├── base.go ├── console_logger.go ├── log_manager.go ├── logger_test.go └── tag.go ├── multiproc ├── apipe │ └── apipe.go ├── npipe │ └── npipe.go ├── signal │ └── mysignal.go └── socket │ └── tcpsock.go ├── pkgtool ├── envir.go ├── envir_test.go ├── fpath.go ├── ipath.go ├── pnode.go ├── util.go └── util_test.go ├── sync1 ├── datafile1 │ └── datafile1.go ├── datafile2 │ └── datafile2.go ├── datafile3 │ └── datafile3.go └── pool │ └── pool_demo.go ├── testing ├── bmt │ └── bmt_test.go ├── ct │ └── ct_demo.go ├── et │ └── et_test.go └── rp │ └── rp_test.go └── webcrawler ├── analyzer ├── analyzer.go ├── parser.go └── pool.go ├── base ├── argument.go ├── data.go ├── error.go └── logger.go ├── demo └── demo.go ├── downloader ├── downloader.go └── pool.go ├── itempipeline ├── pipeline.go └── processor.go ├── middleware ├── chanman.go ├── id.go ├── pool.go └── stopsign.go ├── scheduler ├── cache.go ├── helper.go ├── scheduler.go └── summary.go └── tool ├── cookie └── cookiejar.go └── monitor.go /.gitignore: -------------------------------------------------------------------------------- 1 | bin**/* 2 | pkg**/* 3 | logs**/* 4 | pprof**/* 5 | _obj 6 | *.exe 7 | *.out 8 | *.di 9 | *.callgrind 10 | .DS_Store 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | goc2p 2 | ===== 3 | 4 | Go Concurrency Programming Project. 5 | 6 | An example project for book 'Go Programming & Concurrency in Practice' (《Go并发编程实战》). 7 | 8 | ------------------------------------------------ 9 | 10 | The sample project for the second edition of this book is [here](https://github.com/gopcp/example.v2). 11 | -------------------------------------------------------------------------------- /src/basic/cgo/cgo_demo.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | cgolib "basic/cgo/lib" 5 | "fmt" 6 | ) 7 | 8 | func main() { 9 | input := float32(2.33) 10 | output, err := cgolib.Sqrt(input) 11 | if err != nil { 12 | fmt.Errorf("Error: %s\n", err) 13 | } 14 | fmt.Printf("The square root of %f is %f.\n", input, output) 15 | 16 | cgolib.Print("ABC\n") 17 | 18 | cgolib.CallCFunc() 19 | } 20 | -------------------------------------------------------------------------------- /src/basic/cgo/lib/go_export.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | /* 4 | #include 5 | extern void CFunction1(); 6 | */ 7 | import "C" 8 | 9 | import "fmt" 10 | 11 | //export GoFunction1 12 | func GoFunction1() { 13 | fmt.Println("GoFunction1() is called.") 14 | } 15 | 16 | func CallCFunc() { 17 | C.CFunction1() 18 | } 19 | -------------------------------------------------------------------------------- /src/basic/cgo/lib/go_export_def.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | /* 4 | #include 5 | void CFunction1() { 6 | printf("CFunction1() is called.\n"); 7 | GoFunction1(); 8 | } 9 | */ 10 | import "C" 11 | -------------------------------------------------------------------------------- /src/basic/cgo/lib/math.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | /* 4 | #cgo LDFLAGS: -lm 5 | #include 6 | */ 7 | import "C" 8 | 9 | func Sqrt(p float32) (float32, error) { 10 | result, err := C.sqrt(C.double(p)) 11 | return float32(result), err 12 | } 13 | -------------------------------------------------------------------------------- /src/basic/cgo/lib/print.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | /* 4 | #include 5 | #include 6 | 7 | void myprint(char* s) { 8 | printf("%s", s); 9 | } 10 | */ 11 | import "C" 12 | import "unsafe" 13 | 14 | func Print(s string) { 15 | cs := C.CString(s) 16 | defer C.free(unsafe.Pointer(cs)) 17 | C.myprint(cs) 18 | } 19 | -------------------------------------------------------------------------------- /src/basic/cgo/lib/rand.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | /* 4 | #include 5 | */ 6 | import "C" 7 | 8 | func Random() int { 9 | return int(C.rand()) 10 | } 11 | 12 | func Seed(i int) { 13 | C.srand(C.uint(i)) 14 | } 15 | -------------------------------------------------------------------------------- /src/basic/map1/cmap.go: -------------------------------------------------------------------------------- 1 | package map1 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "reflect" 7 | "sync" 8 | ) 9 | 10 | type ConcurrentMap interface { 11 | GenericMap 12 | } 13 | 14 | type myConcurrentMap struct { 15 | m map[interface{}]interface{} 16 | keyType reflect.Type 17 | elemType reflect.Type 18 | rwmutex sync.RWMutex 19 | } 20 | 21 | func (cmap *myConcurrentMap) Get(key interface{}) interface{} { 22 | cmap.rwmutex.RLock() 23 | defer cmap.rwmutex.RUnlock() 24 | return cmap.m[key] 25 | } 26 | 27 | func (cmap *myConcurrentMap) isAcceptablePair(k, e interface{}) bool { 28 | if k == nil || reflect.TypeOf(k) != cmap.keyType { 29 | return false 30 | } 31 | if e == nil || reflect.TypeOf(e) != cmap.elemType { 32 | return false 33 | } 34 | return true 35 | } 36 | 37 | func (cmap *myConcurrentMap) Put(key interface{}, elem interface{}) (interface{}, bool) { 38 | if !cmap.isAcceptablePair(key, elem) { 39 | return nil, false 40 | } 41 | cmap.rwmutex.Lock() 42 | defer cmap.rwmutex.Unlock() 43 | oldElem := cmap.m[key] 44 | cmap.m[key] = elem 45 | return oldElem, true 46 | } 47 | 48 | func (cmap *myConcurrentMap) Remove(key interface{}) interface{} { 49 | cmap.rwmutex.Lock() 50 | defer cmap.rwmutex.Unlock() 51 | oldElem := cmap.m[key] 52 | delete(cmap.m, key) 53 | return oldElem 54 | } 55 | 56 | func (cmap *myConcurrentMap) Clear() { 57 | cmap.rwmutex.Lock() 58 | defer cmap.rwmutex.Unlock() 59 | cmap.m = make(map[interface{}]interface{}) 60 | } 61 | 62 | func (cmap *myConcurrentMap) Len() int { 63 | cmap.rwmutex.RLock() 64 | defer cmap.rwmutex.RUnlock() 65 | return len(cmap.m) 66 | } 67 | 68 | func (cmap *myConcurrentMap) Contains(key interface{}) bool { 69 | cmap.rwmutex.RLock() 70 | defer cmap.rwmutex.RUnlock() 71 | _, ok := cmap.m[key] 72 | return ok 73 | } 74 | 75 | func (cmap *myConcurrentMap) Keys() []interface{} { 76 | cmap.rwmutex.RLock() 77 | defer cmap.rwmutex.RUnlock() 78 | initialLen := len(cmap.m) 79 | keys := make([]interface{}, initialLen) 80 | index := 0 81 | for k, _ := range cmap.m { 82 | keys[index] = k 83 | index++ 84 | } 85 | return keys 86 | } 87 | 88 | func (cmap *myConcurrentMap) Elems() []interface{} { 89 | cmap.rwmutex.RLock() 90 | defer cmap.rwmutex.RUnlock() 91 | initialLen := len(cmap.m) 92 | elems := make([]interface{}, initialLen) 93 | index := 0 94 | for _, v := range cmap.m { 95 | elems[index] = v 96 | index++ 97 | } 98 | return elems 99 | } 100 | 101 | func (cmap *myConcurrentMap) ToMap() map[interface{}]interface{} { 102 | cmap.rwmutex.RLock() 103 | defer cmap.rwmutex.RUnlock() 104 | replica := make(map[interface{}]interface{}) 105 | for k, v := range cmap.m { 106 | replica[k] = v 107 | } 108 | return replica 109 | } 110 | 111 | func (cmap *myConcurrentMap) KeyType() reflect.Type { 112 | return cmap.keyType 113 | } 114 | 115 | func (cmap *myConcurrentMap) ElemType() reflect.Type { 116 | return cmap.elemType 117 | } 118 | 119 | func (cmap *myConcurrentMap) String() string { 120 | var buf bytes.Buffer 121 | buf.WriteString("ConcurrentMap<") 122 | buf.WriteString(cmap.keyType.Kind().String()) 123 | buf.WriteString(",") 124 | buf.WriteString(cmap.elemType.Kind().String()) 125 | buf.WriteString(">{") 126 | first := true 127 | for k, v := range cmap.m { 128 | if first { 129 | first = false 130 | } else { 131 | buf.WriteString(" ") 132 | } 133 | buf.WriteString(fmt.Sprintf("%v", k)) 134 | buf.WriteString(":") 135 | buf.WriteString(fmt.Sprintf("%v", v)) 136 | } 137 | buf.WriteString("}") 138 | return buf.String() 139 | } 140 | 141 | func NewConcurrentMap(keyType, elemType reflect.Type) ConcurrentMap { 142 | return &myConcurrentMap{ 143 | keyType: keyType, 144 | elemType: elemType, 145 | m: make(map[interface{}]interface{})} 146 | } 147 | -------------------------------------------------------------------------------- /src/basic/map1/cmap_test.go: -------------------------------------------------------------------------------- 1 | package map1 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "reflect" 7 | "runtime/debug" 8 | "testing" 9 | ) 10 | 11 | func testConcurrentMap( 12 | t *testing.T, 13 | newConcurrentMap func() ConcurrentMap, 14 | genKey func() interface{}, 15 | genElem func() interface{}, 16 | keyKind reflect.Kind, 17 | elemKind reflect.Kind) { 18 | mapType := fmt.Sprintf("ConcurrentMap", keyKind, elemKind) 19 | defer func() { 20 | if err := recover(); err != nil { 21 | debug.PrintStack() 22 | t.Errorf("Fatal Error: %s: %s\n", mapType, err) 23 | } 24 | }() 25 | t.Logf("Starting Test%s...", mapType) 26 | 27 | // Basic 28 | cmap := newConcurrentMap() 29 | expectedLen := 0 30 | if cmap.Len() != expectedLen { 31 | t.Errorf("ERROR: The length of %s value %d is not %d!\n", 32 | mapType, cmap.Len(), expectedLen) 33 | t.FailNow() 34 | } 35 | expectedLen = 5 36 | testMap := make(map[interface{}]interface{}, expectedLen) 37 | var invalidKey interface{} 38 | for i := 0; i < expectedLen; i++ { 39 | key := genKey() 40 | testMap[key] = genElem() 41 | if invalidKey == nil { 42 | invalidKey = key 43 | } 44 | } 45 | for key, elem := range testMap { 46 | oldElem, ok := cmap.Put(key, elem) 47 | if !ok { 48 | t.Errorf("ERROR: Put (%v, %v) to %s value %d is failing!\n", 49 | key, elem, mapType, cmap) 50 | t.FailNow() 51 | } 52 | if oldElem != nil { 53 | t.Errorf("ERROR: Already had a (%v, %v) in %s value %d!\n", 54 | key, elem, mapType, cmap) 55 | t.FailNow() 56 | } 57 | t.Logf("Put (%v, %v) to the %s value %v.", 58 | key, elem, mapType, cmap) 59 | } 60 | if cmap.Len() != expectedLen { 61 | t.Errorf("ERROR: The length of %s value %d is not %d!\n", 62 | mapType, cmap.Len(), expectedLen) 63 | t.FailNow() 64 | } 65 | for key, elem := range testMap { 66 | contains := cmap.Contains(key) 67 | if !contains { 68 | t.Errorf("ERROR: The %s value %v do not contains %v!", 69 | mapType, cmap, key) 70 | t.FailNow() 71 | } 72 | actualElem := cmap.Get(key) 73 | if actualElem == nil { 74 | t.Errorf("ERROR: The %s value %v do not contains %v!", 75 | mapType, cmap, key) 76 | t.FailNow() 77 | } 78 | t.Logf("The %s value %v contains key %v.", mapType, cmap, key) 79 | if actualElem != elem { 80 | t.Errorf("ERROR: The element of %s value %v with key %v do not equals %v!\n", 81 | mapType, actualElem, key, elem) 82 | t.FailNow() 83 | } 84 | t.Logf("The element of %s value %v to key %v is %v.", 85 | mapType, cmap, key, actualElem) 86 | } 87 | oldElem := cmap.Remove(invalidKey) 88 | if oldElem == nil { 89 | t.Errorf("ERROR: Remove %v from %s value %d is failing!\n", 90 | invalidKey, mapType, cmap) 91 | t.FailNow() 92 | } 93 | t.Logf("Removed (%v, %v) from the %s value %v.", 94 | invalidKey, oldElem, mapType, cmap) 95 | delete(testMap, invalidKey) 96 | 97 | // Type 98 | actualElemType := cmap.ElemType() 99 | if actualElemType == nil { 100 | t.Errorf("ERROR: The element type of %s value is nil!\n", 101 | mapType) 102 | t.FailNow() 103 | } 104 | actualElemKind := actualElemType.Kind() 105 | if actualElemKind != elemKind { 106 | t.Errorf("ERROR: The element type of %s value %s is not %s!\n", 107 | mapType, actualElemKind, elemKind) 108 | t.FailNow() 109 | } 110 | t.Logf("The element type of %s value %v is %s.", 111 | mapType, cmap, actualElemKind) 112 | actualKeyKind := cmap.KeyType().Kind() 113 | if actualKeyKind != elemKind { 114 | t.Errorf("ERROR: The key type of %s value %s is not %s!\n", 115 | mapType, actualKeyKind, keyKind) 116 | t.FailNow() 117 | } 118 | t.Logf("The key type of %s value %v is %s.", 119 | mapType, cmap, actualKeyKind) 120 | 121 | // Export 122 | keys := cmap.Keys() 123 | elems := cmap.Elems() 124 | pairs := cmap.ToMap() 125 | for key, elem := range testMap { 126 | var hasKey bool 127 | for _, k := range keys { 128 | if k == key { 129 | hasKey = true 130 | } 131 | } 132 | if !hasKey { 133 | t.Errorf("ERROR: The keys of %s value %v do not contains %v!\n", 134 | mapType, cmap, key) 135 | t.FailNow() 136 | } 137 | var hasElem bool 138 | for _, e := range elems { 139 | if e == elem { 140 | hasElem = true 141 | } 142 | } 143 | if !hasElem { 144 | t.Errorf("ERROR: The elems of %s value %v do not contains %v!\n", 145 | mapType, cmap, elem) 146 | t.FailNow() 147 | } 148 | var hasPair bool 149 | for k, e := range pairs { 150 | if k == key && e == elem { 151 | hasPair = true 152 | } 153 | } 154 | if !hasPair { 155 | t.Errorf("ERROR: The elems of %s value %v do not contains (%v, %v)!\n", 156 | mapType, cmap, key, elem) 157 | t.FailNow() 158 | } 159 | } 160 | 161 | // Clear 162 | cmap.Clear() 163 | if cmap.Len() != 0 { 164 | t.Errorf("ERROR: Clear %s value %d is failing!\n", 165 | mapType, cmap) 166 | t.FailNow() 167 | } 168 | t.Logf("The %s value %v has been cleared.", mapType, cmap) 169 | } 170 | 171 | func TestInt64Cmap(t *testing.T) { 172 | newCmap := func() ConcurrentMap { 173 | keyType := reflect.TypeOf(int64(2)) 174 | elemType := keyType 175 | return NewConcurrentMap(keyType, elemType) 176 | } 177 | testConcurrentMap( 178 | t, 179 | newCmap, 180 | func() interface{} { return rand.Int63n(1000) }, 181 | func() interface{} { return rand.Int63n(1000) }, 182 | reflect.Int64, 183 | reflect.Int64) 184 | } 185 | 186 | func TestFloat64Cmap(t *testing.T) { 187 | newCmap := func() ConcurrentMap { 188 | keyType := reflect.TypeOf(float64(2)) 189 | elemType := keyType 190 | return NewConcurrentMap(keyType, elemType) 191 | } 192 | testConcurrentMap( 193 | t, 194 | newCmap, 195 | func() interface{} { return rand.Float64() }, 196 | func() interface{} { return rand.Float64() }, 197 | reflect.Float64, 198 | reflect.Float64) 199 | } 200 | 201 | func TestStringCmap(t *testing.T) { 202 | newCmap := func() ConcurrentMap { 203 | keyType := reflect.TypeOf(string(2)) 204 | elemType := keyType 205 | return NewConcurrentMap(keyType, elemType) 206 | } 207 | testConcurrentMap( 208 | t, 209 | newCmap, 210 | func() interface{} { return genRandString() }, 211 | func() interface{} { return genRandString() }, 212 | reflect.String, 213 | reflect.String) 214 | } 215 | 216 | func BenchmarkConcurrentMap(b *testing.B) { 217 | keyType := reflect.TypeOf(int32(2)) 218 | elemType := keyType 219 | cmap := NewConcurrentMap(keyType, elemType) 220 | var key, elem int32 221 | fmt.Printf("N=%d.\n", b.N) 222 | b.ResetTimer() 223 | for i := 0; i < b.N; i++ { 224 | b.StopTimer() 225 | seed := int32(i) 226 | key = seed 227 | elem = seed << 10 228 | b.StartTimer() 229 | cmap.Put(key, elem) 230 | _ = cmap.Get(key) 231 | b.StopTimer() 232 | b.SetBytes(8) 233 | b.StartTimer() 234 | } 235 | ml := cmap.Len() 236 | b.StopTimer() 237 | mapType := fmt.Sprintf("ConcurrentMap<%s, %s>", 238 | keyType.Kind().String(), elemType.Kind().String()) 239 | b.Logf("The length of %s value is %d.\n", mapType, ml) 240 | b.StartTimer() 241 | } 242 | 243 | func BenchmarkMap(b *testing.B) { 244 | keyType := reflect.TypeOf(int32(2)) 245 | elemType := keyType 246 | imap := make(map[interface{}]interface{}) 247 | var key, elem int32 248 | fmt.Printf("N=%d.\n", b.N) 249 | b.ResetTimer() 250 | for i := 0; i < b.N; i++ { 251 | b.StopTimer() 252 | seed := int32(i) 253 | key = seed 254 | elem = seed << 10 255 | b.StartTimer() 256 | imap[key] = elem 257 | b.StopTimer() 258 | _ = imap[key] 259 | b.StopTimer() 260 | b.SetBytes(8) 261 | b.StartTimer() 262 | } 263 | ml := len(imap) 264 | b.StopTimer() 265 | mapType := fmt.Sprintf("Map<%s, %s>", 266 | keyType.Kind().String(), elemType.Kind().String()) 267 | b.Logf("The length of %s value is %d.\n", mapType, ml) 268 | b.StartTimer() 269 | } 270 | -------------------------------------------------------------------------------- /src/basic/map1/common.go: -------------------------------------------------------------------------------- 1 | package map1 2 | 3 | import ( 4 | "reflect" 5 | ) 6 | 7 | // 泛化的Map的接口类型 8 | type GenericMap interface { 9 | // 获取给定键值对应的元素值。若没有对应元素值则返回nil。 10 | Get(key interface{}) interface{} 11 | // 添加键值对，并返回与给定键值对应的旧的元素值。若没有旧元素值则返回(nil, true)。 12 | Put(key interface{}, elem interface{}) (interface{}, bool) 13 | // 删除与给定键值对应的键值对，并返回旧的元素值。若没有旧元素值则返回nil。 14 | Remove(key interface{}) interface{} 15 | // 清除所有的键值对。 16 | Clear() 17 | // 获取键值对的数量。 18 | Len() int 19 | // 判断是否包含给定的键值。 20 | Contains(key interface{}) bool 21 | // 获取已排序的键值所组成的切片值。 22 | Keys() []interface{} 23 | // 获取已排序的元素值所组成的切片值。 24 | Elems() []interface{} 25 | // 获取已包含的键值对所组成的字典值。 26 | ToMap() map[interface{}]interface{} 27 | // 获取键的类型。 28 | KeyType() reflect.Type 29 | // 获取元素的类型。 30 | ElemType() reflect.Type 31 | } 32 | -------------------------------------------------------------------------------- /src/basic/map1/keys.go: -------------------------------------------------------------------------------- 1 | package map1 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "reflect" 7 | "sort" 8 | ) 9 | 10 | type CompareFunction func(interface{}, interface{}) int8 11 | 12 | type Keys interface { 13 | sort.Interface 14 | Add(k interface{}) bool 15 | Remove(k interface{}) bool 16 | Clear() 17 | Get(index int) interface{} 18 | GetAll() []interface{} 19 | Search(k interface{}) (index int, contains bool) 20 | CompareFunc() CompareFunction 21 | ElemType() reflect.Type 22 | } 23 | 24 | // compareFunc的结果值： 25 | // 小于0: 第一个参数小于第二个参数 26 | // 等于0: 第一个参数等于第二个参数 27 | // 大于1: 第一个参数大于第二个参数 28 | type myKeys struct { 29 | container []interface{} 30 | compareFunc CompareFunction 31 | elemType reflect.Type 32 | } 33 | 34 | func (keys *myKeys) Len() int { 35 | return len(keys.container) 36 | } 37 | 38 | func (keys *myKeys) Less(i, j int) bool { 39 | return keys.compareFunc(keys.container[i], keys.container[j]) == -1 40 | } 41 | 42 | func (keys *myKeys) Swap(i, j int) { 43 | keys.container[i], keys.container[j] = keys.container[j], keys.container[i] 44 | } 45 | 46 | func (keys *myKeys) isAcceptableElem(k interface{}) bool { 47 | if k == nil { 48 | return false 49 | } 50 | if reflect.TypeOf(k) != keys.elemType { 51 | return false 52 | } 53 | return true 54 | } 55 | 56 | func (keys *myKeys) Add(k interface{}) bool { 57 | ok := keys.isAcceptableElem(k) 58 | if !ok { 59 | return false 60 | } 61 | keys.container = append(keys.container, k) 62 | sort.Sort(keys) 63 | return true 64 | } 65 | 66 | func (keys *myKeys) Remove(k interface{}) bool { 67 | index, contains := keys.Search(k) 68 | if !contains { 69 | return false 70 | } 71 | keys.container = append(keys.container[0:index], keys.container[index+1:]...) 72 | return true 73 | } 74 | 75 | func (keys *myKeys) Clear() { 76 | keys.container = make([]interface{}, 0) 77 | } 78 | 79 | func (keys *myKeys) Get(index int) interface{} { 80 | if index >= keys.Len() { 81 | return nil 82 | } 83 | return keys.container[index] 84 | } 85 | 86 | func (keys *myKeys) GetAll() []interface{} { 87 | initialLen := len(keys.container) 88 | snapshot := make([]interface{}, initialLen) 89 | actualLen := 0 90 | for _, key := range keys.container { 91 | if actualLen < initialLen { 92 | snapshot[actualLen] = key 93 | } else { 94 | snapshot = append(snapshot, key) 95 | } 96 | actualLen++ 97 | } 98 | if actualLen < initialLen { 99 | snapshot = snapshot[:actualLen] 100 | } 101 | return snapshot 102 | } 103 | 104 | func (keys *myKeys) Search(k interface{}) (index int, contains bool) { 105 | ok := keys.isAcceptableElem(k) 106 | if !ok { 107 | return 108 | } 109 | index = sort.Search( 110 | keys.Len(), 111 | func(i int) bool { return keys.compareFunc(keys.container[i], k) >= 0 }) 112 | if index < keys.Len() && keys.container[index] == k { 113 | contains = true 114 | } 115 | return 116 | } 117 | 118 | func (keys *myKeys) ElemType() reflect.Type { 119 | return keys.elemType 120 | } 121 | 122 | func (keys *myKeys) CompareFunc() CompareFunction { 123 | return keys.compareFunc 124 | } 125 | 126 | func (keys *myKeys) String() string { 127 | var buf bytes.Buffer 128 | buf.WriteString("Keys<") 129 | buf.WriteString(keys.elemType.Kind().String()) 130 | buf.WriteString(">{") 131 | first := true 132 | buf.WriteString("[") 133 | for _, key := range keys.container { 134 | if first { 135 | first = false 136 | } else { 137 | buf.WriteString(" ") 138 | } 139 | buf.WriteString(fmt.Sprintf("%v", key)) 140 | } 141 | buf.WriteString("]") 142 | buf.WriteString("}") 143 | return buf.String() 144 | } 145 | 146 | func NewKeys( 147 | compareFunc func(interface{}, interface{}) int8, 148 | elemType reflect.Type) Keys { 149 | return &myKeys{ 150 | container: make([]interface{}, 0), 151 | compareFunc: compareFunc, 152 | elemType: elemType, 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /src/basic/map1/keys_test.go: -------------------------------------------------------------------------------- 1 | package map1 2 | 3 | import ( 4 | "bytes" 5 | "math/rand" 6 | "reflect" 7 | "runtime/debug" 8 | "sort" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func testKeys( 14 | t *testing.T, 15 | newKeys func() Keys, 16 | genKey func() interface{}, 17 | elemKind reflect.Kind) { 18 | defer func() { 19 | if err := recover(); err != nil { 20 | debug.PrintStack() 21 | t.Errorf("Fatal Error: Keys(%s): %s\n", elemKind, err) 22 | } 23 | }() 24 | t.Logf("Starting TestKeys<%s>...", elemKind) 25 | keys := newKeys() 26 | expectedLen := 0 27 | if keys.Len() != expectedLen { 28 | t.Errorf("ERROR: The length of Keys(%s) value %d is not %d!\n", 29 | elemKind, keys.Len(), expectedLen) 30 | t.FailNow() 31 | } 32 | expectedLen = 5 33 | testKeys := make([]interface{}, expectedLen) 34 | for i := 0; i < expectedLen; i++ { 35 | testKeys[i] = genKey() 36 | } 37 | for _, key := range testKeys { 38 | result := keys.Add(key) 39 | if !result { 40 | t.Errorf("ERROR: Add %v to Keys(%s) value %d is failing!\n", 41 | key, elemKind, keys) 42 | t.FailNow() 43 | } 44 | t.Logf("Added %v to the Keys(%s) value %v.", key, elemKind, keys) 45 | } 46 | if keys.Len() != expectedLen { 47 | t.Errorf("ERROR: The length of Keys(%s) value %d is not %d!\n", 48 | elemKind, keys.Len(), expectedLen) 49 | t.FailNow() 50 | } 51 | for _, key := range testKeys { 52 | index, contains := keys.Search(key) 53 | if !contains { 54 | t.Errorf("ERROR: The Keys(%s) value %v do not contains %v!", 55 | elemKind, keys, key) 56 | t.FailNow() 57 | } 58 | t.Logf("The Keys(%s) value %v contains key %v.", elemKind, keys, key) 59 | actualElem := keys.Get(index) 60 | if actualElem != key { 61 | t.Errorf("ERROR: The element of Keys(%s) value %v with index %d do not equals %v!\n", 62 | elemKind, actualElem, index, key) 63 | t.FailNow() 64 | } 65 | t.Logf("The element of Keys(%s) value %v with index %d is %v.", 66 | elemKind, keys, index, actualElem) 67 | } 68 | invalidElem := testKeys[len(testKeys)/2] 69 | result := keys.Remove(invalidElem) 70 | if !result { 71 | t.Errorf("ERROR: Remove %v from Keys(%s) value %d is failing!\n", 72 | invalidElem, elemKind, keys) 73 | t.FailNow() 74 | } 75 | t.Logf("Removed %v from the Keys(%s) value %v.", invalidElem, elemKind, keys) 76 | if !sort.IsSorted(keys) { 77 | t.Errorf("ERROR: The Keys(%s) value %v is not sorted yet?!\n", 78 | elemKind, keys) 79 | t.FailNow() 80 | } 81 | t.Logf("The Keys(%s) value %v is sorted.", elemKind, keys) 82 | actualElemType := keys.ElemType() 83 | if actualElemType == nil { 84 | t.Errorf("ERROR: The element type of Keys(%s) value is nil!\n", 85 | elemKind) 86 | t.FailNow() 87 | } 88 | actualElemKind := actualElemType.Kind() 89 | if actualElemKind != elemKind { 90 | t.Errorf("ERROR: The element type of Keys(%s) value %s is not %s!\n", 91 | elemKind, actualElemKind, elemKind) 92 | t.FailNow() 93 | } 94 | t.Logf("The element type of Keys(%s) value %v is %s.", elemKind, keys, actualElemKind) 95 | currCompFunc := keys.CompareFunc() 96 | if currCompFunc == nil { 97 | t.Errorf("ERROR: The compare function of Keys(%s) value is nil!\n", 98 | elemKind) 99 | t.FailNow() 100 | } 101 | keys.Clear() 102 | if keys.Len() != 0 { 103 | t.Errorf("ERROR: Clear Keys(%s) value %d is failing!\n", 104 | elemKind, keys) 105 | t.FailNow() 106 | } 107 | t.Logf("The Keys(%s) value %v have been cleared.", elemKind, keys) 108 | } 109 | 110 | func TestInt64Keys(t *testing.T) { 111 | testKeys(t, 112 | func() Keys { 113 | //return NewKeys( 114 | //func(e1 interface{}, e2 interface{}) int8 { 115 | // k1 := e1.(int64) 116 | // k2 := e2.(int64) 117 | // if k1 < k2 { 118 | // return -1 119 | // } else if k1 > k2 { 120 | // return 1 121 | // } else { 122 | // return 0 123 | // } 124 | //}, 125 | //reflect.TypeOf(int64(1))) 126 | int64Keys := &myKeys{ 127 | container: make([]interface{}, 0), 128 | compareFunc: func(e1 interface{}, e2 interface{}) int8 { 129 | k1 := e1.(int64) 130 | k2 := e2.(int64) 131 | if k1 < k2 { 132 | return -1 133 | } else if k1 > k2 { 134 | return 1 135 | } else { 136 | return 0 137 | } 138 | }, 139 | elemType: reflect.TypeOf(int64(1))} 140 | return int64Keys 141 | }, 142 | func() interface{} { return rand.Int63n(1000) }, 143 | reflect.Int64) 144 | } 145 | 146 | func TestFloat64Keys(t *testing.T) { 147 | testKeys(t, 148 | func() Keys { 149 | return NewKeys( 150 | func(e1 interface{}, e2 interface{}) int8 { 151 | k1 := e1.(float64) 152 | k2 := e2.(float64) 153 | if k1 < k2 { 154 | return -1 155 | } else if k1 > k2 { 156 | return 1 157 | } else { 158 | return 0 159 | } 160 | }, 161 | reflect.TypeOf(float64(1))) 162 | }, 163 | func() interface{} { return rand.Float64() }, 164 | reflect.Float64) 165 | } 166 | 167 | func TestStringKeys(t *testing.T) { 168 | testKeys(t, 169 | func() Keys { 170 | return NewKeys( 171 | func(e1 interface{}, e2 interface{}) int8 { 172 | k1 := e1.(string) 173 | k2 := e2.(string) 174 | if k1 < k2 { 175 | return -1 176 | } else if k1 > k2 { 177 | return 1 178 | } else { 179 | return 0 180 | } 181 | }, 182 | reflect.TypeOf(string(1))) 183 | }, 184 | func() interface{} { return genRandString() }, 185 | reflect.String) 186 | } 187 | 188 | func genRandString() string { 189 | var buff bytes.Buffer 190 | var prev string 191 | var curr string 192 | for i := 0; buff.Len() < 3; i++ { 193 | curr = string(genRandAZAscii()) 194 | if curr == prev { 195 | continue 196 | } 197 | prev = curr 198 | buff.WriteString(curr) 199 | } 200 | return buff.String() 201 | } 202 | 203 | func genRandAZAscii() int { 204 | min := 65 // A 205 | max := 90 // Z 206 | rand.Seed(time.Now().UnixNano()) 207 | return min + rand.Intn(max-min) 208 | } 209 | -------------------------------------------------------------------------------- /src/basic/map1/omap.go: -------------------------------------------------------------------------------- 1 | package map1 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "reflect" 7 | ) 8 | 9 | // 有序的Map的接口类型。 10 | type OrderedMap interface { 11 | GenericMap // 泛化的Map接口 12 | // 获取第一个键值。若无任何键值对则返回nil。 13 | FirstKey() interface{} 14 | // 获取最后一个键值。若无任何键值对则返回nil。 15 | LastKey() interface{} 16 | // 获取由小于键值toKey的键值所对应的键值对组成的OrderedMap类型值。 17 | HeadMap(toKey interface{}) OrderedMap 18 | // 获取由小于键值toKey且大于等于键值fromKey的键值所对应的键值对组成的OrderedMap类型值。 19 | SubMap(fromKey interface{}, toKey interface{}) OrderedMap 20 | // 获取由大于等于键值fromKey的键值所对应的键值对组成的OrderedMap类型值。 21 | TailMap(fromKey interface{}) OrderedMap 22 | } 23 | 24 | type myOrderedMap struct { 25 | keys Keys 26 | elemType reflect.Type 27 | m map[interface{}]interface{} 28 | } 29 | 30 | func (omap *myOrderedMap) Get(key interface{}) interface{} { 31 | return omap.m[key] 32 | } 33 | 34 | func (omap *myOrderedMap) isAcceptableElem(e interface{}) bool { 35 | if e == nil { 36 | return false 37 | } 38 | if reflect.TypeOf(e) != omap.elemType { 39 | return false 40 | } 41 | return true 42 | } 43 | 44 | func (omap *myOrderedMap) Put(key interface{}, elem interface{}) (interface{}, bool) { 45 | if !omap.isAcceptableElem(elem) { 46 | return nil, false 47 | } 48 | oldElem, ok := omap.m[key] 49 | omap.m[key] = elem 50 | if !ok { 51 | omap.keys.Add(key) 52 | } 53 | return oldElem, true 54 | } 55 | 56 | func (omap *myOrderedMap) Remove(key interface{}) interface{} { 57 | oldElem, ok := omap.m[key] 58 | delete(omap.m, key) 59 | if ok { 60 | omap.keys.Remove(key) 61 | } 62 | return oldElem 63 | } 64 | 65 | func (omap *myOrderedMap) Clear() { 66 | omap.m = make(map[interface{}]interface{}) 67 | omap.keys.Clear() 68 | } 69 | 70 | // 获取键值对的数量 71 | func (omap *myOrderedMap) Len() int { 72 | return len(omap.m) 73 | } 74 | 75 | func (omap *myOrderedMap) Contains(key interface{}) bool { 76 | _, ok := omap.m[key] 77 | return ok 78 | } 79 | 80 | func (omap *myOrderedMap) FirstKey() interface{} { 81 | if omap.Len() == 0 { 82 | return nil 83 | } 84 | return omap.keys.Get(0) 85 | } 86 | 87 | func (omap *myOrderedMap) LastKey() interface{} { 88 | length := omap.Len() 89 | if length == 0 { 90 | return nil 91 | } 92 | return omap.keys.Get(length - 1) 93 | } 94 | 95 | func (omap *myOrderedMap) SubMap(fromKey interface{}, toKey interface{}) OrderedMap { 96 | newOmap := &myOrderedMap{ 97 | keys: NewKeys(omap.keys.CompareFunc(), omap.keys.ElemType()), 98 | elemType: omap.elemType, 99 | m: make(map[interface{}]interface{})} 100 | omapLen := omap.Len() 101 | if omapLen == 0 { 102 | return newOmap 103 | } 104 | beginIndex, contains := omap.keys.Search(fromKey) 105 | if !contains { 106 | beginIndex = 0 107 | } 108 | endIndex, contains := omap.keys.Search(toKey) 109 | if !contains { 110 | endIndex = omapLen 111 | } 112 | var key, elem interface{} 113 | for i := beginIndex; i < endIndex; i++ { 114 | key = omap.keys.Get(i) 115 | elem = omap.m[key] 116 | newOmap.Put(key, elem) 117 | } 118 | return newOmap 119 | } 120 | 121 | func (omap *myOrderedMap) HeadMap(toKey interface{}) OrderedMap { 122 | return omap.SubMap(nil, toKey) 123 | } 124 | 125 | func (omap *myOrderedMap) TailMap(fromKey interface{}) OrderedMap { 126 | return omap.SubMap(fromKey, nil) 127 | } 128 | 129 | func (omap *myOrderedMap) Keys() []interface{} { 130 | initialLen := omap.keys.Len() 131 | keys := make([]interface{}, initialLen) 132 | actualLen := 0 133 | for _, key := range omap.keys.GetAll() { 134 | if actualLen < initialLen { 135 | keys[actualLen] = key 136 | } else { 137 | keys = append(keys, key) 138 | } 139 | actualLen++ 140 | } 141 | if actualLen < initialLen { 142 | keys = keys[:actualLen] 143 | } 144 | return keys 145 | } 146 | 147 | func (omap *myOrderedMap) Elems() []interface{} { 148 | initialLen := omap.Len() 149 | elems := make([]interface{}, initialLen) 150 | actualLen := 0 151 | for _, key := range omap.keys.GetAll() { 152 | elem := omap.m[key] 153 | if actualLen < initialLen { 154 | elems[actualLen] = elem 155 | } else { 156 | elems = append(elems, elem) 157 | } 158 | actualLen++ 159 | } 160 | if actualLen < initialLen { 161 | elems = elems[:actualLen] 162 | } 163 | return elems 164 | } 165 | 166 | func (omap *myOrderedMap) ToMap() map[interface{}]interface{} { 167 | replica := make(map[interface{}]interface{}) 168 | for k, v := range omap.m { 169 | replica[k] = v 170 | } 171 | return replica 172 | } 173 | 174 | func (omap *myOrderedMap) KeyType() reflect.Type { 175 | return omap.keys.ElemType() 176 | } 177 | 178 | func (omap *myOrderedMap) ElemType() reflect.Type { 179 | return omap.elemType 180 | } 181 | 182 | func (omap *myOrderedMap) String() string { 183 | var buf bytes.Buffer 184 | buf.WriteString("OrderedMap<") 185 | buf.WriteString(omap.keys.ElemType().Kind().String()) 186 | buf.WriteString(",") 187 | buf.WriteString(omap.elemType.Kind().String()) 188 | buf.WriteString(">{") 189 | first := true 190 | omapLen := omap.Len() 191 | for i := 0; i < omapLen; i++ { 192 | if first { 193 | first = false 194 | } else { 195 | buf.WriteString(" ") 196 | } 197 | key := omap.keys.Get(i) 198 | buf.WriteString(fmt.Sprintf("%v", key)) 199 | buf.WriteString(":") 200 | buf.WriteString(fmt.Sprintf("%v", omap.m[key])) 201 | } 202 | buf.WriteString("}") 203 | return buf.String() 204 | } 205 | 206 | func NewOrderedMap(keys Keys, elemType reflect.Type) OrderedMap { 207 | return &myOrderedMap{ 208 | keys: keys, 209 | elemType: elemType, 210 | m: make(map[interface{}]interface{})} 211 | } 212 | -------------------------------------------------------------------------------- /src/basic/map1/omap_test.go: -------------------------------------------------------------------------------- 1 | package map1 2 | 3 | import ( 4 | "math/rand" 5 | "reflect" 6 | "runtime/debug" 7 | "testing" 8 | ) 9 | 10 | func testOrderedMap( 11 | t *testing.T, 12 | newOrderedMap func() OrderedMap, 13 | genKey func() interface{}, 14 | genElem func() interface{}, 15 | elemKind reflect.Kind) { 16 | defer func() { 17 | if err := recover(); err != nil { 18 | debug.PrintStack() 19 | t.Errorf("Fatal Error: OrderedMap(type=%s): %s\n", elemKind, err) 20 | } 21 | }() 22 | t.Logf("Starting TestOrderedMap...", elemKind) 23 | 24 | // Basic 25 | omap := newOrderedMap() 26 | expectedLen := 0 27 | if omap.Len() != expectedLen { 28 | t.Errorf("ERROR: The length of OrderedMap(elemType=%s) value %d is not %d!\n", 29 | elemKind, omap.Len(), expectedLen) 30 | t.FailNow() 31 | } 32 | expectedLen = 5 33 | testMap := make(map[interface{}]interface{}, expectedLen) 34 | var invalidKey interface{} 35 | for i := 0; i < expectedLen; i++ { 36 | key := genKey() 37 | testMap[key] = genElem() 38 | if invalidKey == nil { 39 | invalidKey = key 40 | } 41 | } 42 | for key, elem := range testMap { 43 | oldElem, ok := omap.Put(key, elem) 44 | if !ok { 45 | t.Errorf("ERROR: Put (%v, %v) to OrderedMap(elemType=%s) value %d is failing!\n", 46 | key, elem, elemKind, omap) 47 | t.FailNow() 48 | } 49 | if oldElem != nil { 50 | t.Errorf("ERROR: Already had a (%v, %v) in OrderedMap(elemType=%s) value %d!\n", 51 | key, elem, elemKind, omap) 52 | t.FailNow() 53 | } 54 | t.Logf("Put (%v, %v) to the OrderedMap(elemType=%s) value %v.", 55 | key, elem, elemKind, omap) 56 | } 57 | if omap.Len() != expectedLen { 58 | t.Errorf("ERROR: The length of OrderedMap(elemType=%s) value %d is not %d!\n", 59 | elemKind, omap.Len(), expectedLen) 60 | t.FailNow() 61 | } 62 | for key, elem := range testMap { 63 | contains := omap.Contains(key) 64 | if !contains { 65 | t.Errorf("ERROR: The OrderedMap(elemType=%s) value %v do not contains %v!", 66 | elemKind, omap, key) 67 | t.FailNow() 68 | } 69 | actualElem := omap.Get(key) 70 | if actualElem == nil { 71 | t.Errorf("ERROR: The OrderedMap(elemType=%s) value %v do not contains %v!", 72 | elemKind, omap, key) 73 | t.FailNow() 74 | } 75 | t.Logf("The OrderedMap(elemType=%s) value %v contains key %v.", elemKind, omap, key) 76 | if actualElem != elem { 77 | t.Errorf("ERROR: The element of OrderedMap(elemType=%s) value %v with key %v do not equals %v!\n", 78 | elemKind, actualElem, key, elem) 79 | t.FailNow() 80 | } 81 | t.Logf("The element of OrderedMap(elemType=%s) value %v to key %v is %v.", 82 | elemKind, omap, key, actualElem) 83 | } 84 | oldElem := omap.Remove(invalidKey) 85 | if oldElem == nil { 86 | t.Errorf("ERROR: Remove %v from OrderedMap(elemType=%s) value %d is failing!\n", 87 | invalidKey, elemKind, omap) 88 | t.FailNow() 89 | } 90 | t.Logf("Removed (%v, %v) from the OrderedMap(elemType=%s) value %v.", 91 | invalidKey, oldElem, elemKind, omap) 92 | delete(testMap, invalidKey) 93 | 94 | // Type 95 | actualElemType := omap.ElemType() 96 | if actualElemType == nil { 97 | t.Errorf("ERROR: The element type of OrderedMap(elemType=%s) value is nil!\n", 98 | elemKind) 99 | t.FailNow() 100 | } 101 | actualElemKind := actualElemType.Kind() 102 | if actualElemKind != elemKind { 103 | t.Errorf("ERROR: The element type of OrderedMap(elemType=%s) value %s is not %s!\n", 104 | elemKind, actualElemKind, elemKind) 105 | t.FailNow() 106 | } 107 | t.Logf("The element type of OrderedMap(elemType=%s) value %v is %s.", 108 | elemKind, omap, actualElemKind) 109 | actualKeyKind := omap.KeyType().Kind() 110 | keyKind := reflect.TypeOf(genKey()).Kind() 111 | if actualKeyKind != elemKind { 112 | t.Errorf("ERROR: The key type of OrderedMap(elemType=%s) value %s is not %s!\n", 113 | keyKind, actualKeyKind, keyKind) 114 | t.FailNow() 115 | } 116 | t.Logf("The key type of OrderedMap(elemType=%s) value %v is %s.", 117 | keyKind, omap, actualKeyKind) 118 | 119 | // Export 120 | keys := omap.Keys() 121 | elems := omap.Elems() 122 | pairs := omap.ToMap() 123 | for key, elem := range testMap { 124 | var hasKey bool 125 | for _, k := range keys { 126 | if k == key { 127 | hasKey = true 128 | } 129 | } 130 | if !hasKey { 131 | t.Errorf("ERROR: The keys of OrderedMap(elemType=%s) value %v do not contains %v!\n", 132 | elemKind, omap, key) 133 | t.FailNow() 134 | } 135 | var hasElem bool 136 | for _, e := range elems { 137 | if e == elem { 138 | hasElem = true 139 | } 140 | } 141 | if !hasElem { 142 | t.Errorf("ERROR: The elems of OrderedMap(elemType=%s) value %v do not contains %v!\n", 143 | elemKind, omap, elem) 144 | t.FailNow() 145 | } 146 | var hasPair bool 147 | for k, e := range pairs { 148 | if k == key && e == elem { 149 | hasPair = true 150 | } 151 | } 152 | if !hasPair { 153 | t.Errorf("ERROR: The elems of OrderedMap(elemType=%s) value %v do not contains (%v, %v)!\n", 154 | elemKind, omap, key, elem) 155 | t.FailNow() 156 | } 157 | } 158 | 159 | // Advance 160 | fKey := omap.FirstKey() 161 | if fKey != keys[0] { 162 | t.Errorf("ERROR: The first key of OrderedMap(elemType=%s) value %v is not equals %v!\n", 163 | elemKind, fKey, keys[0]) 164 | t.FailNow() 165 | } 166 | t.Logf("The first key of OrderedMap(elemType=%s) value %v is %s.", 167 | elemKind, omap, fKey) 168 | lKey := omap.LastKey() 169 | if lKey != keys[len(keys)-1] { 170 | t.Errorf("ERROR: The last key of OrderedMap(elemType=%s) value %v is not equals %v!\n", 171 | elemKind, lKey, keys[len(keys)-1]) 172 | t.FailNow() 173 | } 174 | t.Logf("The last key of OrderedMap(elemType=%s) value %v is %s.", 175 | elemKind, omap, lKey) 176 | endIndex := len(keys)/2 + 1 177 | toKey := keys[endIndex] 178 | headMap := omap.HeadMap(toKey) 179 | headKeys := headMap.Keys() 180 | for i := 0; i < endIndex; i++ { 181 | hKey := headKeys[i] 182 | tempKey := keys[i] 183 | if hKey != tempKey { 184 | t.Errorf("ERROR: The key of OrderedMap(elemType=%s) value %v with index %d is not equals %v!\n", 185 | elemKind, tempKey, i, hKey) 186 | t.FailNow() 187 | } 188 | } 189 | beginIndex := len(keys)/2 - 1 190 | endIndex = len(keys) - 1 191 | fromKey := keys[beginIndex] 192 | tailMap := omap.TailMap(fromKey) 193 | tailKeys := tailMap.Keys() 194 | for i := beginIndex; i < endIndex; i++ { 195 | tKey := tailKeys[i-beginIndex] 196 | tempKey := keys[i] 197 | if tKey != tempKey { 198 | t.Errorf("ERROR: The key of OrderedMap(elemType=%s) value %v with index %d is not equals %v!\n", 199 | elemKind, tempKey, i, tKey) 200 | t.FailNow() 201 | } 202 | } 203 | beginIndex = len(keys)/2 - 1 204 | endIndex = len(keys)/2 + 1 205 | fromKey = keys[beginIndex] 206 | toKey = keys[endIndex] 207 | subMap := omap.SubMap(fromKey, toKey) 208 | subKeys := subMap.Keys() 209 | for i := beginIndex; i < endIndex; i++ { 210 | sKey := subKeys[i-beginIndex] 211 | tempKey := keys[i] 212 | if sKey != tempKey { 213 | t.Errorf("ERROR: The key of OrderedMap(elemType=%s) value %v with index %d is not equals %v!\n", 214 | elemKind, tempKey, i, sKey) 215 | t.FailNow() 216 | } 217 | } 218 | 219 | // Clear 220 | omap.Clear() 221 | if omap.Len() != 0 { 222 | t.Errorf("ERROR: Clear OrderedMap(elemType=%s) value %d is failing!\n", 223 | elemKind, omap) 224 | t.FailNow() 225 | } 226 | t.Logf("The OrderedMap(elemType=%s) value %v has been cleared.", elemKind, omap) 227 | } 228 | 229 | func TestInt64Omap(t *testing.T) { 230 | keys := NewKeys( 231 | func(e1 interface{}, e2 interface{}) int8 { 232 | k1 := e1.(int64) 233 | k2 := e2.(int64) 234 | if k1 < k2 { 235 | return -1 236 | } else if k1 > k2 { 237 | return 1 238 | } else { 239 | return 0 240 | } 241 | }, 242 | reflect.TypeOf(int64(1))) 243 | newOmap := func() OrderedMap { 244 | return NewOrderedMap(keys, reflect.TypeOf(int64(1))) 245 | } 246 | testOrderedMap( 247 | t, 248 | newOmap, 249 | func() interface{} { return rand.Int63n(1000) }, 250 | func() interface{} { return rand.Int63n(1000) }, 251 | reflect.Int64) 252 | } 253 | 254 | func TestFloat64Omap(t *testing.T) { 255 | keys := NewKeys( 256 | func(e1 interface{}, e2 interface{}) int8 { 257 | k1 := e1.(float64) 258 | k2 := e2.(float64) 259 | if k1 < k2 { 260 | return -1 261 | } else if k1 > k2 { 262 | return 1 263 | } else { 264 | return 0 265 | } 266 | }, 267 | reflect.TypeOf(float64(1))) 268 | newOmap := func() OrderedMap { 269 | return NewOrderedMap(keys, reflect.TypeOf(float64(1))) 270 | } 271 | testOrderedMap( 272 | t, 273 | newOmap, 274 | func() interface{} { return rand.Float64() }, 275 | func() interface{} { return rand.Float64() }, 276 | reflect.Float64) 277 | } 278 | 279 | func TestStringOmap(t *testing.T) { 280 | keys := NewKeys( 281 | func(e1 interface{}, e2 interface{}) int8 { 282 | k1 := e1.(string) 283 | k2 := e2.(string) 284 | if k1 < k2 { 285 | return -1 286 | } else if k1 > k2 { 287 | return 1 288 | } else { 289 | return 0 290 | } 291 | }, 292 | reflect.TypeOf(string(1))) 293 | newOmap := func() OrderedMap { 294 | return NewOrderedMap(keys, reflect.TypeOf(string(1))) 295 | } 296 | testOrderedMap( 297 | t, 298 | newOmap, 299 | func() interface{} { return genRandString() }, 300 | func() interface{} { return genRandString() }, 301 | reflect.String) 302 | } 303 | -------------------------------------------------------------------------------- /src/basic/pkginit/initpkg_demo.go: -------------------------------------------------------------------------------- 1 | package main // 命令源码文件必须在这里声明自己属于main包 2 | 3 | import ( // 引入了代码包fmt和runtime 4 | "fmt" 5 | "runtime" 6 | ) 7 | 8 | func init() { // 包初始化函数 9 | fmt.Printf("Map: %v\n", m) // 先格式化再打印 10 | // 通过调用runtime包的代码获取当前机器所运行的操作系统以及计算架构 11 | // 而后通过fmt包的Sprintf方法进行字符串格式化并赋值给变量info 12 | info = fmt.Sprintf("OS: %s, Arch: %s", runtime.GOOS, runtime.GOARCH) 13 | } 14 | 15 | var m map[int]string = map[int]string{1: "A", 2: "B", 3: "C"} // 非局部变量，map类型，且已初始化 16 | 17 | var info string // 非局部变量，string类型，未被初始化 18 | 19 | func main() { // 命令源码文件必须有的入口函数 20 | fmt.Println(info) // 打印变量info 21 | } 22 | -------------------------------------------------------------------------------- /src/basic/prof/profiling.go: -------------------------------------------------------------------------------- 1 | package prof 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "runtime" 10 | "runtime/pprof" 11 | "sync" 12 | ) 13 | 14 | type ProfileType string 15 | 16 | const ( 17 | GoroutineProfile ProfileType = "goroutine" 18 | ThreadcreateProfile ProfileType = "threadcreate" 19 | HeapProfile ProfileType = "heap" 20 | BlockProfile ProfileType = "block" 21 | ) 22 | 23 | var ( // profile flags 24 | memProfile = flag.String("memprofile", "", "write a memory profile to the named file after execution") 25 | memProfileRate = flag.Int("memprofilerate", 0, "if > 0, sets runtime.MemProfileRate") 26 | cpuProfile = flag.String("cpuprofile", "", "write a cpu profile to the named file during execution") 27 | blockProfile = flag.String("blockprofile", "", "write a goroutine blocking profile to the named file after execution") 28 | blockProfileRate = flag.Int("blockprofilerate", 1, "if > 0, calls runtime.SetBlockProfileRate()") 29 | ) 30 | 31 | var running bool 32 | var lock *sync.Mutex = new(sync.Mutex) 33 | 34 | func parseProfFlags() { 35 | if !flag.Parsed() { 36 | flag.Parse() 37 | } 38 | *cpuProfile = getAbsFilePath(*cpuProfile) 39 | *blockProfile = getAbsFilePath(*blockProfile) 40 | *memProfile = getAbsFilePath(*memProfile) 41 | } 42 | 43 | func getAbsFilePath(path string) string { 44 | if path == "" { 45 | return "" 46 | } 47 | path = filepath.FromSlash(path) 48 | if !filepath.IsAbs(path) { 49 | baseDir, err := os.Getwd() 50 | if err != nil { 51 | panic(errors.New(fmt.Sprintf("Can not get current work dir: %s\n", err))) 52 | } 53 | path = filepath.Join(baseDir, path) 54 | } 55 | return path 56 | } 57 | 58 | func Start() { 59 | lock.Lock() 60 | defer lock.Unlock() 61 | parseProfFlags() 62 | startBlockProfile() 63 | startCPUProfile() 64 | startMemProfile() 65 | running = true 66 | } 67 | 68 | func startBlockProfile() { 69 | if *blockProfile != "" && *blockProfileRate > 0 { 70 | runtime.SetBlockProfileRate(*blockProfileRate) 71 | } 72 | } 73 | 74 | func startCPUProfile() { 75 | if *cpuProfile != "" { 76 | f, err := os.Create(*cpuProfile) 77 | if err != nil { 78 | fmt.Fprintf(os.Stderr, "Can not create cpu profile output file: %s\n", err) 79 | return 80 | } 81 | if err := pprof.StartCPUProfile(f); err != nil { 82 | fmt.Fprintf(os.Stderr, "Can not start cpu profile: %s\n", err) 83 | f.Close() 84 | return 85 | } 86 | } 87 | } 88 | 89 | func startMemProfile() { 90 | if *memProfile != "" && *memProfileRate > 0 { 91 | runtime.MemProfileRate = *memProfileRate 92 | } 93 | } 94 | 95 | func Stop() { 96 | lock.Lock() 97 | defer lock.Unlock() 98 | stopBlockProfile() 99 | stopCPUProfile() 100 | stopMemProfile() 101 | running = false 102 | } 103 | 104 | func stopBlockProfile() { 105 | if *blockProfile != "" && *blockProfileRate >= 0 { 106 | f, err := os.Create(*blockProfile) 107 | if err != nil { 108 | fmt.Fprintf(os.Stderr, "Can not create block profile output file: %s\n", err) 109 | return 110 | } 111 | if err = pprof.Lookup("block").WriteTo(f, 0); err != nil { 112 | fmt.Fprintf(os.Stderr, "Can not write %s: %s\n", *blockProfile, err) 113 | } 114 | f.Close() 115 | } 116 | } 117 | 118 | func stopCPUProfile() { 119 | if *cpuProfile != "" { 120 | pprof.StopCPUProfile() // 把记录的概要信息写到已指定的文件 121 | } 122 | } 123 | 124 | func stopMemProfile() { 125 | if *memProfile != "" { 126 | f, err := os.Create(*memProfile) 127 | if err != nil { 128 | fmt.Fprintf(os.Stderr, "Can not create mem profile output file: %s\n", err) 129 | return 130 | } 131 | if err = pprof.WriteHeapProfile(f); err != nil { 132 | fmt.Fprintf(os.Stderr, "Can not write %s: %s\n", *memProfile, err) 133 | } 134 | f.Close() 135 | } 136 | } 137 | 138 | func SaveProfile(workDir string, profileName string, ptype ProfileType, debug int) { 139 | absWorkDir := getAbsFilePath(workDir) 140 | if profileName == "" { 141 | profileName = string(ptype) 142 | } 143 | profileName += ".out" 144 | profilePath := filepath.Join(absWorkDir, profileName) 145 | f, err := os.Create(profilePath) 146 | if err != nil { 147 | fmt.Fprintf(os.Stderr, "Can not create profile output file: %s\n", err) 148 | return 149 | } 150 | if err = pprof.Lookup(string(ptype)).WriteTo(f, debug); err != nil { 151 | fmt.Fprintf(os.Stderr, "Can not write %s: %s\n", profilePath, err) 152 | } 153 | f.Close() 154 | } 155 | -------------------------------------------------------------------------------- /src/basic/prof/profiling_test.go: -------------------------------------------------------------------------------- 1 | package prof 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "runtime/debug" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | const ( 13 | tempWorkDir = "../../../pprof" 14 | cpuProfilePath = tempWorkDir + "/cpu.out" 15 | memProfilePath = tempWorkDir + "/mem.out" 16 | blockProfilePath = tempWorkDir + "/block.out" 17 | ) 18 | 19 | var newTempWorkDir bool 20 | var removeTempWorkDir bool = false 21 | 22 | func TestCommonProf(t *testing.T) { 23 | defer func() { 24 | if err := recover(); err != nil { 25 | debug.PrintStack() 26 | t.Errorf("Fatal Error: %s\n", err) 27 | } 28 | }() 29 | makeWorkDir() 30 | *cpuProfile = cpuProfilePath 31 | *blockProfile = blockProfilePath 32 | *memProfile = memProfilePath 33 | Start() 34 | doSomething() 35 | Stop() 36 | cleanWorkDir() 37 | } 38 | 39 | func doSomething() { 40 | size := 10000000 41 | randomNumbers := make([]int, size) 42 | max := int32(size) 43 | for i := 0; i < size; i++ { 44 | target := rand.Int31n(max) 45 | randomNumbers[target] = i 46 | } 47 | } 48 | 49 | func makeWorkDir() { 50 | absTempWorkDir := getAbsFilePath(tempWorkDir) 51 | f, err := os.Open(absTempWorkDir) 52 | if err != nil { 53 | if _, ok := err.(*os.PathError); !ok { 54 | panic(err) 55 | } else { 56 | newTempWorkDir = true 57 | } 58 | } 59 | if f != nil { 60 | fi, err := f.Stat() 61 | if err != nil { 62 | panic(err) 63 | } 64 | if !fi.IsDir() { 65 | panic("There are name of a file conflict with temp work dir name!") 66 | } else { 67 | fmt.Printf("Temp work dir: '%s'\n", absTempWorkDir) 68 | } 69 | } 70 | if f == nil { 71 | fmt.Printf("Make temp work dir '%s'...\n", absTempWorkDir) 72 | err = os.Mkdir(absTempWorkDir, os.ModeDir | os.ModePerm) 73 | if err != nil { 74 | panic(err) 75 | } 76 | } 77 | _ = f 78 | } 79 | 80 | func cleanWorkDir() { 81 | if removeTempWorkDir { 82 | err := remove(cpuProfilePath, 5, 2) 83 | if err != nil { 84 | fmt.Errorf("Error: Can not remove cpu profile '%s': %s\n", 85 | getAbsFilePath(cpuProfilePath), err) 86 | } 87 | err = remove(blockProfilePath, 5, 2) 88 | if err != nil { 89 | fmt.Errorf("Error: Can not remove block profile '%s': %s\n", 90 | getAbsFilePath(blockProfilePath), err) 91 | } 92 | err = remove(memProfilePath, 5, 2) 93 | if err != nil { 94 | fmt.Errorf("Error: Can not remove mem profile '%s': %s\n", 95 | getAbsFilePath(memProfilePath), err) 96 | } 97 | } 98 | if newTempWorkDir && removeTempWorkDir { 99 | err := remove(tempWorkDir, 5, 3) 100 | if err != nil { 101 | fmt.Errorf("Error: Can not remove temp work dir '%s': %s\n", 102 | getAbsFilePath(tempWorkDir), err) 103 | } 104 | } 105 | } 106 | 107 | func remove(path string, repeat int, intervalSeconds int) error { 108 | if repeat < 0 { 109 | repeat = 5 110 | } 111 | if intervalSeconds <= 0 { 112 | intervalSeconds = 2 113 | } 114 | loopNumber := repeat + 1 115 | absPath := getAbsFilePath(path) 116 | for i := 1; i <= loopNumber; i++ { 117 | fmt.Printf("Try to remove file/dir '%s'...", absPath) 118 | err := os.Remove(absPath) 119 | if err == nil { 120 | fmt.Println("ok.") 121 | break 122 | } else { 123 | fmt.Println("failing!") 124 | } 125 | if err != nil && i == loopNumber { 126 | return err 127 | } 128 | if err != nil && i < loopNumber { 129 | time.Sleep(time.Duration(intervalSeconds) * time.Second) 130 | } 131 | } 132 | return nil 133 | } 134 | 135 | func TestRuntimeProf(t *testing.T) { 136 | defer func() { 137 | if err := recover(); err != nil { 138 | debug.PrintStack() 139 | t.Errorf("Fatal Error: %s\n", err) 140 | } 141 | }() 142 | for _, v := range []int{0, 1, 2} { 143 | for _, w := range []string{"goroutine", "threadcreate", "heap", "block"} { 144 | profileName := fmt.Sprintf("%s.%d", w, v) 145 | fmt.Printf("Try to save %s profile to file '%s' ...\n", w, profileName) 146 | SaveProfile(tempWorkDir, profileName, ProfileType(w), v) 147 | } 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/basic/seq.go: -------------------------------------------------------------------------------- 1 | package basic 2 | 3 | import ( 4 | "reflect" 5 | "sort" 6 | ) 7 | 8 | type Sortable interface { 9 | sort.Interface 10 | Sort() 11 | } 12 | 13 | type GenericSeq interface { 14 | Sortable 15 | Append(e interface{}) bool 16 | Set(index int, e interface{}) bool 17 | Delete(index int) (interface{}, bool) 18 | ElemValue(index int) interface{} 19 | ElemType() reflect.Type 20 | Value() interface{} 21 | } 22 | 23 | type StringSeq struct { 24 | value []string 25 | } 26 | 27 | func (self *StringSeq) Len() int { 28 | return len(self.value) 29 | } 30 | 31 | func (self *StringSeq) Less(i, j int) bool { 32 | return self.value[i] < self.value[j] 33 | } 34 | 35 | func (self *StringSeq) Swap(i, j int) { 36 | self.value[i], self.value[j] = self.value[j], self.value[i] 37 | } 38 | 39 | func (self *StringSeq) Sort() { 40 | sort.Sort(self) 41 | } 42 | 43 | func (self *StringSeq) Append(e interface{}) bool { 44 | s, ok := e.(string) 45 | if !ok { 46 | return false 47 | } 48 | self.value = append(self.value, s) 49 | return true 50 | } 51 | 52 | func (self *StringSeq) Set(index int, e interface{}) bool { 53 | if index >= self.Len() { 54 | return false 55 | } 56 | s, ok := e.(string) 57 | if !ok { 58 | return false 59 | } 60 | self.value[index] = s 61 | return true 62 | } 63 | 64 | func (self *StringSeq) Delete(index int) (interface{}, bool) { 65 | length := self.Len() 66 | if index >= length { 67 | return nil, false 68 | } 69 | s := self.value[index] 70 | if index < (length - 1) { 71 | copy(self.value[index:], self.value[index+1:]) 72 | } 73 | invalidIndex := length - 1 74 | self.value[invalidIndex] = "" 75 | self.value = self.value[:invalidIndex] 76 | return s, true 77 | } 78 | 79 | func (self StringSeq) ElemValue(index int) interface{} { 80 | if index >= self.Len() { 81 | return nil 82 | } 83 | return self.value[index] 84 | } 85 | 86 | func (self *StringSeq) ElemType() reflect.Type { 87 | return reflect.TypeOf(self.value).Elem() 88 | } 89 | 90 | func (self StringSeq) Value() interface{} { 91 | return self.value 92 | } 93 | 94 | type Sequence struct { 95 | GenericSeq 96 | sorted bool 97 | elemType reflect.Type 98 | } 99 | 100 | func (self *Sequence) Sort() { 101 | self.GenericSeq.Sort() 102 | self.sorted = true 103 | } 104 | 105 | func (self *Sequence) Append(e interface{}) bool { 106 | result := self.GenericSeq.Append(e) 107 | if result && self.sorted { 108 | self.sorted = false 109 | } 110 | return result 111 | } 112 | 113 | func (self *Sequence) Set(index int, e interface{}) bool { 114 | result := self.GenericSeq.Set(index, e) 115 | if result && self.sorted { 116 | self.sorted = false 117 | } 118 | return result 119 | } 120 | 121 | func (self *Sequence) ElemType() reflect.Type { 122 | if self.elemType == nil && self.GenericSeq != nil { 123 | self.elemType = self.GenericSeq.ElemType() 124 | } 125 | return self.elemType 126 | } 127 | 128 | func (self *Sequence) Init() (ok bool) { 129 | if self.GenericSeq != nil { 130 | self.elemType = self.GenericSeq.ElemType() 131 | ok = true 132 | } 133 | return ok 134 | } 135 | 136 | func (self *Sequence) Sorted() bool { 137 | return self.sorted 138 | } 139 | -------------------------------------------------------------------------------- /src/basic/set/hash_set.go: -------------------------------------------------------------------------------- 1 | package set 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | ) 7 | 8 | type HashSet struct { 9 | m map[interface{}]bool 10 | } 11 | 12 | func NewHashSet() *HashSet { 13 | return &HashSet{m: make(map[interface{}]bool)} 14 | } 15 | 16 | func (set *HashSet) Add(e interface{}) bool { 17 | if !set.m[e] { 18 | set.m[e] = true 19 | return true 20 | } 21 | return false 22 | } 23 | 24 | func (set *HashSet) Remove(e interface{}) { 25 | delete(set.m, e) 26 | } 27 | 28 | func (set *HashSet) Clear() { 29 | set.m = make(map[interface{}]bool) 30 | } 31 | 32 | func (set *HashSet) Contains(e interface{}) bool { 33 | return set.m[e] 34 | } 35 | 36 | func (set *HashSet) Len() int { 37 | return len(set.m) 38 | } 39 | 40 | func (set *HashSet) Same(other Set) bool { 41 | if other == nil { 42 | return false 43 | } 44 | if set.Len() != other.Len() { 45 | return false 46 | } 47 | for key := range set.m { 48 | if !other.Contains(key) { 49 | return false 50 | } 51 | } 52 | return true 53 | } 54 | 55 | func (set *HashSet) Elements() []interface{} { 56 | initialLen := len(set.m) 57 | snapshot := make([]interface{}, initialLen) 58 | actualLen := 0 59 | for key := range set.m { 60 | if actualLen < initialLen { 61 | snapshot[actualLen] = key 62 | } else { 63 | snapshot = append(snapshot, key) 64 | } 65 | actualLen++ 66 | } 67 | if actualLen < initialLen { 68 | snapshot = snapshot[:actualLen] 69 | } 70 | return snapshot 71 | } 72 | 73 | func (set *HashSet) String() string { 74 | var buf bytes.Buffer 75 | buf.WriteString("HashSet{") 76 | first := true 77 | for key := range set.m { 78 | if first { 79 | first = false 80 | } else { 81 | buf.WriteString(" ") 82 | } 83 | buf.WriteString(fmt.Sprintf("%v", key)) 84 | } 85 | buf.WriteString("}") 86 | return buf.String() 87 | } 88 | -------------------------------------------------------------------------------- /src/basic/set/hash_set_test.go: -------------------------------------------------------------------------------- 1 | package set 2 | 3 | import ( 4 | "fmt" 5 | "runtime/debug" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func TestHashSetCreation(t *testing.T) { 11 | defer func() { 12 | if err := recover(); err != nil { 13 | debug.PrintStack() 14 | t.Errorf("Fatal Error: %s\n", err) 15 | } 16 | }() 17 | t.Log("Starting TestHashSetCreation...") 18 | hs := NewHashSet() 19 | t.Logf("Create a HashSet value: %v\n", hs) 20 | if hs == nil { 21 | t.Errorf("The result of func NewHashSet is nil!\n") 22 | } 23 | isSet := IsSet(hs) 24 | if !isSet { 25 | t.Errorf("The value of HashSet is not Set!\n") 26 | } else { 27 | t.Logf("The HashSet value is a Set.\n") 28 | } 29 | } 30 | 31 | func TestHashSetLenAndContains(t *testing.T) { 32 | testSetLenAndContains(t, func() Set { return NewHashSet() }, "HashSet") 33 | } 34 | 35 | func TestHashSetAdd(t *testing.T) { 36 | testSetAdd(t, func() Set { return NewHashSet() }, "HashSet") 37 | } 38 | 39 | func TestHashSetRemove(t *testing.T) { 40 | testSetRemove(t, func() Set { return NewHashSet() }, "HashSet") 41 | } 42 | 43 | func TestHashSetClear(t *testing.T) { 44 | testSetClear(t, func() Set { return NewHashSet() }, "HashSet") 45 | } 46 | 47 | func TestHashSetElements(t *testing.T) { 48 | testSetElements(t, func() Set { return NewHashSet() }, "HashSet") 49 | } 50 | 51 | func TestHashSetSame(t *testing.T) { 52 | testSetSame(t, func() Set { return NewHashSet() }, "HashSet") 53 | } 54 | 55 | func TestSetString(t *testing.T) { 56 | testSetString(t, func() Set { return NewHashSet() }, "HashSet") 57 | } 58 | 59 | func testSetOp(t *testing.T) { 60 | defer func() { 61 | if err := recover(); err != nil { 62 | debug.PrintStack() 63 | t.Errorf("Fatal Error: %s\n", err) 64 | } 65 | }() 66 | fmt.Println(222) 67 | t.Logf("Starting TestHashSetOp...") 68 | hs := NewHashSet() 69 | if hs.Len() != 0 { 70 | t.Errorf("ERROR: The length of original HashSet value is not 0!\n") 71 | t.FailNow() 72 | } 73 | randElem := genRandElement() 74 | expectedElemMap := make(map[interface{}]bool) 75 | t.Logf("Add %v to the HashSet value %v.\n", randElem, hs) 76 | hs.Add(randElem) 77 | expectedElemMap[randElem] = true 78 | expectedLen := len(expectedElemMap) 79 | if hs.Len() != expectedLen { 80 | t.Errorf("ERROR: The length of HashSet value %d is not %d!\n", hs.Len(), expectedLen) 81 | t.FailNow() 82 | } 83 | var result bool 84 | for i := 0; i < 8; i++ { 85 | randElem = genRandElement() 86 | t.Logf("Add %v to the HashSet value %v.\n", randElem, hs) 87 | result = hs.Add(randElem) 88 | if expectedElemMap[randElem] && result { 89 | t.Errorf("ERROR: The element adding (%v => %v) is successful but should be failing!\n", 90 | randElem, hs) 91 | t.FailNow() 92 | } 93 | if !expectedElemMap[randElem] && !result { 94 | t.Errorf("ERROR: The element adding (%v => %v) is failing!\n", 95 | randElem, hs) 96 | t.FailNow() 97 | } 98 | expectedElemMap[randElem] = true 99 | } 100 | expectedLen = len(expectedElemMap) 101 | if hs.Len() != expectedLen { 102 | t.Errorf("ERROR: The length of HashSet value %d is not %d!\n", hs.Len(), expectedLen) 103 | t.FailNow() 104 | } 105 | for k, _ := range expectedElemMap { 106 | if !hs.Contains(k) { 107 | t.Errorf("ERROR: The HashSet value %v do not contains %v!", hs, k) 108 | t.FailNow() 109 | } 110 | } 111 | number := 2 112 | for k, _ := range expectedElemMap { 113 | if number%2 == 0 { 114 | t.Logf("Remove %v from the HashSet value %v.\n", k, hs) 115 | hs.Remove(k) 116 | if hs.Contains(k) { 117 | t.Errorf("ERROR: The element adding (%v => %v) is failing!\n", 118 | randElem, hs) 119 | t.FailNow() 120 | } 121 | delete(expectedElemMap, k) 122 | } 123 | number++ 124 | } 125 | expectedLen = len(expectedElemMap) 126 | if hs.Len() != expectedLen { 127 | t.Errorf("ERROR: The length of HashSet value %d is not %d!\n", hs.Len(), expectedLen) 128 | t.FailNow() 129 | } 130 | for _, v := range hs.Elements() { 131 | if !expectedElemMap[v] { 132 | t.Errorf("ERROR: The HashSet value %v contains %v!", hs, v) 133 | t.FailNow() 134 | } 135 | } 136 | hs2 := NewHashSet() 137 | for k, _ := range expectedElemMap { 138 | hs2.Add(k) 139 | } 140 | if !hs.Same(hs2) { 141 | t.Errorf("ERROR: HashSet value %v do not same %v!\n", hs, hs2) 142 | t.FailNow() 143 | } 144 | str := hs.String() 145 | t.Logf("The string of HashSet value %v is '%s'.\n", hs, str) 146 | for _, v := range hs.Elements() { 147 | if !strings.Contains(str, fmt.Sprintf("%v", v)) { 148 | t.Errorf("ERROR: '%s' do not contains '%v'!", str, v) 149 | t.FailNow() 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/basic/set/set.go: -------------------------------------------------------------------------------- 1 | package set 2 | 3 | type Set interface { 4 | Add(e interface{}) bool 5 | Remove(e interface{}) 6 | Clear() 7 | Contains(e interface{}) bool 8 | Len() int 9 | Same(other Set) bool 10 | Elements() []interface{} 11 | String() string 12 | } 13 | 14 | // 判断集合 one 是否是集合 other 的超集 15 | func IsSuperset(one Set, other Set) bool { 16 | if one == nil || other == nil { 17 | return false 18 | } 19 | oneLen := one.Len() 20 | otherLen := other.Len() 21 | if oneLen == 0 || oneLen == otherLen { 22 | return false 23 | } 24 | if oneLen > 0 && otherLen == 0 { 25 | return true 26 | } 27 | for _, v := range other.Elements() { 28 | if !one.Contains(v) { 29 | return false 30 | } 31 | } 32 | return true 33 | } 34 | 35 | // 生成集合 one 和集合 other 的并集 36 | func Union(one Set, other Set) Set { 37 | if one == nil || other == nil { 38 | return nil 39 | } 40 | unionedSet := NewSimpleSet() 41 | for _, v := range one.Elements() { 42 | unionedSet.Add(v) 43 | } 44 | if other.Len() == 0 { 45 | return unionedSet 46 | } 47 | for _, v := range other.Elements() { 48 | unionedSet.Add(v) 49 | } 50 | return unionedSet 51 | } 52 | 53 | // 生成集合 one 和集合 other 的交集 54 | func Intersect(one Set, other Set) Set { 55 | if one == nil || other == nil { 56 | return nil 57 | } 58 | intersectedSet := NewSimpleSet() 59 | if other.Len() == 0 { 60 | return intersectedSet 61 | } 62 | if one.Len() < other.Len() { 63 | for _, v := range one.Elements() { 64 | if other.Contains(v) { 65 | intersectedSet.Add(v) 66 | } 67 | } 68 | } else { 69 | for _, v := range other.Elements() { 70 | if one.Contains(v) { 71 | intersectedSet.Add(v) 72 | } 73 | } 74 | } 75 | return intersectedSet 76 | } 77 | 78 | // 生成集合 one 对集合 other 的差集 79 | func Difference(one Set, other Set) Set { 80 | if one == nil || other == nil { 81 | return nil 82 | } 83 | differencedSet := NewSimpleSet() 84 | if other.Len() == 0 { 85 | for _, v := range one.Elements() { 86 | differencedSet.Add(v) 87 | } 88 | return differencedSet 89 | } 90 | for _, v := range one.Elements() { 91 | if !other.Contains(v) { 92 | differencedSet.Add(v) 93 | } 94 | } 95 | return differencedSet 96 | } 97 | 98 | // 生成集合 one 和集合 other 的对称差集 99 | func SymmetricDifference(one Set, other Set) Set { 100 | if one == nil || other == nil { 101 | return nil 102 | } 103 | diffA := Difference(one, other) 104 | if other.Len() == 0 { 105 | return diffA 106 | } 107 | diffB := Difference(other, one) 108 | return Union(diffA, diffB) 109 | } 110 | 111 | func NewSimpleSet() Set { 112 | return NewHashSet() 113 | } 114 | 115 | func IsSet(value interface{}) bool { 116 | if _, ok := value.(Set); ok { 117 | return true 118 | } 119 | return false 120 | } 121 | -------------------------------------------------------------------------------- /src/basic/set2.go: -------------------------------------------------------------------------------- 1 | package basic 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | ) 7 | 8 | type KeyGeneratorFunc func(x interface{}) string 9 | 10 | type ComparisonFunc func(i, j interface{}) int 11 | 12 | type SimpleSetIterator func() (interface{}, bool) 13 | 14 | type SimpleSet struct { 15 | KeyGenerator KeyGeneratorFunc 16 | Comparator ComparisonFunc 17 | keys []string 18 | elementMap map[string]interface{} 19 | } 20 | 21 | func (self *SimpleSet) Len() int { 22 | return len(self.keys) 23 | } 24 | 25 | func (self *SimpleSet) Less(i, j int) bool { 26 | ki := self.keys[i] 27 | kj := self.keys[j] 28 | var result bool 29 | if !self.Sortable() { 30 | result = ki < kj 31 | } else { 32 | ii := self.elementMap[ki] 33 | ij := self.elementMap[kj] 34 | sign := self.Comparator(ii, ij) 35 | if sign < 0 { 36 | result = true 37 | } else { 38 | result = false 39 | } 40 | } 41 | return result 42 | } 43 | 44 | func (self *SimpleSet) Swap(i, j int) { 45 | self.keys[i], self.keys[j] = self.keys[j], self.keys[i] 46 | } 47 | 48 | func (self *SimpleSet) initialize(rebuild bool) { 49 | if rebuild { 50 | if self.elementMap != nil { 51 | self.elementMap = nil 52 | self.elementMap = make(map[string]interface{}) 53 | } 54 | if self.keys != nil { 55 | self.keys = nil 56 | self.keys = make([]string, 0) 57 | } 58 | } else { 59 | if self.elementMap == nil { 60 | self.elementMap = make(map[string]interface{}) 61 | } 62 | if self.keys == nil { 63 | self.keys = make([]string, 0) 64 | } 65 | } 66 | } 67 | 68 | func (self *SimpleSet) generateKey(value interface{}) string { 69 | var k string 70 | if self.KeyGenerator != nil { 71 | k = self.KeyGenerator(value) + "}" 72 | } else { 73 | k = fmt.Sprintf("%v", value) 74 | } 75 | return "K{" + k + "}" 76 | } 77 | 78 | func (self *SimpleSet) Add(element interface{}) bool { 79 | self.initialize(false) 80 | if element == nil { 81 | return false 82 | } 83 | done := false 84 | key := self.generateKey(element) 85 | if self.elementMap[key] == nil { 86 | self.elementMap[key] = element 87 | self.keys = append(self.keys, key) 88 | done = true 89 | } 90 | return done 91 | } 92 | 93 | func (self *SimpleSet) Remove(element interface{}) bool { 94 | if len(self.elementMap) == 0 { 95 | return false 96 | } 97 | key := self.generateKey(element) 98 | done := false 99 | if self.elementMap[key] != nil { 100 | delete(self.elementMap, key) 101 | var keyX string 102 | sort.Strings(self.keys) 103 | keySize := len(self.keys) 104 | index := sort.Search(keySize, func(x int) bool { 105 | keyX = self.keys[x] 106 | return keyX >= key 107 | }) 108 | if index >= 0 || index < keySize && keyX == key { 109 | copy(self.keys[index:], self.keys[index+1:]) 110 | endIndex := keySize - 1 111 | self.keys[endIndex] = "" 112 | self.keys = self.keys[:endIndex] 113 | done = true 114 | } 115 | } 116 | return done 117 | } 118 | 119 | func (self *SimpleSet) Clear() bool { 120 | self.initialize(true) 121 | return true 122 | } 123 | 124 | func (self *SimpleSet) Contain(element interface{}) bool { 125 | if self.elementMap == nil || len(self.elementMap) == 0 { 126 | return false 127 | } 128 | key := self.generateKey(element) 129 | if self.elementMap[key] == nil { 130 | return false 131 | } 132 | return true 133 | } 134 | 135 | func (self *SimpleSet) Iterator() SimpleSetIterator { 136 | self.initialize(false) 137 | return func() SimpleSetIterator { 138 | index := 0 139 | snapshots := self.Slice() 140 | return func() (interface{}, bool) { 141 | if index >= 0 && index < len(snapshots) { 142 | element := snapshots[index] 143 | index++ 144 | return element, true 145 | } 146 | return nil, false 147 | } 148 | }() 149 | } 150 | 151 | func (self *SimpleSet) Slice() []interface{} { 152 | if len(self.keys) == 0 { 153 | return make([]interface{}, 0) 154 | } 155 | snapshots := make([]interface{}, len(self.keys)) 156 | if self.Sortable() { 157 | sort.Sort(self) 158 | for i, k := range self.keys { 159 | snapshots[i] = self.elementMap[k] 160 | } 161 | } else { 162 | count := 0 163 | for _, v := range self.elementMap { 164 | snapshots[count] = v 165 | count++ 166 | } 167 | } 168 | return snapshots 169 | } 170 | 171 | func (self *SimpleSet) String() string { 172 | return fmt.Sprintf("%v", self.Slice()) 173 | } 174 | 175 | func (self *SimpleSet) Sortable() bool { 176 | return self.Comparator != nil 177 | } 178 | 179 | func (self *SimpleSet) GetComparator() ComparisonFunc { 180 | return self.Comparator 181 | } 182 | -------------------------------------------------------------------------------- /src/basic/set2_test.go: -------------------------------------------------------------------------------- 1 | package basic 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "runtime/debug" 7 | "sort" 8 | "testing" 9 | ) 10 | 11 | func TestSimpleSet(t *testing.T) { 12 | debugTag := false 13 | defer func() { 14 | if err := recover(); err != nil { 15 | debug.PrintStack() 16 | t.Errorf("Fatal Error: %s\n", err) 17 | } 18 | }() 19 | n := 10 20 | r := 3 21 | seeds := make([]int, r) 22 | for k, _ := range seeds { 23 | seeds[k] = k 24 | } 25 | t.Logf("Seeds: %v\n", seeds) 26 | t.Logf("Seeds Length: %v\n", len(seeds)) 27 | matrix := make([][]int, n) 28 | for i, _ := range matrix { 29 | indexs := make([]int, r) 30 | for j, _ := range indexs { 31 | m := rand.Intn(3) 32 | indexs[j] = seeds[m] 33 | } 34 | if debugTag { 35 | t.Logf("%v (i=%d)\n", indexs, i) 36 | } 37 | matrix[i] = indexs 38 | } 39 | t.Logf("Matrix: %v\n", matrix) 40 | t.Logf("Matrix Length: %v\n", len(matrix)) 41 | matrixSet := SimpleSet{KeyGenerator: generateKey, Comparator: compare} 42 | for _, v := range matrix { 43 | matrixSet.Add(interface{}(v)) 44 | } 45 | t.Logf("Matrix Set: %v\n", matrixSet) 46 | t.Logf("Matrix Set Length: %v\n", matrixSet.Len()) 47 | matrixSlice := matrixSet.Slice() 48 | t.Logf("Matrix Sorted Slice: %v\n", matrixSlice) 49 | matrixIterator := matrixSet.Iterator() 50 | var pe interface{} 51 | for j := 0; j < matrixSet.Len(); j++ { 52 | e, has := matrixIterator() 53 | if !has { 54 | break 55 | } 56 | if pe != nil { 57 | if compare(pe, e) > 0 { 58 | t.Errorf("Error: %v should be LE %v. (j=%d)\n", pe, e, j) 59 | t.FailNow() 60 | } 61 | if debugTag { 62 | t.Logf("%v <= %v. (j=%d)\n", pe, e, j) 63 | } 64 | } 65 | pe = e 66 | } 67 | randElement := matrixSlice[rand.Intn(len(matrixSlice))] 68 | t.Logf("Rand Elements: %v\n", randElement) 69 | if !matrixSet.Contain(randElement) { 70 | t.Errorf("Error: The element '%v' shoud be in marix set '%v'.\n", randElement, matrixSet) 71 | } 72 | t.Logf("The matrix contains element '%v'.\n", randElement) 73 | if !matrixSet.Remove(randElement) { 74 | t.Errorf("Error: Remove element '%v' shoud be successful.\n", randElement) 75 | } 76 | t.Logf("The element '%v' is removed.\n", randElement) 77 | if matrixSet.Contain(randElement) { 78 | t.Errorf("Error: The removed element '%v' shoud not be in marix set '%v'.\n", randElement, matrixSet) 79 | } 80 | t.Logf("The matrix not contains element '%v'.\n", randElement) 81 | if matrixSet.Remove(randElement) { 82 | t.Errorf("Error: Remove removed element '%v' shoud not failing.\n", randElement) 83 | } 84 | t.Logf("Can not remove removed element '%v'.\n", randElement) 85 | if !matrixSet.Clear() { 86 | t.Errorf("Error: Clear matrix should be successful.\n", randElement) 87 | } 88 | t.Logf("The matrix is cleared.\n") 89 | if matrixSet.Len() > 0 { 90 | t.Errorf("Error: The length of matrix should be 0.\n", randElement) 91 | } 92 | t.Logf("The length of matrix is 0.\n") 93 | } 94 | 95 | func generateKey(x interface{}) string { 96 | xa := interface{}(x).([]int) 97 | xac := make([]int, len(xa)) 98 | copy(xac, xa) 99 | sort.Ints(xac) 100 | return fmt.Sprintf("%v", xac) 101 | } 102 | 103 | func compare(i, j interface{}) int { 104 | ia := interface{}(i).([]int) 105 | ja := interface{}(j).([]int) 106 | sort.Ints(ia) 107 | sort.Ints(ja) 108 | il := len(ia) 109 | jl := len(ja) 110 | result := 0 111 | if il < jl { 112 | result = -1 113 | } else if il > jl { 114 | result = 1 115 | } else { 116 | for i, iv := range ia { 117 | jv := ja[i] 118 | if iv != jv { 119 | if iv < jv { 120 | result = -1 121 | } else if iv > jv { 122 | result = 1 123 | } 124 | break 125 | } 126 | } 127 | } 128 | return result 129 | } 130 | -------------------------------------------------------------------------------- /src/chan1/oneway/phandler.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type Person struct { 9 | Name string 10 | Age uint8 11 | Address Addr 12 | } 13 | 14 | type Addr struct { 15 | city string 16 | district string 17 | } 18 | 19 | type PersonHandler interface { 20 | Batch(origs <-chan Person) <-chan Person 21 | Handle(orig *Person) 22 | } 23 | 24 | type PersonHandlerImpl struct{} 25 | 26 | func (handler PersonHandlerImpl) Batch(origs <-chan Person) <-chan Person { 27 | dests := make(chan Person, 100) 28 | go func() { 29 | for p := range origs { 30 | handler.Handle(&p) 31 | dests <- p 32 | } 33 | fmt.Println("All the information has been handled.") 34 | close(dests) 35 | }() 36 | return dests 37 | } 38 | 39 | func (handler PersonHandlerImpl) Handle(orig *Person) { 40 | if orig.Address.district == "Haidian" { 41 | orig.Address.district = "Shijingshan" 42 | } 43 | } 44 | 45 | var personTotal = 200 46 | 47 | var persons []Person = make([]Person, personTotal) 48 | 49 | var personCount int 50 | 51 | func init() { 52 | for i := 0; i < 200; i++ { 53 | name := fmt.Sprintf("%s%d", "P", i) 54 | p := Person{name, 32, Addr{"Beijing", "Haidian"}} 55 | persons[i] = p 56 | } 57 | } 58 | 59 | func main() { 60 | handler := getPersonHandler() 61 | origs := make(chan Person, 100) 62 | dests := handler.Batch(origs) 63 | fecthPerson(origs) 64 | sign := savePerson(dests) 65 | <-sign 66 | } 67 | 68 | func getPersonHandler() PersonHandler { 69 | return PersonHandlerImpl{} 70 | } 71 | 72 | func savePerson(dest <-chan Person) <-chan byte { 73 | sign := make(chan byte, 1) 74 | go func() { 75 | for { 76 | p, ok := <-dest 77 | if !ok { 78 | fmt.Println("All the information has been saved.") 79 | sign <- 0 80 | break 81 | } 82 | savePerson1(p) 83 | } 84 | }() 85 | return sign 86 | } 87 | 88 | func fecthPerson(origs chan<- Person) { 89 | origsCap := cap(origs) 90 | buffered := origsCap > 0 91 | goTicketTotal := origsCap / 2 92 | goTicket := initGoTicket(goTicketTotal) 93 | go func() { 94 | for { 95 | p, ok := fecthPerson1() 96 | if !ok { 97 | for { 98 | if !buffered || len(goTicket) == goTicketTotal { 99 | break 100 | } 101 | time.Sleep(time.Nanosecond) 102 | } 103 | fmt.Println("All the information has been fetched.") 104 | close(origs) 105 | break 106 | } 107 | if buffered { 108 | <-goTicket 109 | go func() { 110 | origs <- p 111 | goTicket <- 1 112 | }() 113 | } else { 114 | origs <- p 115 | } 116 | } 117 | }() 118 | } 119 | 120 | func initGoTicket(total int) chan byte { 121 | var goTicket chan byte 122 | if total == 0 { 123 | return goTicket 124 | } 125 | goTicket = make(chan byte, total) 126 | for i := 0; i < total; i++ { 127 | goTicket <- 1 128 | } 129 | return goTicket 130 | } 131 | 132 | func fecthPerson1() (Person, bool) { 133 | if personCount < personTotal { 134 | p := persons[personCount] 135 | personCount++ 136 | return p, true 137 | } 138 | return Person{}, false 139 | } 140 | 141 | func savePerson1(p Person) bool { 142 | return true 143 | } 144 | -------------------------------------------------------------------------------- /src/cnet/ctcp/base.go: -------------------------------------------------------------------------------- 1 | package ctcp 2 | 3 | import ( 4 | "net" 5 | "time" 6 | ) 7 | 8 | type TcpMessage struct { 9 | content string 10 | err error 11 | } 12 | 13 | func (self TcpMessage) Content() string { 14 | return self.content 15 | } 16 | 17 | func (self TcpMessage) Err() error { 18 | return self.err 19 | } 20 | 21 | func NewTcpMessage(content string, err error) TcpMessage { 22 | return TcpMessage{content: content, err: err} 23 | } 24 | 25 | type TcpListener interface { 26 | Init(addr string) error 27 | Listen(handler func(conn net.Conn)) error 28 | Close() bool 29 | Addr() net.Addr 30 | } 31 | 32 | type TcpSender interface { 33 | Init(remoteAddr string, timeout time.Duration) error 34 | Send(content string) error 35 | Receive(delim byte) <-chan TcpMessage 36 | Close() bool 37 | Addr() net.Addr 38 | RemoteAddr() net.Addr 39 | } 40 | -------------------------------------------------------------------------------- /src/cnet/ctcp/tcp.go: -------------------------------------------------------------------------------- 1 | package ctcp 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "errors" 7 | "logging" 8 | "net" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | const ( 14 | DELIMITER = '\t' 15 | ) 16 | 17 | var logger logging.Logger = logging.NewSimpleLogger() 18 | 19 | func Read(conn net.Conn, delim byte) (string, error) { 20 | readBytes := make([]byte, 1) 21 | var buffer bytes.Buffer 22 | for { 23 | _, err := conn.Read(readBytes) 24 | if err != nil { 25 | return "", err 26 | } 27 | readByte := readBytes[0] 28 | if readByte == DELIMITER { 29 | break 30 | } 31 | buffer.WriteByte(readByte) 32 | } 33 | return buffer.String(), nil 34 | } 35 | 36 | func Write(conn net.Conn, content string) (int, error) { 37 | writer := bufio.NewWriter(conn) 38 | number, err := writer.WriteString(content) 39 | if err == nil { 40 | err = writer.Flush() 41 | } 42 | return number, err 43 | } 44 | 45 | type AsyncTcpListener struct { 46 | listener net.Listener 47 | active bool 48 | lock *sync.Mutex 49 | } 50 | 51 | func (self *AsyncTcpListener) Init(addr string) error { 52 | self.lock.Lock() 53 | defer self.lock.Unlock() 54 | if self.active { 55 | return nil 56 | } 57 | ln, err := net.Listen("tcp", addr) 58 | if err != nil { 59 | return err 60 | } 61 | self.listener = ln 62 | self.active = true 63 | return nil 64 | } 65 | 66 | func (self *AsyncTcpListener) Listen(handler func(conn net.Conn)) error { 67 | if !self.active { 68 | return errors.New("Listen Error: Uninitialized listener!") 69 | } 70 | go func(active *bool) { 71 | for { 72 | if *active { 73 | return 74 | } 75 | conn, err := self.listener.Accept() 76 | if err != nil { 77 | logger.Errorf("Listener: Accept Request Error: %s\n", err) 78 | continue 79 | } 80 | go handler(conn) 81 | } 82 | }(&self.active) 83 | return nil 84 | } 85 | 86 | func (self *AsyncTcpListener) Close() bool { 87 | self.lock.Lock() 88 | defer self.lock.Unlock() 89 | if self.active { 90 | self.listener.Close() 91 | self.active = false 92 | return true 93 | } else { 94 | return false 95 | } 96 | } 97 | 98 | func (self *AsyncTcpListener) Addr() net.Addr { 99 | if self.active { 100 | return self.listener.Addr() 101 | } else { 102 | return nil 103 | } 104 | } 105 | 106 | func NewTcpListener() TcpListener { 107 | return &AsyncTcpListener{lock: new(sync.Mutex)} 108 | } 109 | 110 | type AsyncTcpSender struct { 111 | active bool 112 | lock *sync.Mutex 113 | conn net.Conn 114 | } 115 | 116 | func (self *AsyncTcpSender) Init(remoteAddr string, timeout time.Duration) error { 117 | self.lock.Lock() 118 | defer self.lock.Unlock() 119 | if !self.active { 120 | conn, err := net.DialTimeout("tcp", remoteAddr, timeout) 121 | if err != nil { 122 | return err 123 | } 124 | self.conn = conn 125 | self.active = true 126 | } 127 | return nil 128 | } 129 | 130 | func (self *AsyncTcpSender) Send(content string) error { 131 | self.lock.Lock() 132 | defer self.lock.Unlock() 133 | if !self.active { 134 | return errors.New("Send Error: Uninitialized sender!") 135 | } 136 | _, err := Write(self.conn, content) 137 | return err 138 | } 139 | 140 | func (self *AsyncTcpSender) Receive(delim byte) <-chan TcpMessage { 141 | respChan := make(chan TcpMessage, 1) 142 | go func(conn net.Conn, ch chan<- TcpMessage) { 143 | content, err := Read(conn, delim) 144 | ch <- NewTcpMessage(content, err) 145 | }(self.conn, respChan) 146 | return respChan 147 | } 148 | 149 | func (self *AsyncTcpSender) Addr() net.Addr { 150 | if self.active { 151 | return self.conn.LocalAddr() 152 | } else { 153 | return nil 154 | } 155 | } 156 | 157 | func (self *AsyncTcpSender) RemoteAddr() net.Addr { 158 | if self.active { 159 | return self.conn.RemoteAddr() 160 | } else { 161 | return nil 162 | } 163 | } 164 | 165 | func (self *AsyncTcpSender) Close() bool { 166 | self.lock.Lock() 167 | defer self.lock.Unlock() 168 | if self.active { 169 | self.conn.Close() 170 | self.active = false 171 | return true 172 | } else { 173 | return false 174 | } 175 | } 176 | 177 | func NewTcpSender() TcpSender { 178 | return &AsyncTcpSender{lock: new(sync.Mutex)} 179 | } 180 | -------------------------------------------------------------------------------- /src/cnet/ctcp/tcp_test.go: -------------------------------------------------------------------------------- 1 | package ctcp 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net" 7 | "runtime" 8 | "strings" 9 | "sync" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | const ( 15 | DELIM byte = '\t' 16 | ) 17 | 18 | var once sync.Once 19 | var benchmarkServerAddr string = "127.0.0.1:8081" 20 | var benchmarkListener TcpListener 21 | 22 | func TestPrimeFuncs(t *testing.T) { 23 | t.Parallel() 24 | showLog := false 25 | serverAddr := "127.0.0.1:8080" 26 | t.Logf("Test tcp listener & sender (serverAddr=%s)... %s\n", 27 | serverAddr, generateRuntimeInfo()) 28 | listener := generateTcpListener(serverAddr, showLog) 29 | if listener == nil { 30 | t.Fatalf("Listener startup failing! (addr=%s)!\n", serverAddr) 31 | } 32 | defer func() { 33 | if listener != nil { 34 | listener.Close() 35 | } 36 | }() 37 | if testing.Short() { 38 | multiSend(serverAddr, "SenderT", 1, (2 * time.Second), showLog) 39 | } else { 40 | var wg sync.WaitGroup 41 | wg.Add(2) 42 | go func() { 43 | defer wg.Done() 44 | multiSend(serverAddr, "SenderT1", 2, (2 * time.Second), showLog) 45 | }() 46 | go func() { 47 | defer wg.Done() 48 | multiSend(serverAddr, "SenderT2", 1, (2 * time.Second), showLog) 49 | }() 50 | wg.Wait() 51 | } 52 | } 53 | 54 | func BenchmarkPrimeFuncs(t *testing.B) { 55 | showLog := false 56 | t.Logf("Benchmark tcp listener & sender (serverAddr=%s)... %s\n", 57 | benchmarkServerAddr, generateRuntimeInfo()) 58 | once.Do(startupListenerOnce) 59 | if benchmarkListener == nil { 60 | t.Errorf("Listener startup failing! (addr=%s)!\n", benchmarkServerAddr) 61 | } 62 | //if t.N == 1 { 63 | // fmt.Printf("\nIterations (N): %d\n", t.N) 64 | //} else { 65 | // fmt.Printf("Iterations (N): %d\n", t.N) 66 | //} 67 | for i := 0; i < t.N; i++ { 68 | multiSend(benchmarkServerAddr, "SenderB", 3, (2 * time.Second), showLog) 69 | } 70 | if benchmarkListener != nil { 71 | benchmarkListener.Close() 72 | } 73 | } 74 | 75 | func startupListenerOnce() { 76 | benchmarkListener = generateTcpListener(benchmarkServerAddr, false) 77 | } 78 | 79 | func generateTcpListener(serverAddr string, showLog bool) TcpListener { 80 | var listener TcpListener = NewTcpListener() 81 | var hasError bool 82 | if showLog { 83 | fmt.Printf("Start Listening at address %s ...\n", serverAddr) 84 | } 85 | err := listener.Init(serverAddr) 86 | if err != nil { 87 | hasError = true 88 | fmt.Errorf("Listener Init error: %s", err) 89 | } 90 | err = listener.Listen(requestHandler(showLog)) 91 | if err != nil { 92 | hasError = true 93 | fmt.Errorf("Listener Listen error: %s", err) 94 | } 95 | if !hasError { 96 | return listener 97 | } else { 98 | if listener != nil { 99 | listener.Close() 100 | } 101 | return nil 102 | } 103 | } 104 | 105 | func multiSend( 106 | remoteAddr string, 107 | clientName string, 108 | number int, 109 | timeout time.Duration, 110 | showLog bool) { 111 | sender := NewTcpSender() 112 | if showLog { 113 | fmt.Printf("Initializing sender (%s) (remote address: %s, timeout: %d) ...", clientName, remoteAddr, timeout) 114 | } 115 | err := sender.Init(remoteAddr, timeout) 116 | if err != nil { 117 | fmt.Errorf("%s: Init Error: %s\n", clientName, err) 118 | return 119 | } 120 | if number <= 0 { 121 | number = 5 122 | } 123 | for i := 0; i < number; i++ { 124 | content := generateTestContent(fmt.Sprintf("%s-%d", clientName, i)) 125 | if showLog { 126 | fmt.Printf("%s: Send content: '%s'\n", clientName, content) 127 | } 128 | err := sender.Send(content) 129 | if err != nil { 130 | fmt.Errorf("%s: Send Error: %s\n", clientName, err) 131 | } 132 | respChan := sender.Receive(DELIM) 133 | var resp TcpMessage 134 | timeoutChan := time.After(1 * time.Second) 135 | select { 136 | case resp = <-respChan: 137 | case <-timeoutChan: 138 | break 139 | } 140 | if err = resp.Err(); err != nil { 141 | fmt.Errorf("Sender: Receive Error: %s\n", err) 142 | } else { 143 | if showLog { 144 | respContent := resp.Content() 145 | fmt.Printf("%s: Received response: '%s'\n", clientName, respContent) 146 | } 147 | } 148 | } 149 | content := generateTestContent(fmt.Sprintf("%s-quit", clientName)) 150 | if showLog { 151 | fmt.Printf("%s: Send content: '%s'\n", clientName, content) 152 | } 153 | err = sender.Send(content) 154 | if err != nil { 155 | fmt.Errorf("%s: Send Error: %s\n", clientName, err) 156 | } 157 | sender.Close() 158 | } 159 | 160 | func generateTestContent(content string) string { 161 | var respBuffer bytes.Buffer 162 | respBuffer.WriteString(strings.TrimSpace(content)) 163 | respBuffer.WriteByte(DELIM) 164 | return respBuffer.String() 165 | } 166 | 167 | func requestHandler(showLog bool) func(conn net.Conn) { 168 | return func(conn net.Conn) { 169 | for { 170 | content, err := Read(conn, DELIM) 171 | if err != nil { 172 | fmt.Errorf("Listener Read error: %s", err) 173 | } else { 174 | if showLog { 175 | fmt.Printf("Listener: Received content: '%s'\n", content) 176 | } 177 | content = strings.TrimSpace(content) 178 | if strings.HasSuffix(content, "quit") { 179 | if showLog { 180 | fmt.Println("Listener: Quit!") 181 | } 182 | break 183 | } 184 | resp := generateTestContent(fmt.Sprintf("Resp: %s", content)) 185 | n, err := Write(conn, resp) 186 | if err != nil { 187 | fmt.Errorf("Listener Write error: %s", err) 188 | } 189 | if showLog { 190 | fmt.Println("Listener: Send response: '%s' (n=%d)\n", resp, n) 191 | } 192 | } 193 | } 194 | } 195 | } 196 | 197 | func generateRuntimeInfo() string { 198 | return fmt.Sprintf("[GOMAXPROCS=%d, NUM_CPU=%d, NUM_GOROUTINE=%d]", 199 | runtime.GOMAXPROCS(-1), runtime.NumCPU(), runtime.NumGoroutine()) 200 | } 201 | -------------------------------------------------------------------------------- /src/helper/ds/showds.go: -------------------------------------------------------------------------------- 1 | // Show the specified directory structure 2 | package main 3 | 4 | import ( 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path" 9 | "strings" 10 | ) 11 | 12 | const ( 13 | INDENT = " " 14 | ) 15 | 16 | var ( 17 | rootPath string 18 | ) 19 | 20 | func init() { 21 | flag.StringVar(&rootPath, "p", "", "The path of target directory.") 22 | } 23 | 24 | func showFiles(basePath string, prefix string, showAll bool) error { 25 | base, err := os.Open(basePath) 26 | if err != nil { 27 | return err 28 | } 29 | subs, err := base.Readdir(-1) 30 | if err != nil { 31 | return err 32 | } 33 | for _, v := range subs { 34 | fi := v.(os.FileInfo) 35 | fp := fi.Name() 36 | if strings.HasPrefix(fp, ".") && !showAll { 37 | continue 38 | } 39 | if fi.IsDir() { 40 | absFp := path.Join(basePath, fp) 41 | if err != nil { 42 | return err 43 | } 44 | fmt.Printf("%s/\n", prefix+fp) 45 | err = showFiles(absFp, INDENT+prefix, showAll) 46 | if err != nil { 47 | return err 48 | } 49 | } else { 50 | fmt.Printf("%s\n", prefix+fp) 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | func main() { 57 | flag.Parse() 58 | if len(rootPath) == 0 { 59 | defaultPath, err := os.Getwd() 60 | if err != nil { 61 | fmt.Println("GetwdError:", err) 62 | return 63 | } 64 | rootPath = defaultPath 65 | } 66 | fmt.Printf("%s:\n", rootPath) 67 | err := showFiles(rootPath, INDENT, false) 68 | if err != nil { 69 | fmt.Println("showFilesError:", err) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/helper/pds/showpds.go: -------------------------------------------------------------------------------- 1 | // Show the dependency structure of specified package 2 | package main 3 | 4 | import ( 5 | "basic/prof" 6 | "bytes" 7 | "errors" 8 | "flag" 9 | "fmt" 10 | "os" 11 | "pkgtool" 12 | "runtime/debug" 13 | "strings" 14 | ) 15 | 16 | const ( 17 | ARROWS = "->" 18 | ) 19 | 20 | var ( 21 | pkgImportPathFlag string 22 | ) 23 | 24 | func init() { 25 | flag.StringVar(&pkgImportPathFlag, "p", "", "The path of target package.") 26 | } 27 | 28 | func main() { 29 | prof.Start() 30 | defer func() { 31 | prof.Stop() 32 | if err := recover(); err != nil { 33 | fmt.Errorf("FATAL ERROR: %s", err) 34 | debug.PrintStack() 35 | } 36 | }() 37 | flag.Parse() 38 | pkgImportPath := getPkgImportPath() 39 | pn := pkgtool.NewPkgNode(pkgImportPath) 40 | fmt.Printf("The package node of '%s': %v\n", pkgImportPath, *pn) 41 | err := pn.Grow() 42 | if err != nil { 43 | fmt.Printf("GROW ERROR: %s\n", err) 44 | } 45 | fmt.Printf("The dependency structure of package '%s':\n", pkgImportPath) 46 | ShowDepStruct(pn, "") 47 | } 48 | 49 | func ShowDepStruct(pnode *pkgtool.PkgNode, prefix string) { 50 | var buf bytes.Buffer 51 | buf.WriteString(prefix) 52 | importPath := pnode.ImportPath() 53 | buf.WriteString(importPath) 54 | deps := pnode.Deps() 55 | //fmt.Printf("P_NODE: '%s', DEP_LEN: %d\n", importPath, len(deps)) 56 | if len(deps) == 0 { 57 | fmt.Printf("%s\n", buf.String()) 58 | return 59 | } 60 | buf.WriteString(ARROWS) 61 | for _, v := range deps { 62 | ShowDepStruct(v, buf.String()) 63 | } 64 | } 65 | 66 | func getPkgImportPath() string { 67 | if len(pkgImportPathFlag) > 0 { 68 | return pkgImportPathFlag 69 | } 70 | fmt.Printf("The flag p is invalid, use current dir as package import path.") 71 | currentDir, err := os.Getwd() 72 | if err != nil { 73 | panic(err) 74 | } 75 | srcDirs := pkgtool.GetSrcDirs(false) 76 | var importPath string 77 | for _, v := range srcDirs { 78 | if strings.HasPrefix(currentDir, v) { 79 | importPath = currentDir[len(v):] 80 | break 81 | } 82 | } 83 | if strings.TrimSpace(importPath) == "" { 84 | panic(errors.New("Can not parse the import path!")) 85 | } 86 | return importPath 87 | } 88 | -------------------------------------------------------------------------------- /src/loadgen/gen.go: -------------------------------------------------------------------------------- 1 | package loadgen 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | lib "loadgen/lib" 8 | "math" 9 | "time" 10 | ) 11 | 12 | // 载荷发生器的实现。 13 | type myGenerator struct { 14 | caller lib.Caller // 调用器。 15 | timeoutNs time.Duration // 处理超时时间，单位：纳秒。 16 | lps uint32 // 每秒载荷量。 17 | durationNs time.Duration // 负载持续时间，单位：纳秒。 18 | concurrency uint32 // 并发量。 19 | tickets lib.GoTickets // Goroutine票池。 20 | stopSign chan byte // 停止信号的传递通道。 21 | cancelSign byte // 取消发送后续结果的信号。 22 | endSign chan uint64 // 完结信号的传递通道，同时被用于传递调用执行计数。 23 | callCount uint64 // 调用执行计数。 24 | status lib.GenStatus // 状态。 25 | resultCh chan *lib.CallResult // 调用结果通道。 26 | } 27 | 28 | func NewGenerator( 29 | caller lib.Caller, 30 | timeoutNs time.Duration, 31 | lps uint32, 32 | durationNs time.Duration, 33 | resultCh chan *lib.CallResult) (lib.Generator, error) { 34 | logger.Infoln("New a load generator...") 35 | logger.Infoln("Checking the parameters...") 36 | var errMsg string 37 | if caller == nil { 38 | errMsg = fmt.Sprintln("Invalid caller!") 39 | } 40 | if timeoutNs == 0 { 41 | errMsg = fmt.Sprintln("Invalid timeoutNs!") 42 | } 43 | if lps == 0 { 44 | errMsg = fmt.Sprintln("Invalid lps(load per second)!") 45 | } 46 | if durationNs == 0 { 47 | errMsg = fmt.Sprintln("Invalid durationNs!") 48 | } 49 | if resultCh == nil { 50 | errMsg = fmt.Sprintln("Invalid result channel!") 51 | } 52 | if errMsg != "" { 53 | return nil, errors.New(errMsg) 54 | } 55 | gen := &myGenerator{ 56 | caller: caller, 57 | timeoutNs: timeoutNs, 58 | lps: lps, 59 | durationNs: durationNs, 60 | stopSign: make(chan byte, 1), 61 | cancelSign: 0, 62 | status: lib.STATUS_ORIGINAL, 63 | resultCh: resultCh, 64 | } 65 | logger.Infof("Passed. (timeoutNs=%v, lps=%d, durationNs=%v)", 66 | timeoutNs, lps, durationNs) 67 | err := gen.init() 68 | if err != nil { 69 | return nil, err 70 | } 71 | return gen, nil 72 | } 73 | 74 | func (gen *myGenerator) init() error { 75 | logger.Infoln("Initializing the load generator...") 76 | // 载荷的并发量 ≈ 载荷的响应超时时间 / 载荷的发送间隔时间 77 | var total64 int64 = int64(gen.timeoutNs)/int64(1e9/gen.lps) + 1 78 | if total64 > math.MaxInt32 { 79 | total64 = math.MaxInt32 80 | } 81 | gen.concurrency = uint32(total64) 82 | tickets, err := lib.NewGoTickets(gen.concurrency) 83 | if err != nil { 84 | return err 85 | } 86 | gen.tickets = tickets 87 | logger.Infof("Initialized. (concurrency=%d)", gen.concurrency) 88 | return nil 89 | } 90 | 91 | func (gen *myGenerator) interact(rawReq *lib.RawReq) *lib.RawResp { 92 | if rawReq == nil { 93 | return &lib.RawResp{Id: -1, Err: errors.New("Invalid raw request.")} 94 | } 95 | start := time.Now().Nanosecond() 96 | resp, err := gen.caller.Call(rawReq.Req, gen.timeoutNs) 97 | end := time.Now().Nanosecond() 98 | elapsedTime := time.Duration(end - start) 99 | var rawResp lib.RawResp 100 | if err != nil { 101 | errMsg := fmt.Sprintf("Sync Call Error: %s.", err) 102 | rawResp = lib.RawResp{ 103 | Id: rawReq.Id, 104 | Err: errors.New(errMsg), 105 | Elapse: elapsedTime} 106 | } else { 107 | rawResp = lib.RawResp{ 108 | Id: rawReq.Id, 109 | Resp: resp, 110 | Elapse: elapsedTime} 111 | } 112 | return &rawResp 113 | } 114 | 115 | func (gen *myGenerator) asyncCall() { 116 | gen.tickets.Take() 117 | go func() { 118 | defer func() { 119 | if p := recover(); p != nil { 120 | err, ok := interface{}(p).(error) 121 | var buff bytes.Buffer 122 | buff.WriteString("Async Call Panic! (") 123 | if ok { 124 | buff.WriteString("error: ") 125 | buff.WriteString(err.Error()) 126 | } else { 127 | buff.WriteString("clue: ") 128 | buff.WriteString(fmt.Sprintf("%v", p)) 129 | } 130 | buff.WriteString(")") 131 | errMsg := buff.String() 132 | logger.Fatalln(errMsg) 133 | result := &lib.CallResult{ 134 | Id: -1, 135 | Code: lib.RESULT_CODE_FATAL_CALL, 136 | Msg: errMsg} 137 | gen.sendResult(result) 138 | } 139 | }() 140 | rawReq := gen.caller.BuildReq() 141 | var timeout bool 142 | timer := time.AfterFunc(gen.timeoutNs, func() { 143 | timeout = true 144 | result := &lib.CallResult{ 145 | Id: rawReq.Id, 146 | Req: rawReq, 147 | Code: lib.RESULT_CODE_WARNING_CALL_TIMEOUT, 148 | Msg: fmt.Sprintf("Timeout! (expected: < %v)", gen.timeoutNs)} 149 | gen.sendResult(result) 150 | }) 151 | rawResp := gen.interact(&rawReq) 152 | if !timeout { 153 | timer.Stop() 154 | var result *lib.CallResult 155 | if rawResp.Err != nil { 156 | result = &lib.CallResult{ 157 | Id: rawResp.Id, 158 | Req: rawReq, 159 | Code: lib.RESULT_CODE_ERROR_CALL, 160 | Msg: rawResp.Err.Error(), 161 | Elapse: rawResp.Elapse} 162 | } else { 163 | result = gen.caller.CheckResp(rawReq, *rawResp) 164 | result.Elapse = rawResp.Elapse 165 | } 166 | gen.sendResult(result) 167 | } 168 | gen.tickets.Return() 169 | }() 170 | } 171 | 172 | func (gen *myGenerator) sendResult(result *lib.CallResult) bool { 173 | if gen.status == lib.STATUS_STARTED && gen.cancelSign == 0 { 174 | gen.resultCh <- result 175 | return true 176 | } 177 | logger.Warnf("Ignore result: %s.\n", 178 | fmt.Sprintf( 179 | "Id=%d, Code=%d, Msg=%s, Elapse=%v", 180 | result.Id, result.Code, result.Msg, result.Elapse)) 181 | return false 182 | } 183 | 184 | func (gen *myGenerator) handleStopSign(callCount uint64) { 185 | gen.cancelSign = 1 186 | logger.Infof("Closing result channel...") 187 | close(gen.resultCh) 188 | gen.endSign <- callCount 189 | gen.endSign <- callCount 190 | } 191 | 192 | func (gen *myGenerator) genLoad(throttle <-chan time.Time) { 193 | callCount := uint64(0) 194 | Loop: 195 | for ; ; callCount++ { 196 | select { 197 | case <-gen.stopSign: 198 | gen.handleStopSign(callCount) 199 | break Loop 200 | default: 201 | } 202 | gen.asyncCall() 203 | if gen.lps > 0 { 204 | select { 205 | case <-throttle: 206 | case <-gen.stopSign: 207 | gen.handleStopSign(callCount) 208 | break Loop 209 | } 210 | } 211 | } 212 | } 213 | 214 | func (gen *myGenerator) Start() { 215 | logger.Infoln("Starting load generator...") 216 | 217 | // 设定节流阀 218 | var throttle <-chan time.Time 219 | if gen.lps > 0 { 220 | interval := time.Duration(1e9 / gen.lps) 221 | logger.Infof("Setting throttle (%v)...", interval) 222 | throttle = time.Tick(interval) 223 | } 224 | 225 | // 初始化停止信号 226 | go func() { 227 | time.AfterFunc(gen.durationNs, func() { 228 | logger.Infof("Stopping load generator...") 229 | gen.stopSign <- 0 230 | }) 231 | }() 232 | 233 | // 初始化完结信号通道 234 | gen.endSign = make(chan uint64, 2) 235 | 236 | // 初始化调用执行计数 237 | gen.callCount = 0 238 | 239 | // 设置已启动状态 240 | gen.status = lib.STATUS_STARTED 241 | 242 | go func() { 243 | // 生成载荷 244 | logger.Infoln("Generating loads...") 245 | gen.genLoad(throttle) 246 | 247 | // 接收调用执行计数 248 | callCount := <-gen.endSign 249 | gen.status = lib.STATUS_STOPPED 250 | logger.Infof("Stopped. (callCount=%d)\n", callCount) 251 | }() 252 | } 253 | 254 | func (gen *myGenerator) Stop() (uint64, bool) { 255 | if gen.stopSign == nil { 256 | return 0, false 257 | } 258 | if gen.status != lib.STATUS_STARTED { 259 | return 0, false 260 | } 261 | gen.status = lib.STATUS_STOPPED 262 | gen.stopSign <- 1 263 | callCount := <-gen.endSign 264 | return callCount, true 265 | } 266 | 267 | func (gen *myGenerator) Status() lib.GenStatus { 268 | return gen.status 269 | } 270 | -------------------------------------------------------------------------------- /src/loadgen/gen_test.go: -------------------------------------------------------------------------------- 1 | package loadgen 2 | 3 | import ( 4 | loadgenlib "loadgen/lib" 5 | thelper "loadgen/testhelper" 6 | "runtime" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | var printDetail = false 12 | 13 | func TestStart(t *testing.T) { 14 | // 设置P最大数量 15 | runtime.GOMAXPROCS(runtime.NumCPU()) 16 | 17 | // 初始化服务器 18 | server := thelper.NewTcpServer() 19 | defer server.Close() 20 | serverAddr := "127.0.0.1:8080" 21 | t.Logf("Startup TCP server(%s)...\n", serverAddr) 22 | err := server.Listen(serverAddr) 23 | if err != nil { 24 | t.Fatalf("TCP Server startup failing! (addr=%s)!\n", serverAddr) 25 | t.FailNow() 26 | } 27 | 28 | // 初始化调用器 29 | comm := thelper.NewTcpComm(serverAddr) 30 | 31 | // 初始化载荷发生器 32 | resultCh := make(chan *loadgenlib.CallResult, 50) 33 | timeoutNs := 3 * time.Millisecond 34 | lps := uint32(200) 35 | durationNs := 12 * time.Second 36 | t.Logf("Initialize load generator (timeoutNs=%v, lps=%d, durationNs=%v)...", 37 | timeoutNs, lps, durationNs) 38 | gen, err := NewGenerator( 39 | comm, 40 | timeoutNs, 41 | lps, 42 | durationNs, 43 | resultCh) 44 | if err != nil { 45 | t.Fatalf("Load generator initialization failing: %s.\n", 46 | err) 47 | t.FailNow() 48 | } 49 | 50 | // 开始！ 51 | t.Log("Start load generator...") 52 | gen.Start() 53 | 54 | // 显示结果 55 | countMap := make(map[loadgenlib.ResultCode]int) 56 | for r := range resultCh { 57 | countMap[r.Code] = countMap[r.Code] + 1 58 | if printDetail { 59 | t.Logf("Result: Id=%d, Code=%d, Msg=%s, Elapse=%v.\n", 60 | r.Id, r.Code, r.Msg, r.Elapse) 61 | } 62 | } 63 | 64 | var total int 65 | t.Log("Code Count:") 66 | for k, v := range countMap { 67 | codePlain := loadgenlib.GetResultCodePlain(k) 68 | t.Logf(" Code plain: %s (%d), Count: %d.\n", 69 | codePlain, k, v) 70 | total += v 71 | } 72 | 73 | t.Logf("Total load: %d.\n", total) 74 | successCount := countMap[loadgenlib.RESULT_CODE_SUCCESS] 75 | tps := float64(successCount) / float64(durationNs/1e9) 76 | t.Logf("Loads per second: %d; Treatments per second: %f.\n", lps, tps) 77 | } 78 | 79 | func TestStop(t *testing.T) { 80 | // 设置P最大数量 81 | runtime.GOMAXPROCS(runtime.NumCPU()) 82 | 83 | // 初始化服务器 84 | server := thelper.NewTcpServer() 85 | defer server.Close() 86 | serverAddr := "127.0.0.1:8081" 87 | t.Logf("Startup TCP server(%s)...\n", serverAddr) 88 | err := server.Listen(serverAddr) 89 | if err != nil { 90 | t.Fatalf("TCP Server startup failing! (addr=%s)!\n", serverAddr) 91 | t.FailNow() 92 | } 93 | 94 | // 初始化调用器 95 | comm := thelper.NewTcpComm(serverAddr) 96 | 97 | // 初始化载荷发生器 98 | resultCh := make(chan *loadgenlib.CallResult, 50) 99 | timeoutNs := 3 * time.Millisecond 100 | lps := uint32(200) 101 | durationNs := 10 * time.Second 102 | t.Logf("Initialize load generator (timeoutNs=%v, lps=%d, durationNs=%v)...", 103 | timeoutNs, lps, durationNs) 104 | gen, err := NewGenerator( 105 | comm, 106 | timeoutNs, 107 | lps, 108 | durationNs, 109 | resultCh) 110 | if err != nil { 111 | t.Fatalf("Load generator initialization failing: %s.\n", 112 | err) 113 | t.FailNow() 114 | } 115 | 116 | // 开始！ 117 | t.Log("Start load generator...") 118 | gen.Start() 119 | 120 | // 显示调用结果 121 | countMap := make(map[loadgenlib.ResultCode]int) 122 | count := 0 123 | for r := range resultCh { 124 | countMap[r.Code] = countMap[r.Code] + 1 125 | if printDetail { 126 | t.Logf("Result: Id=%d, Code=%d, Msg=%s, Elapse=%v.\n", 127 | r.Id, r.Code, r.Msg, r.Elapse) 128 | } 129 | count++ 130 | if count > 3 { 131 | gen.Stop() 132 | } 133 | } 134 | 135 | var total int 136 | t.Log("Code Count:") 137 | for k, v := range countMap { 138 | codePlain := loadgenlib.GetResultCodePlain(k) 139 | t.Logf(" Code plain: %s (%d), Count: %d.\n", 140 | codePlain, k, v) 141 | total += v 142 | } 143 | 144 | t.Logf("Total load: %d.\n", total) 145 | successCount := countMap[loadgenlib.RESULT_CODE_SUCCESS] 146 | tps := float64(successCount) / float64(durationNs/1e9) 147 | t.Logf("Loads per second: %d; Treatments per second: %f.\n", lps, tps) 148 | } 149 | -------------------------------------------------------------------------------- /src/loadgen/lib/base.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // 原生请求的结构。 8 | type RawReq struct { 9 | Id int64 10 | Req []byte 11 | } 12 | 13 | // 原生响应的结构。 14 | type RawResp struct { 15 | Id int64 16 | Resp []byte 17 | Err error 18 | Elapse time.Duration 19 | } 20 | 21 | type ResultCode int 22 | 23 | // 保留 1 ~ 1000 给载荷承受者使用。 24 | const ( 25 | RESULT_CODE_SUCCESS = 0 // 成功。 26 | RESULT_CODE_WARNING_CALL_TIMEOUT ResultCode = 1001 // 调用超时警告。 27 | RESULT_CODE_ERROR_CALL ResultCode = 2001 // 调用错误。 28 | RESULT_CODE_ERROR_RESPONSE ResultCode = 2002 // 响应内容错误。 29 | RESULT_CODE_ERROR_CALEE ResultCode = 2003 // 被调用方（被测软件）的内部错误。 30 | RESULT_CODE_FATAL_CALL ResultCode = 3001 // 调用过程中发生了致命错误！ 31 | ) 32 | 33 | func GetResultCodePlain(code ResultCode) string { 34 | var codePlain string 35 | switch code { 36 | case RESULT_CODE_SUCCESS: 37 | codePlain = "Success" 38 | case RESULT_CODE_WARNING_CALL_TIMEOUT: 39 | codePlain = "Call Timeout Warning" 40 | case RESULT_CODE_ERROR_CALL: 41 | codePlain = "Call Error" 42 | case RESULT_CODE_ERROR_RESPONSE: 43 | codePlain = "Response Error" 44 | case RESULT_CODE_ERROR_CALEE: 45 | codePlain = "Callee Error" 46 | case RESULT_CODE_FATAL_CALL: 47 | codePlain = "Call Fatal Error" 48 | default: 49 | codePlain = "Unknown result code" 50 | } 51 | return codePlain 52 | } 53 | 54 | // 调用结果的结构。 55 | type CallResult struct { 56 | Id int64 // ID。 57 | Req RawReq // 原生请求。 58 | Resp RawResp // 原生响应。 59 | Code ResultCode // 响应代码。 60 | Msg string // 结果成因的简述。 61 | Elapse time.Duration // 耗时。 62 | } 63 | 64 | // 载荷发生器的状态的类型。 65 | type GenStatus int 66 | 67 | const ( 68 | STATUS_ORIGINAL GenStatus = 0 69 | STATUS_STARTED GenStatus = 1 70 | STATUS_STOPPED GenStatus = 2 71 | ) 72 | 73 | // 载荷发生器的接口。 74 | type Generator interface { 75 | // 启动载荷发生器。 76 | Start() 77 | // 停止载荷发生器。 78 | // 第一个结果值代表已发载荷总数，且仅在第二个结果值为true时有效。 79 | // 第二个结果值代表是否成功将载荷发生器转变为已停止状态。 80 | Stop() (uint64, bool) 81 | // 获取状态。 82 | Status() GenStatus 83 | } 84 | -------------------------------------------------------------------------------- /src/loadgen/lib/caller.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // 调用器的接口。 8 | type Caller interface { 9 | // 构建请求。 10 | BuildReq() RawReq 11 | // 调用。 12 | Call(req []byte, timeoutNs time.Duration) ([]byte, error) 13 | // 检查响应。 14 | CheckResp(rawReq RawReq, rawResp RawResp) *CallResult 15 | } 16 | -------------------------------------------------------------------------------- /src/loadgen/lib/tickets.go: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | // Goroutine票池的接口。 9 | type GoTickets interface { 10 | // 拿走一张票。 11 | Take() 12 | // 归还一张票。 13 | Return() 14 | // 票池是否已被激活。 15 | Active() bool 16 | // 票的总数。 17 | Total() uint32 18 | // 剩余的票数。 19 | Remainder() uint32 20 | } 21 | 22 | // Goroutine票池的实现。 23 | type myGoTickets struct { 24 | total uint32 // 票的总数。 25 | ticketCh chan byte // 票的容器。 26 | active bool // 票池是否已被激活。 27 | } 28 | 29 | func NewGoTickets(total uint32) (GoTickets, error) { 30 | gt := myGoTickets{} 31 | if !gt.init(total) { 32 | errMsg := 33 | fmt.Sprintf("The goroutine ticket pool can NOT be initialized! (total=%d)\n", total) 34 | return nil, errors.New(errMsg) 35 | } 36 | return >, nil 37 | } 38 | 39 | func (gt *myGoTickets) init(total uint32) bool { 40 | if gt.active { 41 | return false 42 | } 43 | if total == 0 { 44 | return false 45 | } 46 | ch := make(chan byte, total) 47 | n := int(total) 48 | for i := 0; i < n; i++ { 49 | ch <- 1 50 | } 51 | gt.ticketCh = ch 52 | gt.total = total 53 | gt.active = true 54 | return true 55 | } 56 | 57 | func (gt *myGoTickets) Take() { 58 | <-gt.ticketCh 59 | } 60 | 61 | func (gt *myGoTickets) Return() { 62 | gt.ticketCh <- 1 63 | } 64 | 65 | func (gt *myGoTickets) Active() bool { 66 | return gt.active 67 | } 68 | 69 | func (gt *myGoTickets) Total() uint32 { 70 | return gt.total 71 | } 72 | 73 | func (gt *myGoTickets) Remainder() uint32 { 74 | return uint32(len(gt.ticketCh)) 75 | } 76 | -------------------------------------------------------------------------------- /src/loadgen/log.go: -------------------------------------------------------------------------------- 1 | package loadgen 2 | 3 | import ( 4 | "logging" 5 | ) 6 | 7 | var logger logging.Logger 8 | 9 | func init() { 10 | logger = logging.NewSimpleLogger() 11 | } 12 | -------------------------------------------------------------------------------- /src/loadgen/testhelper/comm.go: -------------------------------------------------------------------------------- 1 | package testhelper 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/json" 7 | "fmt" 8 | loadgenlib "loadgen/lib" 9 | "math/rand" 10 | "net" 11 | "time" 12 | ) 13 | 14 | const ( 15 | DELIM = '\n' 16 | ) 17 | 18 | type TcpComm struct { 19 | addr string 20 | } 21 | 22 | func NewTcpComm(addr string) loadgenlib.Caller { 23 | return &TcpComm{addr: addr} 24 | } 25 | 26 | func (comm *TcpComm) BuildReq() loadgenlib.RawReq { 27 | id := time.Now().UnixNano() 28 | sreq := ServerReq{ 29 | Id: id, 30 | Operands: []int{ 31 | int(rand.Int31n(1000) + 1), 32 | int(rand.Int31n(1000) + 1)}, 33 | Operator: func() string { 34 | op := []string{"+", "-", "*", "/"} 35 | return op[rand.Int31n(100)%4] 36 | }(), 37 | } 38 | bytes, err := json.Marshal(sreq) 39 | if err != nil { 40 | panic(err) 41 | } 42 | rawReq := loadgenlib.RawReq{Id: id, Req: bytes} 43 | return rawReq 44 | } 45 | 46 | func (comm *TcpComm) Call(req []byte, timeoutNs time.Duration) ([]byte, error) { 47 | conn, err := net.DialTimeout("tcp", comm.addr, timeoutNs) 48 | if err != nil { 49 | return nil, err 50 | } 51 | _, err = write(conn, req, DELIM) 52 | if err != nil { 53 | return nil, err 54 | } 55 | return read(conn, DELIM) 56 | } 57 | 58 | func (comm *TcpComm) CheckResp( 59 | rawReq loadgenlib.RawReq, rawResp loadgenlib.RawResp) *loadgenlib.CallResult { 60 | var commResult loadgenlib.CallResult 61 | commResult.Id = rawResp.Id 62 | commResult.Req = rawReq 63 | commResult.Resp = rawResp 64 | var sreq ServerReq 65 | err := json.Unmarshal(rawReq.Req, &sreq) 66 | if err != nil { 67 | commResult.Code = loadgenlib.RESULT_CODE_FATAL_CALL 68 | commResult.Msg = 69 | fmt.Sprintf("Incorrectly formatted Req: %s!\n", string(rawReq.Req)) 70 | return &commResult 71 | } 72 | var sresp ServerResp 73 | err = json.Unmarshal(rawResp.Resp, &sresp) 74 | if err != nil { 75 | commResult.Code = loadgenlib.RESULT_CODE_ERROR_RESPONSE 76 | commResult.Msg = 77 | fmt.Sprintf("Incorrectly formatted Resp: %s!\n", string(rawResp.Resp)) 78 | return &commResult 79 | } 80 | if sresp.Id != sreq.Id { 81 | commResult.Code = loadgenlib.RESULT_CODE_ERROR_RESPONSE 82 | commResult.Msg = 83 | fmt.Sprintf("Inconsistent raw id! (%d != %d)\n", rawReq.Id, rawResp.Id) 84 | return &commResult 85 | } 86 | if sresp.Err != nil { 87 | commResult.Code = loadgenlib.RESULT_CODE_ERROR_CALEE 88 | commResult.Msg = 89 | fmt.Sprintf("Abnormal server: %s!\n", sresp.Err) 90 | return &commResult 91 | } 92 | if sresp.Result != op(sreq.Operands, sreq.Operator) { 93 | commResult.Code = loadgenlib.RESULT_CODE_ERROR_RESPONSE 94 | commResult.Msg = 95 | fmt.Sprintf( 96 | "Incorrect result: %s!\n", 97 | genFormula(sreq.Operands, sreq.Operator, sresp.Result, false)) 98 | return &commResult 99 | } 100 | commResult.Code = loadgenlib.RESULT_CODE_SUCCESS 101 | commResult.Msg = fmt.Sprintf("Success. (%s)", sresp.Formula) 102 | return &commResult 103 | } 104 | 105 | func read(conn net.Conn, delim byte) ([]byte, error) { 106 | readBytes := make([]byte, 1) 107 | var buffer bytes.Buffer 108 | for { 109 | _, err := conn.Read(readBytes) 110 | if err != nil { 111 | return nil, err 112 | } 113 | readByte := readBytes[0] 114 | if readByte == delim { 115 | break 116 | } 117 | buffer.WriteByte(readByte) 118 | } 119 | return buffer.Bytes(), nil 120 | } 121 | 122 | func write(conn net.Conn, content []byte, delim byte) (int, error) { 123 | writer := bufio.NewWriter(conn) 124 | n, err := writer.Write(content) 125 | if err == nil { 126 | writer.WriteByte(delim) 127 | } 128 | if err == nil { 129 | err = writer.Flush() 130 | } 131 | return n, err 132 | } 133 | -------------------------------------------------------------------------------- /src/loadgen/testhelper/server.go: -------------------------------------------------------------------------------- 1 | package testhelper 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "net" 9 | "runtime" 10 | "strconv" 11 | "sync" 12 | ) 13 | 14 | type ServerReq struct { 15 | Id int64 16 | Operands []int 17 | Operator string 18 | } 19 | 20 | type ServerResp struct { 21 | Id int64 22 | Formula string 23 | Result int 24 | Err error 25 | } 26 | 27 | func op(operands []int, operator string) int { 28 | var result int 29 | switch { 30 | case operator == "+": 31 | for _, v := range operands { 32 | if result == 0 { 33 | result = v 34 | } else { 35 | result += v 36 | } 37 | } 38 | case operator == "-": 39 | for _, v := range operands { 40 | if result == 0 { 41 | result = v 42 | } else { 43 | result -= v 44 | } 45 | } 46 | case operator == "*": 47 | for _, v := range operands { 48 | if result == 0 { 49 | result = v 50 | } else { 51 | result *= v 52 | } 53 | } 54 | case operator == "/": 55 | for _, v := range operands { 56 | if result == 0 { 57 | result = v 58 | } else { 59 | result /= v 60 | } 61 | } 62 | } 63 | return result 64 | } 65 | 66 | func genFormula(operands []int, operator string, result int, equal bool) string { 67 | var buff bytes.Buffer 68 | n := len(operands) 69 | for i := 0; i < n; i++ { 70 | if i > 0 { 71 | buff.WriteString(" ") 72 | buff.WriteString(operator) 73 | buff.WriteString(" ") 74 | } 75 | 76 | buff.WriteString(strconv.Itoa(operands[i])) 77 | } 78 | if equal { 79 | buff.WriteString(" = ") 80 | } else { 81 | buff.WriteString(" != ") 82 | } 83 | buff.WriteString(strconv.Itoa(result)) 84 | return buff.String() 85 | } 86 | 87 | func reqHandler(conn net.Conn) { 88 | var errMsg string 89 | var sresp ServerResp 90 | req, err := read(conn, DELIM) 91 | if err != nil { 92 | errMsg = fmt.Sprintf("Server: Req Read Error: %s", err) 93 | } else { 94 | var sreq ServerReq 95 | err := json.Unmarshal(req, &sreq) 96 | if err != nil { 97 | errMsg = fmt.Sprintf("Server: Req Unmarshal Error: %s", err) 98 | } else { 99 | sresp.Id = sreq.Id 100 | sresp.Result = op(sreq.Operands, sreq.Operator) 101 | sresp.Formula = 102 | genFormula(sreq.Operands, sreq.Operator, sresp.Result, true) 103 | } 104 | } 105 | if errMsg != "" { 106 | sresp.Err = errors.New(errMsg) 107 | } 108 | bytes, err := json.Marshal(sresp) 109 | if err != nil { 110 | fmt.Errorf("Server: Resp Marshal Error: %s", err) 111 | } 112 | _, err = write(conn, bytes, DELIM) 113 | if err != nil { 114 | fmt.Errorf("Server: Resp Write error: %s", err) 115 | } 116 | } 117 | 118 | type TcpServer struct { 119 | listener net.Listener 120 | active bool 121 | lock *sync.Mutex 122 | } 123 | 124 | func (self *TcpServer) init(addr string) error { 125 | self.lock.Lock() 126 | defer self.lock.Unlock() 127 | if self.active { 128 | return nil 129 | } 130 | ln, err := net.Listen("tcp", addr) 131 | if err != nil { 132 | return err 133 | } 134 | self.listener = ln 135 | self.active = true 136 | return nil 137 | } 138 | 139 | func (self *TcpServer) Listen(addr string) error { 140 | err := self.init(addr) 141 | if err != nil { 142 | return err 143 | } 144 | go func(active *bool) { 145 | for { 146 | conn, err := self.listener.Accept() 147 | if err != nil { 148 | fmt.Errorf("Server: Request Acception Error: %s\n", err) 149 | continue 150 | } 151 | go reqHandler(conn) 152 | runtime.Gosched() 153 | } 154 | }(&self.active) 155 | return nil 156 | } 157 | 158 | func (self *TcpServer) Close() bool { 159 | self.lock.Lock() 160 | defer self.lock.Unlock() 161 | if self.active { 162 | self.listener.Close() 163 | self.active = false 164 | return true 165 | } else { 166 | return false 167 | } 168 | } 169 | 170 | func NewTcpServer() *TcpServer { 171 | return &TcpServer{lock: new(sync.Mutex)} 172 | } 173 | -------------------------------------------------------------------------------- /src/logging/base.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "runtime" 7 | "strings" 8 | ) 9 | 10 | type Position uint 11 | 12 | const ( 13 | POSITION_SINGLE Position = 1 14 | POSITION_IN_MANAGER Position = 2 15 | ) 16 | 17 | func init() { 18 | log.SetFlags(log.LstdFlags) 19 | } 20 | 21 | type Logger interface { 22 | GetPosition() Position 23 | SetPosition(pos Position) 24 | Error(v ...interface{}) string 25 | Errorf(format string, v ...interface{}) string 26 | Errorln(v ...interface{}) string 27 | Fatal(v ...interface{}) string 28 | Fatalf(format string, v ...interface{}) string 29 | Fatalln(v ...interface{}) string 30 | Info(v ...interface{}) string 31 | Infof(format string, v ...interface{}) string 32 | Infoln(v ...interface{}) string 33 | Panic(v ...interface{}) string 34 | Panicf(format string, v ...interface{}) string 35 | Panicln(v ...interface{}) string 36 | Warn(v ...interface{}) string 37 | Warnf(format string, v ...interface{}) string 38 | Warnln(v ...interface{}) string 39 | } 40 | 41 | func getInvokerLocation(skipNumber int) string { 42 | pc, file, line, ok := runtime.Caller(skipNumber) 43 | if !ok { 44 | return "" 45 | } 46 | simpleFileName := "" 47 | if index := strings.LastIndex(file, "/"); index > 0 { 48 | simpleFileName = file[index+1 : len(file)] 49 | } 50 | funcPath := "" 51 | funcPtr := runtime.FuncForPC(pc) 52 | if funcPtr != nil { 53 | funcPath = funcPtr.Name() 54 | } 55 | return fmt.Sprintf("%s : (%s:%d)", funcPath, simpleFileName, line) 56 | } 57 | 58 | func generateLogContent( 59 | logTag LogTag, 60 | pos Position, 61 | format string, 62 | v ...interface{}) string { 63 | skipNumber := int(pos) + 2 64 | baseInfo := 65 | fmt.Sprintf("%s %s - ", logTag.Prefix(), getInvokerLocation(skipNumber)) 66 | var result string 67 | if len(format) > 0 { 68 | result = fmt.Sprintf((baseInfo + format), v...) 69 | } else { 70 | vLen := len(v) 71 | params := make([]interface{}, (vLen + 1)) 72 | params[0] = baseInfo 73 | for i := 1; i <= vLen; i++ { 74 | params[i] = v[i-1] 75 | } 76 | result = fmt.Sprint(params...) 77 | } 78 | return result 79 | } 80 | 81 | func NewSimpleLogger() Logger { 82 | logger := &ConsoleLogger{} 83 | logger.SetPosition(POSITION_SINGLE) 84 | return logger 85 | } 86 | 87 | func NewLogger(loggers []Logger) Logger { 88 | for _, logger := range loggers { 89 | logger.SetPosition(POSITION_IN_MANAGER) 90 | } 91 | return &LogManager{loggers: loggers} 92 | } 93 | -------------------------------------------------------------------------------- /src/logging/console_logger.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "log" 5 | ) 6 | 7 | type ConsoleLogger struct { 8 | position Position 9 | } 10 | 11 | func (logger *ConsoleLogger) GetPosition() Position { 12 | return logger.position 13 | } 14 | 15 | func (logger *ConsoleLogger) SetPosition(pos Position) { 16 | logger.position = pos 17 | } 18 | 19 | func (logger *ConsoleLogger) Error(v ...interface{}) string { 20 | content := generateLogContent(getErrorLogTag(), logger.GetPosition(), "", v...) 21 | log.Print(content) 22 | return content 23 | } 24 | 25 | func (logger *ConsoleLogger) Errorf(format string, v ...interface{}) string { 26 | content := generateLogContent(getErrorLogTag(), logger.GetPosition(), format, v...) 27 | log.Print(content) 28 | return content 29 | } 30 | 31 | func (logger *ConsoleLogger) Errorln(v ...interface{}) string { 32 | content := generateLogContent(getErrorLogTag(), logger.GetPosition(), "", v...) 33 | log.Println(content) 34 | return content 35 | } 36 | 37 | func (logger *ConsoleLogger) Fatal(v ...interface{}) string { 38 | content := generateLogContent(getFatalLogTag(), logger.GetPosition(), "", v...) 39 | log.Print(content) 40 | return content 41 | } 42 | 43 | func (logger *ConsoleLogger) Fatalf(format string, v ...interface{}) string { 44 | content := generateLogContent(getFatalLogTag(), logger.GetPosition(), format, v...) 45 | log.Print(content) 46 | return content 47 | } 48 | 49 | func (logger *ConsoleLogger) Fatalln(v ...interface{}) string { 50 | content := generateLogContent(getFatalLogTag(), logger.GetPosition(), "", v...) 51 | log.Println(content) 52 | return content 53 | } 54 | 55 | func (logger *ConsoleLogger) Info(v ...interface{}) string { 56 | content := generateLogContent(getInfoLogTag(), logger.GetPosition(), "", v...) 57 | log.Print(content) 58 | return content 59 | } 60 | 61 | func (logger *ConsoleLogger) Infof(format string, v ...interface{}) string { 62 | content := generateLogContent(getInfoLogTag(), logger.GetPosition(), format, v...) 63 | log.Print(content) 64 | return content 65 | } 66 | 67 | func (logger *ConsoleLogger) Infoln(v ...interface{}) string { 68 | content := generateLogContent(getInfoLogTag(), logger.GetPosition(), "", v...) 69 | log.Println(content) 70 | return content 71 | } 72 | 73 | func (logger *ConsoleLogger) Panic(v ...interface{}) string { 74 | content := generateLogContent(getPanicLogTag(), logger.GetPosition(), "", v...) 75 | log.Print(content) 76 | return content 77 | } 78 | 79 | func (logger *ConsoleLogger) Panicf(format string, v ...interface{}) string { 80 | content := generateLogContent(getPanicLogTag(), logger.GetPosition(), format, v...) 81 | log.Print(content) 82 | return content 83 | } 84 | 85 | func (logger *ConsoleLogger) Panicln(v ...interface{}) string { 86 | content := generateLogContent(getPanicLogTag(), logger.GetPosition(), "", v...) 87 | log.Println(content) 88 | return content 89 | } 90 | 91 | func (logger *ConsoleLogger) Warn(v ...interface{}) string { 92 | content := generateLogContent(getWarnLogTag(), logger.GetPosition(), "", v...) 93 | log.Print(content) 94 | return content 95 | } 96 | 97 | func (logger *ConsoleLogger) Warnf(format string, v ...interface{}) string { 98 | content := generateLogContent(getWarnLogTag(), logger.GetPosition(), format, v...) 99 | log.Print(content) 100 | return content 101 | } 102 | 103 | func (logger *ConsoleLogger) Warnln(v ...interface{}) string { 104 | content := generateLogContent(getWarnLogTag(), logger.GetPosition(), "", v...) 105 | log.Println(content) 106 | return content 107 | } 108 | -------------------------------------------------------------------------------- /src/logging/log_manager.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | type LogManager struct { 4 | loggers []Logger 5 | } 6 | 7 | func (logger *LogManager) GetPosition() Position { 8 | return POSITION_SINGLE 9 | } 10 | 11 | func (logger *LogManager) SetPosition(pos Position) {} 12 | 13 | func (self *LogManager) Error(v ...interface{}) string { 14 | var content string 15 | for _, logger := range self.loggers { 16 | content = logger.Error(v...) 17 | } 18 | return content 19 | } 20 | 21 | func (self *LogManager) Errorf(format string, v ...interface{}) string { 22 | var content string 23 | for _, logger := range self.loggers { 24 | content = logger.Errorf(format, v...) 25 | } 26 | return content 27 | } 28 | 29 | func (self *LogManager) Errorln(v ...interface{}) string { 30 | var content string 31 | for _, logger := range self.loggers { 32 | content = logger.Errorln(v...) 33 | } 34 | return content 35 | } 36 | 37 | func (self *LogManager) Fatal(v ...interface{}) string { 38 | var content string 39 | for _, logger := range self.loggers { 40 | content = logger.Fatal(v...) 41 | } 42 | return content 43 | } 44 | 45 | func (self *LogManager) Fatalf(format string, v ...interface{}) string { 46 | var content string 47 | for _, logger := range self.loggers { 48 | content = logger.Fatalf(format, v...) 49 | } 50 | return content 51 | } 52 | 53 | func (self *LogManager) Fatalln(v ...interface{}) string { 54 | var content string 55 | for _, logger := range self.loggers { 56 | content = logger.Fatalln(v...) 57 | } 58 | return content 59 | } 60 | 61 | func (self *LogManager) Info(v ...interface{}) string { 62 | var content string 63 | for _, logger := range self.loggers { 64 | content = logger.Info(v...) 65 | } 66 | return content 67 | } 68 | 69 | func (self *LogManager) Infof(format string, v ...interface{}) string { 70 | var content string 71 | for _, logger := range self.loggers { 72 | content = logger.Infof(format, v...) 73 | } 74 | return content 75 | } 76 | 77 | func (self *LogManager) Infoln(v ...interface{}) string { 78 | var content string 79 | for _, logger := range self.loggers { 80 | content = logger.Infoln(v...) 81 | } 82 | return content 83 | } 84 | 85 | func (self *LogManager) Panic(v ...interface{}) string { 86 | var content string 87 | for _, logger := range self.loggers { 88 | content = logger.Panic(v...) 89 | } 90 | return content 91 | } 92 | 93 | func (self *LogManager) Panicf(format string, v ...interface{}) string { 94 | var content string 95 | for _, logger := range self.loggers { 96 | content = logger.Panicf(format, v...) 97 | } 98 | return content 99 | } 100 | 101 | func (self *LogManager) Panicln(v ...interface{}) string { 102 | var content string 103 | for _, logger := range self.loggers { 104 | content = logger.Panicln(v...) 105 | } 106 | return content 107 | } 108 | 109 | func (self *LogManager) Warn(v ...interface{}) string { 110 | var content string 111 | for _, logger := range self.loggers { 112 | content = logger.Warn(v...) 113 | } 114 | return content 115 | } 116 | 117 | func (self *LogManager) Warnf(format string, v ...interface{}) string { 118 | var content string 119 | for _, logger := range self.loggers { 120 | content = logger.Warnf(format, v...) 121 | } 122 | return content 123 | } 124 | 125 | func (self *LogManager) Warnln(v ...interface{}) string { 126 | var content string 127 | for _, logger := range self.loggers { 128 | content = logger.Warnln(v...) 129 | } 130 | return content 131 | } 132 | -------------------------------------------------------------------------------- /src/logging/logger_test.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "runtime/debug" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | var count = 0 12 | 13 | func TestConsoleLogger(t *testing.T) { 14 | defer func() { 15 | if err := recover(); err != nil { 16 | debug.PrintStack() 17 | t.Errorf("Fatal Error: %s\n", err) 18 | } 19 | }() 20 | logger := &ConsoleLogger{} 21 | logger.SetDefaultInvokingNumber() 22 | expectedInvokingNumber := uint(1) 23 | currentInvokingNumber := logger.getInvokingNumber() 24 | if currentInvokingNumber != expectedInvokingNumber { 25 | t.Errorf("The current invoking number %d should be %d!", currentInvokingNumber, expectedInvokingNumber) 26 | } 27 | testLogger(t, logger) 28 | } 29 | 30 | func TestLogManager(t *testing.T) { 31 | defer func() { 32 | if err := recover(); err != nil { 33 | debug.PrintStack() 34 | t.Errorf("Fatal Error: %s\n", err) 35 | } 36 | }() 37 | logger := &LogManager{loggers: []Logger{&ConsoleLogger{invokingNumber: 2}}} 38 | testLogger(t, logger) 39 | } 40 | 41 | func testLogger(t *testing.T, logger Logger) { 42 | var format string 43 | var content string 44 | var logContent string 45 | 46 | format = "" 47 | logContent = "" 48 | content = logger.Error(logContent) 49 | checkContent(t, getErrorLogTag(), content, format, logContent) 50 | 51 | format = "<%s>" 52 | logContent = "Errorf" 53 | content = logger.Errorf(format, logContent) 54 | checkContent(t, getErrorLogTag(), content, format, logContent) 55 | 56 | format = "" 57 | logContent = "" 58 | content = logger.Errorln(logContent) 59 | checkContent(t, getErrorLogTag(), content, format, logContent) 60 | 61 | format = "" 62 | logContent = "" 63 | content = logger.Fatal(logContent) 64 | checkContent(t, getFatalLogTag(), content, format, logContent) 65 | 66 | format = "<%s>" 67 | logContent = "Fatalf" 68 | content = logger.Fatalf(format, logContent) 69 | checkContent(t, getFatalLogTag(), content, format, logContent) 70 | 71 | format = "" 72 | logContent = "" 73 | content = logger.Fatalln(logContent) 74 | checkContent(t, getFatalLogTag(), content, format, logContent) 75 | 76 | format = "" 77 | logContent = "" 78 | content = logger.Info(logContent) 79 | checkContent(t, getInfoLogTag(), content, format, logContent) 80 | 81 | format = "<%s>" 82 | logContent = "Infof" 83 | content = logger.Infof(format, logContent) 84 | checkContent(t, getInfoLogTag(), content, format, logContent) 85 | 86 | format = "" 87 | logContent = "" 88 | content = logger.Infoln(logContent) 89 | checkContent(t, getInfoLogTag(), content, format, logContent) 90 | 91 | format = "" 92 | logContent = "" 93 | content = logger.Panic(logContent) 94 | checkContent(t, getPanicLogTag(), content, format, logContent) 95 | 96 | format = "<%s>" 97 | logContent = "Panicf" 98 | content = logger.Panicf(format, logContent) 99 | checkContent(t, getPanicLogTag(), content, format, logContent) 100 | 101 | format = "" 102 | logContent = "" 103 | content = logger.Panicln(logContent) 104 | checkContent(t, getPanicLogTag(), content, format, logContent) 105 | 106 | format = "" 107 | logContent = "" 108 | content = logger.Warn(logContent) 109 | checkContent(t, getWarnLogTag(), content, format, logContent) 110 | 111 | format = "<%s>" 112 | logContent = "Warnf" 113 | content = logger.Warnf(format, logContent) 114 | checkContent(t, getWarnLogTag(), content, format, logContent) 115 | 116 | format = "" 117 | logContent = "" 118 | content = logger.Warnln(logContent) 119 | checkContent(t, getWarnLogTag(), content, format, logContent) 120 | } 121 | 122 | func checkContent(t *testing.T, logTag LogTag, content string, format string, logContents ...interface{}) { 123 | var prefixBuffer bytes.Buffer 124 | prefixBuffer.WriteString(logTag.Prefix()) 125 | prefixBuffer.WriteString(" go_lib/logging.testLogger : (logger_test.go:") 126 | prefix := prefixBuffer.String() 127 | var suffixBuffer bytes.Buffer 128 | suffixBuffer.WriteString(") - ") 129 | if len(format) == 0 { 130 | suffixBuffer.WriteString(fmt.Sprint(logContents...)) 131 | } else { 132 | suffixBuffer.WriteString(fmt.Sprintf(format, logContents...)) 133 | } 134 | suffix := suffixBuffer.String() 135 | if !strings.HasPrefix(content, prefix) { 136 | t.Errorf("The content '%s' should has prefix '%s'! ", content, prefix) 137 | } 138 | if !strings.HasSuffix(content, suffix) { 139 | t.Errorf("The content '%s' should has suffix '%s'! ", content, suffix) 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /src/logging/tag.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | const ( 4 | ERROR_LOG_KEY = "ERROR" 5 | FATAL_LOG_KEY = "FATAL" 6 | INFO_LOG_KEY = "INFO" 7 | PANIC_LOG_KEY = "PANIC" 8 | WARN_LOG_KEY = "WARN" 9 | ) 10 | 11 | type LogTag struct { 12 | name string 13 | prefix string 14 | } 15 | 16 | func (self *LogTag) Name() string { 17 | return self.name 18 | } 19 | 20 | func (self *LogTag) Prefix() string { 21 | return self.prefix 22 | } 23 | 24 | var logTagMap map[string]LogTag = map[string]LogTag{ 25 | ERROR_LOG_KEY: LogTag{name: ERROR_LOG_KEY, prefix: "[" + ERROR_LOG_KEY + "]"}, 26 | FATAL_LOG_KEY: LogTag{name: FATAL_LOG_KEY, prefix: "[" + FATAL_LOG_KEY + "]"}, 27 | INFO_LOG_KEY: LogTag{name: INFO_LOG_KEY, prefix: "[" + INFO_LOG_KEY + "]"}, 28 | PANIC_LOG_KEY: LogTag{name: PANIC_LOG_KEY, prefix: "[" + PANIC_LOG_KEY + "]"}, 29 | WARN_LOG_KEY: LogTag{name: WARN_LOG_KEY, prefix: "[" + WARN_LOG_KEY + "]"}, 30 | } 31 | 32 | func getErrorLogTag() LogTag { 33 | return logTagMap[ERROR_LOG_KEY] 34 | } 35 | 36 | func getFatalLogTag() LogTag { 37 | return logTagMap[FATAL_LOG_KEY] 38 | } 39 | 40 | func getInfoLogTag() LogTag { 41 | return logTagMap[INFO_LOG_KEY] 42 | } 43 | 44 | func getPanicLogTag() LogTag { 45 | return logTagMap[PANIC_LOG_KEY] 46 | } 47 | 48 | func getWarnLogTag() LogTag { 49 | return logTagMap[WARN_LOG_KEY] 50 | } 51 | -------------------------------------------------------------------------------- /src/multiproc/apipe/apipe.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "os/exec" 9 | ) 10 | 11 | func main() { 12 | demo1() 13 | fmt.Println() 14 | demo2() 15 | } 16 | 17 | func demo2() { 18 | fmt.Println("Run command `ps aux | grep apipe`: ") 19 | cmd1 := exec.Command("ps", "aux") 20 | cmd2 := exec.Command("grep", "apipe") 21 | stdout1, err := cmd1.StdoutPipe() 22 | if err != nil { 23 | fmt.Printf("Error: Can not obtain the stdout pipe for command: %s", err) 24 | return 25 | } 26 | if err := cmd1.Start(); err != nil { 27 | fmt.Printf("Error: The command can not running: %s\n", err) 28 | return 29 | } 30 | outputBuf1 := bufio.NewReader(stdout1) 31 | stdin2, err := cmd2.StdinPipe() 32 | if err != nil { 33 | fmt.Printf("Error: Can not obtain the stdin pipe for command: %s\n", err) 34 | return 35 | } 36 | outputBuf1.WriteTo(stdin2) 37 | var outputBuf2 bytes.Buffer 38 | cmd2.Stdout = &outputBuf2 39 | if err := cmd2.Start(); err != nil { 40 | fmt.Printf("Error: The command can not be startup: %s\n", err) 41 | return 42 | } 43 | err = stdin2.Close() 44 | if err != nil { 45 | fmt.Printf("Error: Can not close the stdio pipe: %s\n", err) 46 | return 47 | } 48 | if err := cmd2.Wait(); err != nil { 49 | fmt.Printf("Error: Can not wait for the command: %s\n", err) 50 | return 51 | } 52 | fmt.Printf("%s\n", outputBuf2.Bytes()) 53 | } 54 | 55 | func demo1() { 56 | useBufferIo := false 57 | fmt.Println("Run command `echo -n \"My first command from golang.\"`: ") 58 | cmd0 := exec.Command("echo", "-n", "My first command from golang.") 59 | stdout0, err := cmd0.StdoutPipe() 60 | if err != nil { 61 | fmt.Printf("Error: Can not obtain the stdout pipe for command No.0: %s\n", err) 62 | return 63 | } 64 | if err := cmd0.Start(); err != nil { 65 | fmt.Printf("Error: The command No.0 can not be startup: %s\n", err) 66 | return 67 | } 68 | if !useBufferIo { 69 | var outputBuf0 bytes.Buffer 70 | for { 71 | tempOutput := make([]byte, 5) 72 | n, err := stdout0.Read(tempOutput) 73 | if err != nil { 74 | if err == io.EOF { 75 | break 76 | } else { 77 | fmt.Printf("Error: Can not read data from the pipe: %s\n", err) 78 | return 79 | } 80 | } 81 | if n > 0 { 82 | outputBuf0.Write(tempOutput[:n]) 83 | } 84 | } 85 | fmt.Printf("%s\n", outputBuf0.String()) 86 | } else { 87 | outputBuf0 := bufio.NewReader(stdout0) 88 | output0, _, err := outputBuf0.ReadLine() 89 | if err != nil { 90 | fmt.Printf("Error: Can not read data from the pipe: %s\n", err) 91 | return 92 | } 93 | fmt.Printf("%s\n", string(output0)) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/multiproc/npipe/npipe.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "time" 8 | ) 9 | 10 | func main() { 11 | fileBasedPipe() 12 | inMemorySyncPipe() 13 | } 14 | 15 | func fileBasedPipe() { 16 | reader, writer, err := os.Pipe() 17 | if err != nil { 18 | fmt.Printf("Error: Can not create the named pipe: %s\n", err) 19 | } 20 | go func() { 21 | output := make([]byte, 100) 22 | n, err := reader.Read(output) 23 | if err != nil { 24 | fmt.Printf("Error: Can not read data from the named pipe: %s\n", err) 25 | } 26 | fmt.Printf("Read %d byte(s). [file-based pipe]\n", n) 27 | }() 28 | input := make([]byte, 26) 29 | for i := 65; i <= 90; i++ { 30 | input[i-65] = byte(i) 31 | } 32 | n, err := writer.Write(input) 33 | if err != nil { 34 | fmt.Printf("Error: Can not write data to the named pipe: %s\n", err) 35 | } 36 | fmt.Printf("Written %d byte(s). [file-based pipe]\n", n) 37 | time.Sleep(200 * time.Millisecond) 38 | } 39 | 40 | func inMemorySyncPipe() { 41 | reader, writer := io.Pipe() 42 | go func() { 43 | output := make([]byte, 100) 44 | n, err := reader.Read(output) 45 | if err != nil { 46 | fmt.Printf("Error: Can not read data from the named pipe: %s\n", err) 47 | } 48 | fmt.Printf("Read %d byte(s). [in-memory pipe]\n", n) 49 | }() 50 | input := make([]byte, 26) 51 | for i := 65; i <= 90; i++ { 52 | input[i-65] = byte(i) 53 | } 54 | n, err := writer.Write(input) 55 | if err != nil { 56 | fmt.Printf("Error: Can not write data to the named pipe: %s\n", err) 57 | } 58 | fmt.Printf("Written %d byte(s). [in-memory pipe]\n", n) 59 | time.Sleep(200 * time.Millisecond) 60 | } 61 | -------------------------------------------------------------------------------- /src/multiproc/signal/mysignal.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "os" 9 | "os/exec" 10 | "os/signal" 11 | "runtime/debug" 12 | "strconv" 13 | "strings" 14 | "sync" 15 | "syscall" 16 | "time" 17 | ) 18 | 19 | func main() { 20 | go func() { 21 | time.Sleep(5 * time.Second) 22 | sigSendingDemo() 23 | }() 24 | sigHandleDemo() 25 | } 26 | 27 | func sigHandleDemo() { 28 | sigRecv1 := make(chan os.Signal, 1) 29 | sigs1 := []os.Signal{syscall.SIGINT, syscall.SIGQUIT} 30 | fmt.Printf("Set notification for %s... [sigRecv1]\n", sigs1) 31 | signal.Notify(sigRecv1, sigs1...) 32 | sigRecv2 := make(chan os.Signal, 1) 33 | sigs2 := []os.Signal{syscall.SIGQUIT} 34 | fmt.Printf("Set notification for %s... [sigRecv2]\n", sigs2) 35 | signal.Notify(sigRecv2, sigs2...) 36 | 37 | var wg sync.WaitGroup 38 | wg.Add(2) 39 | go func() { 40 | for sig := range sigRecv1 { 41 | fmt.Printf("Received a signal from sigRecv1: %s\n", sig) 42 | } 43 | fmt.Printf("End. [sigRecv1]\n") 44 | wg.Done() 45 | }() 46 | go func() { 47 | for sig := range sigRecv2 { 48 | fmt.Printf("Received a signal from sigRecv2: %s\n", sig) 49 | } 50 | fmt.Printf("End. [sigRecv2]\n") 51 | wg.Done() 52 | }() 53 | 54 | fmt.Println("Wait for 2 seconds... ") 55 | time.Sleep(2 * time.Second) 56 | fmt.Printf("Stop notification...") 57 | signal.Stop(sigRecv1) 58 | close(sigRecv1) 59 | fmt.Printf("done. [sigRecv1]\n") 60 | wg.Wait() 61 | } 62 | 63 | func sigSendingDemo() { 64 | defer func() { 65 | if err := recover(); err != nil { 66 | fmt.Printf("Fatal Error: %s\n", err) 67 | debug.PrintStack() 68 | } 69 | }() 70 | // ps aux | grep "mysignal" | grep -v "grep" | grep -v "go run" | awk '{print $2}' 71 | cmds := []*exec.Cmd{ 72 | exec.Command("ps", "aux"), 73 | exec.Command("grep", "mysignal"), 74 | exec.Command("grep", "-v", "grep"), 75 | exec.Command("grep", "-v", "go run"), 76 | exec.Command("awk", "{print $2}"), 77 | } 78 | output, err := runCmds(cmds) 79 | if err != nil { 80 | fmt.Printf("Command Execution Error: %s\n", err) 81 | return 82 | } 83 | pids, err := getPids(output) 84 | if err != nil { 85 | fmt.Printf("PID Parsing Error: %s\n", err) 86 | return 87 | } 88 | fmt.Printf("Target PID(s):\n%v\n", pids) 89 | for _, pid := range pids { 90 | proc, err := os.FindProcess(pid) 91 | if err != nil { 92 | fmt.Printf("Process Finding Error: %s\n", err) 93 | return 94 | } 95 | sig := syscall.SIGQUIT 96 | fmt.Printf("Send signal '%s' to the process (pid=%d)...\n", sig, pid) 97 | err = proc.Signal(sig) 98 | if err != nil { 99 | fmt.Printf("Signal Sending Error: %s\n", err) 100 | return 101 | } 102 | } 103 | } 104 | 105 | func getPids(strs []string) ([]int, error) { 106 | pids := make([]int, 0) 107 | for _, str := range strs { 108 | pid, err := strconv.Atoi(strings.TrimSpace(str)) 109 | if err != nil { 110 | return nil, err 111 | } 112 | pids = append(pids, pid) 113 | } 114 | return pids, nil 115 | } 116 | 117 | func runCmds(cmds []*exec.Cmd) ([]string, error) { 118 | if cmds == nil || len(cmds) == 0 { 119 | return nil, errors.New("The cmd slice is invalid!") 120 | } 121 | first := true 122 | var output []byte 123 | var err error 124 | for _, cmd := range cmds { 125 | fmt.Printf("Run command: %v ...\n", getCmdPlaintext(cmd)) 126 | if !first { 127 | var stdinBuf bytes.Buffer 128 | stdinBuf.Write(output) 129 | cmd.Stdin = &stdinBuf 130 | } 131 | var stdoutBuf bytes.Buffer 132 | cmd.Stdout = &stdoutBuf 133 | if err = cmd.Start(); err != nil { 134 | return nil, getError(err, cmd) 135 | } 136 | if err = cmd.Wait(); err != nil { 137 | return nil, getError(err, cmd) 138 | } 139 | output = stdoutBuf.Bytes() 140 | //fmt.Printf("Output:\n%s\n", string(output)) 141 | if first { 142 | first = false 143 | } 144 | } 145 | lines := make([]string, 0) 146 | var outputBuf bytes.Buffer 147 | outputBuf.Write(output) 148 | for { 149 | line, err := outputBuf.ReadBytes('\n') 150 | if err != nil { 151 | if err == io.EOF { 152 | break 153 | } else { 154 | return nil, getError(err, nil) 155 | } 156 | } 157 | lines = append(lines, string(line)) 158 | } 159 | return lines, nil 160 | } 161 | 162 | func getCmdPlaintext(cmd *exec.Cmd) string { 163 | var buf bytes.Buffer 164 | buf.WriteString(cmd.Path) 165 | for _, arg := range cmd.Args[1:] { 166 | buf.WriteRune(' ') 167 | buf.WriteString(arg) 168 | } 169 | return buf.String() 170 | } 171 | 172 | func getError(err error, cmd *exec.Cmd, extraInfo ...string) error { 173 | var errMsg string 174 | if cmd != nil { 175 | errMsg = fmt.Sprintf("%s [%s %v]", err, (*cmd).Path, (*cmd).Args) 176 | } else { 177 | errMsg = fmt.Sprintf("%s", err) 178 | } 179 | if len(extraInfo) > 0 { 180 | errMsg = fmt.Sprintf("%s (%v)", errMsg, extraInfo) 181 | } 182 | return errors.New(errMsg) 183 | } 184 | -------------------------------------------------------------------------------- /src/multiproc/socket/tcpsock.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "math" 9 | "math/rand" 10 | "net" 11 | "strconv" 12 | "sync" 13 | "time" 14 | ) 15 | 16 | const ( 17 | SERVER_NETWORK = "tcp" 18 | SERVER_ADDRESS = "127.0.0.1:8085" 19 | DELIMITER = '\t' 20 | ) 21 | 22 | var wg sync.WaitGroup 23 | 24 | var logSn = 1 25 | 26 | func printLog(format string, args ...interface{}) { 27 | fmt.Printf("%d: %s", logSn, fmt.Sprintf(format, args...)) 28 | logSn++ 29 | } 30 | 31 | func convertToInt32(str string) (int32, error) { 32 | num, err := strconv.Atoi(str) 33 | if err != nil { 34 | printLog(fmt.Sprintf("Parse Error: %s\n", err)) 35 | return 0, errors.New(fmt.Sprintf("'%s' is not integer!", str)) 36 | } 37 | if num > math.MaxInt32 || num < math.MinInt32 { 38 | printLog(fmt.Sprintf("Convert Error: The integer %s is too large/small.\n", num)) 39 | return 0, errors.New(fmt.Sprintf("'%s' is not 32-bit integer!", num)) 40 | } 41 | return int32(num), nil 42 | } 43 | 44 | func cbrt(param int32) float64 { 45 | return math.Cbrt(float64(param)) 46 | } 47 | 48 | // 千万不要使用这个版本的read函数！ 49 | //func read(conn net.Conn) (string, error) { 50 | // reader := bufio.NewReader(conn) 51 | // readBytes, err := reader.ReadBytes(DELIMITER) 52 | // if err != nil { 53 | // return "", err 54 | // } 55 | // return string(readBytes[:len(readBytes)-1]), nil 56 | //} 57 | 58 | func read(conn net.Conn) (string, error) { 59 | readBytes := make([]byte, 1) 60 | var buffer bytes.Buffer 61 | for { 62 | _, err := conn.Read(readBytes) 63 | if err != nil { 64 | return "", err 65 | } 66 | readByte := readBytes[0] 67 | if readByte == DELIMITER { 68 | break 69 | } 70 | buffer.WriteByte(readByte) 71 | } 72 | return buffer.String(), nil 73 | } 74 | 75 | func write(conn net.Conn, content string) (int, error) { 76 | var buffer bytes.Buffer 77 | buffer.WriteString(content) 78 | buffer.WriteByte(DELIMITER) 79 | return conn.Write(buffer.Bytes()) 80 | } 81 | 82 | func serverGo() { 83 | var listener net.Listener 84 | listener, err := net.Listen(SERVER_NETWORK, SERVER_ADDRESS) 85 | if err != nil { 86 | printLog("Listen Error: %s\n", err) 87 | return 88 | } 89 | defer listener.Close() 90 | printLog("Got listener for the server. (local address: %s)\n", 91 | listener.Addr()) 92 | for { 93 | conn, err := listener.Accept() // 阻塞直至新连接到来 94 | if err != nil { 95 | printLog("Accept Error: %s\n", err) 96 | } 97 | printLog("Established a connection with a client application. (remote address: %s)\n", 98 | conn.RemoteAddr()) 99 | go handleConn(conn) 100 | } 101 | } 102 | 103 | func handleConn(conn net.Conn) { 104 | defer func() { 105 | conn.Close() 106 | wg.Done() 107 | }() 108 | for { 109 | conn.SetReadDeadline(time.Now().Add(10 * time.Second)) 110 | strReq, err := read(conn) 111 | if err != nil { 112 | if err == io.EOF { 113 | printLog("The connection is closed by another side. (Server)\n") 114 | } else { 115 | printLog("Read Error: %s (Server)\n", err) 116 | } 117 | break 118 | } 119 | printLog("Received request: %s (Server)\n", strReq) 120 | i32Req, err := convertToInt32(strReq) 121 | if err != nil { 122 | n, err := write(conn, err.Error()) 123 | if err != nil { 124 | printLog("Write Error (written %d bytes): %s (Server)\n", err) 125 | } 126 | printLog("Sent response (written %d bytes): %s (Server)\n", n, err) 127 | continue 128 | } 129 | f64Resp := cbrt(i32Req) 130 | respMsg := fmt.Sprintf("The cube root of %d is %f.", i32Req, f64Resp) 131 | n, err := write(conn, respMsg) 132 | if err != nil { 133 | printLog("Write Error: %s (Server)\n", err) 134 | } 135 | printLog("Sent response (written %d bytes): %s (Server)\n", n, respMsg) 136 | } 137 | } 138 | 139 | func clientGo(id int) { 140 | defer wg.Done() 141 | conn, err := net.DialTimeout(SERVER_NETWORK, SERVER_ADDRESS, 2*time.Second) 142 | if err != nil { 143 | printLog("Dial Error: %s (Client[%d])\n", err, id) 144 | return 145 | } 146 | defer conn.Close() 147 | printLog("Connected to server. (remote address: %s, local address: %s) (Client[%d])\n", 148 | conn.RemoteAddr(), conn.LocalAddr(), id) 149 | time.Sleep(200 * time.Millisecond) 150 | requestNumber := 5 151 | conn.SetDeadline(time.Now().Add(5 * time.Millisecond)) 152 | for i := 0; i < requestNumber; i++ { 153 | i32Req := rand.Int31() 154 | n, err := write(conn, fmt.Sprintf("%d", i32Req)) 155 | if err != nil { 156 | printLog("Write Error: %s (Client[%d])\n", err, id) 157 | continue 158 | } 159 | printLog("Sent request (written %d bytes): %d (Client[%d])\n", n, i32Req, id) 160 | } 161 | for j := 0; j < requestNumber; j++ { 162 | strResp, err := read(conn) 163 | if err != nil { 164 | if err == io.EOF { 165 | printLog("The connection is closed by another side. (Client[%d])\n", id) 166 | } else { 167 | printLog("Read Error: %s (Client[%d])\n", err, id) 168 | } 169 | break 170 | } 171 | printLog("Received response: %s (Client[%d])\n", strResp, id) 172 | } 173 | } 174 | 175 | func main() { 176 | wg.Add(2) 177 | go serverGo() 178 | time.Sleep(500 * time.Millisecond) 179 | go clientGo(1) 180 | wg.Wait() 181 | } 182 | -------------------------------------------------------------------------------- /src/pkgtool/envir.go: -------------------------------------------------------------------------------- 1 | package pkgtool 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "runtime" 7 | "strings" 8 | ) 9 | 10 | var srcDirsCache []string 11 | 12 | func init() { 13 | srcDirsCache = GetSrcDirs(true) 14 | } 15 | 16 | func GetGoroot() string { 17 | return runtime.GOROOT() 18 | } 19 | 20 | func GetAllGopath() []string { 21 | gopath := os.Getenv("GOPATH") 22 | var sep string 23 | if runtime.GOOS == "windows" { 24 | sep = ";" 25 | } else { 26 | sep = ":" 27 | } 28 | gopaths := strings.Split(gopath, sep) 29 | result := make([]string, 0) 30 | for _, v := range gopaths { 31 | if strings.TrimSpace(v) != "" { 32 | result = append(result, v) 33 | } 34 | } 35 | return result 36 | } 37 | 38 | func GetSrcDirs(fresh bool) []string { 39 | if len(srcDirsCache) > 0 && !fresh { 40 | return srcDirsCache 41 | } 42 | gorootSrcDir := filepath.Join(GetGoroot(), "src", "pkg") 43 | gopaths := GetAllGopath() 44 | gopathSrcDirs := make([]string, len(gopaths)) 45 | for i, v := range gopaths { 46 | gopathSrcDirs[i] = filepath.Join(v, "src") 47 | } 48 | srcDirs := make([]string, 0) 49 | srcDirs = append(srcDirs, gorootSrcDir) 50 | srcDirs = append(srcDirs, gopathSrcDirs...) 51 | srcDirsCache = make([]string, len(srcDirs)) 52 | copy(srcDirsCache, srcDirs) 53 | return srcDirs 54 | } 55 | -------------------------------------------------------------------------------- /src/pkgtool/envir_test.go: -------------------------------------------------------------------------------- 1 | package pkgtool 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "runtime" 7 | "runtime/debug" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func init() { 13 | runtime.GOMAXPROCS(runtime.NumCPU()) 14 | } 15 | 16 | func TestGetGoroot(t *testing.T) { 17 | t.Parallel() 18 | defer func() { 19 | if err := recover(); err != nil { 20 | debug.PrintStack() 21 | t.Errorf("Fatal Error: %s\n", err) 22 | } 23 | }() 24 | expGoroot := runtime.GOROOT() 25 | actGoroot := GetGoroot() 26 | if actGoroot != expGoroot { 27 | t.Errorf("Error: The goroot should be '%s'' but '%s'.\n", expGoroot, actGoroot) 28 | } 29 | } 30 | 31 | func TestGetAllGopath(t *testing.T) { 32 | t.Parallel() 33 | defer func() { 34 | if err := recover(); err != nil { 35 | debug.PrintStack() 36 | t.Errorf("Fatal Error: %s\n", err) 37 | } 38 | }() 39 | expGopath := os.Getenv("GOPATH") 40 | actGopaths := GetAllGopath() 41 | var sep string 42 | if runtime.GOOS == "windows" { 43 | sep = ";" 44 | } else { 45 | sep = ":" 46 | } 47 | expGopathLen := len(expGopath) 48 | tempString := expGopath 49 | tempIndex := -1 50 | sepNumber := 0 51 | for { 52 | if tempString == "" { 53 | break 54 | } 55 | tempIndex = strings.Index(tempString, sep) 56 | if tempIndex < 0 { 57 | sepNumber++ 58 | break 59 | } else { 60 | sepNumber++ 61 | } 62 | if (tempIndex + 1) >= expGopathLen { 63 | break 64 | } 65 | tempString = tempString[tempIndex+1:] 66 | } 67 | actLen := len(actGopaths) 68 | if actLen != sepNumber { 69 | t.Errorf("Error: The length of gopaths should be %d but %d.\n", sepNumber, actLen) 70 | } 71 | } 72 | 73 | func TestGetSrcDirs(t *testing.T) { 74 | t.Parallel() 75 | defer func() { 76 | if err := recover(); err != nil { 77 | debug.PrintStack() 78 | t.Errorf("Fatal Error: %s\n", err) 79 | } 80 | }() 81 | actSrcDirs := GetSrcDirs(true) 82 | actSrcDirLen := len(actSrcDirs) 83 | expSrcDirLen := len(GetAllGopath()) + 1 84 | if actSrcDirLen != expSrcDirLen { 85 | t.Errorf("Error: The length of gopaths should be %d but %d.\n", expSrcDirLen, actSrcDirLen) 86 | } 87 | fileSep := string(filepath.Separator) 88 | for _, v := range actSrcDirs { 89 | if !strings.HasSuffix(v, (fileSep+"src")) && 90 | !(strings.HasPrefix(v, GetGoroot()) && 91 | strings.HasSuffix(v, fileSep+"src"+fileSep+"pkg")) { 92 | t.Errorf("Error: The src dir '%s' is incorrect.\n", v) 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/pkgtool/fpath.go: -------------------------------------------------------------------------------- 1 | package pkgtool 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strings" 7 | ) 8 | 9 | func getAbsPathOfPackage(importPath string) string { 10 | for _, srcDir := range GetSrcDirs(false) { 11 | absPath := filepath.Join(srcDir, filepath.FromSlash(importPath)) 12 | //fmt.Printf("IP: %s, IP_DIR: %s, AP: %s\n", importPath, filepath.FromSlash(importPath), absPath) 13 | _, err := os.Open(absPath) 14 | if err == nil { 15 | return absPath 16 | } 17 | } 18 | return "" 19 | } 20 | 21 | func getGoSourceFileAbsPaths(packageAbspath string, containsTestFile bool) ([]string, error) { 22 | f, err := os.Open(packageAbspath) 23 | if err != nil { 24 | return nil, err 25 | } 26 | subs, err := f.Readdir(-1) 27 | if err != nil { 28 | return nil, err 29 | } 30 | absPaths := make([]string, 0) 31 | for _, v := range subs { 32 | fi := v.(os.FileInfo) 33 | name := fi.Name() 34 | if !fi.IsDir() && strings.HasSuffix(name, ".go") { 35 | if strings.HasSuffix(name, "_test.go") && !containsTestFile { 36 | continue 37 | } 38 | absPath := filepath.Join(packageAbspath, name) 39 | absPaths = append(absPaths, absPath) 40 | } 41 | } 42 | return absPaths, nil 43 | } 44 | -------------------------------------------------------------------------------- /src/pkgtool/ipath.go: -------------------------------------------------------------------------------- 1 | package pkgtool 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "os" 7 | "sort" 8 | "strings" 9 | ) 10 | 11 | func getImportsFromGoSource(filePath string) ([]string, error) { 12 | f, err := os.Open(filePath) 13 | if err != nil { 14 | return nil, err 15 | } 16 | defer func() { 17 | if f != nil { 18 | f.Close() 19 | } 20 | }() 21 | r := bufio.NewReader(f) 22 | var isMultiImport bool 23 | importPaths := make([]string, 0) 24 | for { 25 | lineBytes, _, err := r.ReadLine() 26 | if err != nil { 27 | if err == io.EOF { 28 | break 29 | } 30 | return nil, err 31 | } 32 | line := strings.TrimSpace(string(lineBytes)) 33 | // Ignore when build ignore 34 | if line == "// +build ignore" { 35 | return importPaths, nil 36 | } 37 | // Ignore command source 38 | // if strings.HasPrefix(line, "package") && strings.Split(line, " ")[1] == "main" { 39 | // return importPaths, nil 40 | // } 41 | if strings.HasPrefix(line, "import") { 42 | if strings.Contains(line, "(") { 43 | isMultiImport = true 44 | } else { 45 | importPath := line[strings.Index(line, "\"")+1 : strings.LastIndex(line, "\"")] 46 | importPaths = appendIfAbsent(importPaths, importPath) 47 | break 48 | } 49 | } else { 50 | if isMultiImport { 51 | if strings.HasPrefix(line, ")") { 52 | break 53 | } else { 54 | // Ignore irregular import 55 | if !strings.HasPrefix(line, "\"") || !strings.HasSuffix(line, "\"") { 56 | continue 57 | } 58 | importPath := strings.Replace(line, "\"", "", 2) 59 | importPaths = appendIfAbsent(importPaths, importPath) 60 | } 61 | } 62 | } 63 | } 64 | sort.Strings(importPaths) 65 | return importPaths, nil 66 | } 67 | 68 | func getImportsFromPackage(importPath string, containsTestFile bool) ([]string, error) { 69 | importPaths := make([]string, 0) 70 | packageAbsPath := getAbsPathOfPackage(importPath) 71 | // return empty slice when the import path is invalid. 72 | if packageAbsPath == "" { 73 | return importPaths, nil 74 | } 75 | srcFileAbsPaths, err := getGoSourceFileAbsPaths(packageAbsPath, containsTestFile) 76 | if err != nil { 77 | return nil, err 78 | } 79 | for _, v := range srcFileAbsPaths { 80 | currentImportPaths, err := getImportsFromGoSource(v) 81 | if err != nil { 82 | return nil, err 83 | } 84 | importPaths = appendIfAbsent(importPaths, currentImportPaths...) 85 | } 86 | return importPaths, nil 87 | } 88 | -------------------------------------------------------------------------------- /src/pkgtool/pnode.go: -------------------------------------------------------------------------------- 1 | package pkgtool 2 | 3 | import ( 4 | "path/filepath" 5 | "strings" 6 | ) 7 | 8 | var pkgNodesCache map[string]*PkgNode 9 | 10 | func init() { 11 | pkgNodesCache = make(map[string]*PkgNode) 12 | } 13 | 14 | type PkgNode struct { 15 | srcDir string 16 | importPath string 17 | triggers []*PkgNode 18 | deps []*PkgNode 19 | growed bool 20 | } 21 | 22 | func (self *PkgNode) SrcDir() string { 23 | return self.srcDir 24 | } 25 | 26 | func (self *PkgNode) ImportPath() string { 27 | return self.importPath 28 | } 29 | 30 | func (self *PkgNode) AddTrigger(pn *PkgNode) { 31 | self.triggers = append(self.triggers, pn) 32 | } 33 | 34 | func (self *PkgNode) AddDep(pn *PkgNode) { 35 | self.deps = append(self.deps, pn) 36 | } 37 | 38 | func (self *PkgNode) Triggers() []*PkgNode { 39 | triggers := make([]*PkgNode, len(self.triggers)) 40 | copy(triggers, self.triggers) 41 | return triggers 42 | } 43 | 44 | func (self *PkgNode) Deps() []*PkgNode { 45 | deps := make([]*PkgNode, len(self.deps)) 46 | copy(deps, self.deps) 47 | return deps 48 | } 49 | 50 | func (self *PkgNode) IsLeaf() bool { 51 | if len(self.deps) == 0 { 52 | return true 53 | } else { 54 | return false 55 | } 56 | } 57 | 58 | func (self *PkgNode) Grow() error { 59 | if self.growed { 60 | return nil 61 | } else { 62 | self.growed = true 63 | } 64 | importPaths, err := getImportsFromPackage(self.importPath, false) 65 | if err != nil { 66 | return err 67 | } 68 | l := len(importPaths) 69 | if l == 0 { 70 | return nil 71 | } 72 | //fmt.Printf("PN: %v, IPs: %v\n", self, importPaths) 73 | subPns := make([]*PkgNode, l) 74 | for i, importPath := range importPaths { 75 | pn, ok := pkgNodesCache[importPath] 76 | if !ok { 77 | pn = NewPkgNode(importPath) 78 | pkgNodesCache[importPath] = pn 79 | } 80 | subPns[i] = pn 81 | } 82 | for _, subPn := range subPns { 83 | subPn.AddTrigger(self) 84 | self.AddDep(subPn) 85 | err = subPn.Grow() 86 | if err != nil { 87 | return err 88 | } 89 | subPn.growed = true 90 | } 91 | return nil 92 | } 93 | 94 | func NewPkgNode(importPath string) *PkgNode { 95 | packageAbsPath := getAbsPathOfPackage(importPath) 96 | var srcDir string 97 | importDir := filepath.FromSlash(importPath) 98 | if strings.HasSuffix(packageAbsPath, importDir) { 99 | srcDir = packageAbsPath[:strings.LastIndex(packageAbsPath, importDir)] 100 | } 101 | return &PkgNode{ 102 | srcDir: srcDir, 103 | importPath: importPath, 104 | triggers: make([]*PkgNode, 0), 105 | deps: make([]*PkgNode, 0)} 106 | } 107 | -------------------------------------------------------------------------------- /src/pkgtool/util.go: -------------------------------------------------------------------------------- 1 | package pkgtool 2 | 3 | func appendIfAbsent(s []string, t ...string) []string { 4 | for _, t1 := range t { 5 | var contains bool 6 | for _, s1 := range s { 7 | if s1 == t1 { 8 | contains = true 9 | break 10 | } 11 | } 12 | if !contains { 13 | s = append(s, t1) 14 | } 15 | } 16 | return s 17 | } 18 | -------------------------------------------------------------------------------- /src/pkgtool/util_test.go: -------------------------------------------------------------------------------- 1 | package pkgtool 2 | 3 | import ( 4 | "runtime/debug" 5 | "testing" 6 | ) 7 | 8 | func TestAppendIfAbsent(t *testing.T) { 9 | t.Parallel() 10 | defer func() { 11 | if err := recover(); err != nil { 12 | debug.PrintStack() 13 | t.Errorf("Fatal Error: %s\n", err) 14 | } 15 | }() 16 | s := make([]string, 0) 17 | e1 := "a" 18 | s = appendIfAbsent(s, e1) 19 | expLen := 1 20 | actLen := len(s) 21 | if actLen != expLen { 22 | t.Errorf("Error: The length of slice should be %d but %d.\n", expLen, actLen) 23 | } 24 | if !contains(s, e1) { 25 | t.Errorf("Error: The slice should contains '%s' but not.\n", e1) 26 | } 27 | s = appendIfAbsent(s, e1) 28 | expLen = 1 29 | actLen = len(s) 30 | if actLen != expLen { 31 | t.Errorf("Error: The length of slice should be %d but %d.\n", expLen, actLen) 32 | } 33 | e2 := "b" 34 | s = appendIfAbsent(s, e2) 35 | expLen = 2 36 | actLen = len(s) 37 | if actLen != expLen { 38 | t.Errorf("Error: The length of slice should be %d but %d.\n", expLen, actLen) 39 | } 40 | expS := []string{"a", "b"} 41 | if !sliceEquels(s, expS) { 42 | t.Errorf("Error: The slice should be %v but %v.\n", expS, s) 43 | } 44 | } 45 | 46 | func contains(s []string, e string) bool { 47 | for _, v := range s { 48 | if v == e { 49 | return true 50 | } 51 | } 52 | return false 53 | } 54 | 55 | func sliceEquels(s1 []string, s2 []string) bool { 56 | if len(s1) != len(s2) { 57 | return false 58 | } 59 | for i, v1 := range s1 { 60 | v2 := s2[i] 61 | if v1 != v2 { 62 | return false 63 | } 64 | } 65 | return true 66 | } 67 | -------------------------------------------------------------------------------- /src/sync1/datafile1/datafile1.go: -------------------------------------------------------------------------------- 1 | package datafile1 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "os" 7 | "sync" 8 | ) 9 | 10 | // 数据的类型 11 | type Data []byte 12 | 13 | // 数据文件的接口类型。 14 | type DataFile interface { 15 | // 读取一个数据块。 16 | Read() (rsn int64, d Data, err error) 17 | // 写入一个数据块。 18 | Write(d Data) (wsn int64, err error) 19 | // 获取最后读取的数据块的序列号。 20 | Rsn() int64 21 | // 获取最后写入的数据块的序列号。 22 | Wsn() int64 23 | // 获取数据块的长度 24 | DataLen() uint32 25 | } 26 | 27 | // 数据文件的实现类型。 28 | type myDataFile struct { 29 | f *os.File // 文件。 30 | fmutex sync.RWMutex // 被用于文件的读写锁。 31 | woffset int64 // 写操作需要用到的偏移量。 32 | roffset int64 // 读操作需要用到的偏移量。 33 | wmutex sync.Mutex // 写操作需要用到的互斥锁。 34 | rmutex sync.Mutex // 读操作需要用到的互斥锁。 35 | dataLen uint32 // 数据块长度。 36 | } 37 | 38 | func NewDataFile(path string, dataLen uint32) (DataFile, error) { 39 | f, err := os.Create(path) 40 | if err != nil { 41 | return nil, err 42 | } 43 | if dataLen == 0 { 44 | return nil, errors.New("Invalid data length!") 45 | } 46 | df := &myDataFile{f: f, dataLen: dataLen} 47 | return df, nil 48 | } 49 | 50 | func (df *myDataFile) Read() (rsn int64, d Data, err error) { 51 | // 读取并更新读偏移量 52 | var offset int64 53 | df.rmutex.Lock() 54 | offset = df.roffset 55 | df.roffset += int64(df.dataLen) 56 | df.rmutex.Unlock() 57 | 58 | //读取一个数据块 59 | rsn = offset / int64(df.dataLen) 60 | bytes := make([]byte, df.dataLen) 61 | for { 62 | df.fmutex.RLock() 63 | _, err = df.f.ReadAt(bytes, offset) 64 | if err != nil { 65 | if err == io.EOF { 66 | df.fmutex.RUnlock() 67 | continue 68 | } 69 | df.fmutex.RUnlock() 70 | return 71 | } 72 | d = bytes 73 | df.fmutex.RUnlock() 74 | return 75 | } 76 | } 77 | 78 | func (df *myDataFile) Write(d Data) (wsn int64, err error) { 79 | // 读取并更新写偏移量 80 | var offset int64 81 | df.wmutex.Lock() 82 | offset = df.woffset 83 | df.woffset += int64(df.dataLen) 84 | df.wmutex.Unlock() 85 | 86 | //写入一个数据块 87 | wsn = offset / int64(df.dataLen) 88 | var bytes []byte 89 | if len(d) > int(df.dataLen) { 90 | bytes = d[0:df.dataLen] 91 | } else { 92 | bytes = d 93 | } 94 | df.fmutex.Lock() 95 | defer df.fmutex.Unlock() 96 | _, err = df.f.Write(bytes) 97 | return 98 | } 99 | 100 | func (df *myDataFile) Rsn() int64 { 101 | df.rmutex.Lock() 102 | defer df.rmutex.Unlock() 103 | return df.roffset / int64(df.dataLen) 104 | } 105 | 106 | func (df *myDataFile) Wsn() int64 { 107 | df.wmutex.Lock() 108 | defer df.wmutex.Unlock() 109 | return df.woffset / int64(df.dataLen) 110 | } 111 | 112 | func (df *myDataFile) DataLen() uint32 { 113 | return df.dataLen 114 | } 115 | -------------------------------------------------------------------------------- /src/sync1/datafile2/datafile2.go: -------------------------------------------------------------------------------- 1 | package datafile2 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "os" 7 | "sync" 8 | ) 9 | 10 | // 数据的类型 11 | type Data []byte 12 | 13 | // 数据文件的接口类型。 14 | type DataFile interface { 15 | // 读取一个数据块。 16 | Read() (rsn int64, d Data, err error) 17 | // 写入一个数据块。 18 | Write(d Data) (wsn int64, err error) 19 | // 获取最后读取的数据块的序列号。 20 | Rsn() int64 21 | // 获取最后写入的数据块的序列号。 22 | Wsn() int64 23 | // 获取数据块的长度 24 | DataLen() uint32 25 | } 26 | 27 | // 数据文件的实现类型。 28 | type myDataFile struct { 29 | f *os.File // 文件。 30 | fmutex sync.RWMutex // 被用于文件的读写锁。 31 | rcond *sync.Cond //读操作需要用到的条件变量 32 | woffset int64 // 写操作需要用到的偏移量。 33 | roffset int64 // 读操作需要用到的偏移量。 34 | wmutex sync.Mutex // 写操作需要用到的互斥锁。 35 | rmutex sync.Mutex // 读操作需要用到的互斥锁。 36 | dataLen uint32 // 数据块长度。 37 | } 38 | 39 | func NewDataFile(path string, dataLen uint32) (DataFile, error) { 40 | f, err := os.Create(path) 41 | if err != nil { 42 | return nil, err 43 | } 44 | if dataLen == 0 { 45 | return nil, errors.New("Invalid data length!") 46 | } 47 | df := &myDataFile{f: f, dataLen: dataLen} 48 | df.rcond = sync.NewCond(df.fmutex.RLocker()) 49 | return df, nil 50 | } 51 | 52 | func (df *myDataFile) Read() (rsn int64, d Data, err error) { 53 | // 读取并更新读偏移量 54 | var offset int64 55 | df.rmutex.Lock() 56 | offset = df.roffset 57 | df.roffset += int64(df.dataLen) 58 | df.rmutex.Unlock() 59 | 60 | //读取一个数据块 61 | rsn = offset / int64(df.dataLen) 62 | bytes := make([]byte, df.dataLen) 63 | df.fmutex.RLock() 64 | defer df.fmutex.RUnlock() 65 | for { 66 | _, err = df.f.ReadAt(bytes, offset) 67 | if err != nil { 68 | if err == io.EOF { 69 | df.rcond.Wait() 70 | continue 71 | } 72 | return 73 | } 74 | d = bytes 75 | return 76 | } 77 | } 78 | 79 | func (df *myDataFile) Write(d Data) (wsn int64, err error) { 80 | // 读取并更新写偏移量 81 | var offset int64 82 | df.wmutex.Lock() 83 | offset = df.woffset 84 | df.woffset += int64(df.dataLen) 85 | df.wmutex.Unlock() 86 | 87 | //写入一个数据块 88 | wsn = offset / int64(df.dataLen) 89 | var bytes []byte 90 | if len(d) > int(df.dataLen) { 91 | bytes = d[0:df.dataLen] 92 | } else { 93 | bytes = d 94 | } 95 | df.fmutex.Lock() 96 | defer df.fmutex.Unlock() 97 | _, err = df.f.Write(bytes) 98 | df.rcond.Signal() 99 | return 100 | } 101 | 102 | func (df *myDataFile) Rsn() int64 { 103 | df.rmutex.Lock() 104 | defer df.rmutex.Unlock() 105 | return df.roffset / int64(df.dataLen) 106 | } 107 | 108 | func (df *myDataFile) Wsn() int64 { 109 | df.wmutex.Lock() 110 | defer df.wmutex.Unlock() 111 | return df.woffset / int64(df.dataLen) 112 | } 113 | 114 | func (df *myDataFile) DataLen() uint32 { 115 | return df.dataLen 116 | } 117 | -------------------------------------------------------------------------------- /src/sync1/datafile3/datafile3.go: -------------------------------------------------------------------------------- 1 | package datafile3 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "os" 7 | "sync" 8 | "sync/atomic" 9 | ) 10 | 11 | // 数据的类型 12 | type Data []byte 13 | 14 | // 数据文件的接口类型。 15 | type DataFile interface { 16 | // 读取一个数据块。 17 | Read() (rsn int64, d Data, err error) 18 | // 写入一个数据块。 19 | Write(d Data) (wsn int64, err error) 20 | // 获取最后读取的数据块的序列号。 21 | Rsn() int64 22 | // 获取最后写入的数据块的序列号。 23 | Wsn() int64 24 | // 获取数据块的长度 25 | DataLen() uint32 26 | } 27 | 28 | // 数据文件的实现类型。 29 | type myDataFile struct { 30 | f *os.File // 文件。 31 | fmutex sync.RWMutex // 被用于文件的读写锁。 32 | rcond *sync.Cond // 读操作需要用到的条件变量 33 | woffset int64 // 写操作需要用到的偏移量。 34 | roffset int64 // 读操作需要用到的偏移量。 35 | dataLen uint32 // 数据块长度。 36 | } 37 | 38 | func NewDataFile(path string, dataLen uint32) (DataFile, error) { 39 | f, err := os.Create(path) 40 | if err != nil { 41 | return nil, err 42 | } 43 | if dataLen == 0 { 44 | return nil, errors.New("Invalid data length!") 45 | } 46 | df := &myDataFile{f: f, dataLen: dataLen} 47 | df.rcond = sync.NewCond(df.fmutex.RLocker()) 48 | return df, nil 49 | } 50 | 51 | func (df *myDataFile) Read() (rsn int64, d Data, err error) { 52 | // 读取并更新读偏移量 53 | var offset int64 54 | for { 55 | offset = atomic.LoadInt64(&df.roffset) 56 | if atomic.CompareAndSwapInt64(&df.roffset, offset, (offset + int64(df.dataLen))) { 57 | break 58 | } 59 | } 60 | 61 | //读取一个数据块 62 | rsn = offset / int64(df.dataLen) 63 | bytes := make([]byte, df.dataLen) 64 | df.fmutex.RLock() 65 | defer df.fmutex.RUnlock() 66 | for { 67 | _, err = df.f.ReadAt(bytes, offset) 68 | if err != nil { 69 | if err == io.EOF { 70 | df.rcond.Wait() 71 | continue 72 | } 73 | return 74 | } 75 | d = bytes 76 | return 77 | } 78 | } 79 | 80 | func (df *myDataFile) Write(d Data) (wsn int64, err error) { 81 | // 读取并更新写偏移量 82 | var offset int64 83 | for { 84 | offset = atomic.LoadInt64(&df.woffset) 85 | if atomic.CompareAndSwapInt64(&df.woffset, offset, (offset + int64(df.dataLen))) { 86 | break 87 | } 88 | } 89 | 90 | //写入一个数据块 91 | wsn = offset / int64(df.dataLen) 92 | var bytes []byte 93 | if len(d) > int(df.dataLen) { 94 | bytes = d[0:df.dataLen] 95 | } else { 96 | bytes = d 97 | } 98 | df.fmutex.Lock() 99 | defer df.fmutex.Unlock() 100 | _, err = df.f.Write(bytes) 101 | df.rcond.Signal() 102 | return 103 | } 104 | 105 | func (df *myDataFile) Rsn() int64 { 106 | offset := atomic.LoadInt64(&df.roffset) 107 | return offset / int64(df.dataLen) 108 | } 109 | 110 | func (df *myDataFile) Wsn() int64 { 111 | offset := atomic.LoadInt64(&df.woffset) 112 | return offset / int64(df.dataLen) 113 | } 114 | 115 | func (df *myDataFile) DataLen() uint32 { 116 | return df.dataLen 117 | } 118 | -------------------------------------------------------------------------------- /src/sync1/pool/pool_demo.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "runtime/debug" 7 | "sync" 8 | "sync/atomic" 9 | ) 10 | 11 | func main() { 12 | // 禁用GC，并保证在main函数执行结束前恢复GC 13 | defer debug.SetGCPercent(debug.SetGCPercent(-1)) 14 | var count int32 15 | newFunc := func() interface{} { 16 | return atomic.AddInt32(&count, 1) 17 | } 18 | pool := sync.Pool{New: newFunc} 19 | 20 | // New 字段值的作用 21 | v1 := pool.Get() 22 | fmt.Printf("v1: %v\n", v1) 23 | 24 | // 临时对象池的存取 25 | pool.Put(newFunc()) 26 | pool.Put(newFunc()) 27 | pool.Put(newFunc()) 28 | v2 := pool.Get() 29 | fmt.Printf("v2: %v\n", v2) 30 | 31 | // 垃圾回收对临时对象池的影响 32 | debug.SetGCPercent(100) 33 | runtime.GC() 34 | v3 := pool.Get() 35 | fmt.Printf("v3: %v\n", v3) 36 | pool.New = nil 37 | v4 := pool.Get() 38 | fmt.Printf("v4: %v\n", v4) 39 | } 40 | -------------------------------------------------------------------------------- /src/testing/bmt/bmt_test.go: -------------------------------------------------------------------------------- 1 | package bmt 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func Benchmark(b *testing.B) { 9 | customTimerTag := false 10 | if customTimerTag { 11 | b.StopTimer() 12 | } 13 | b.SetBytes(12345678) 14 | time.Sleep(time.Second) 15 | if customTimerTag { 16 | b.StartTimer() 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/testing/ct/ct_demo.go: -------------------------------------------------------------------------------- 1 | package ct 2 | 3 | func TypeCategoryOf(v interface{}) string { 4 | switch v.(type) { 5 | case bool: 6 | return "boolean" 7 | case int, uint, int8, uint8, int16, uint16, int32, uint32, int64, uint64: 8 | return "integer" 9 | case float32, float64: 10 | return "float" 11 | case complex64, complex128: 12 | return "complex" 13 | case string: 14 | a := "string" 15 | return a 16 | } 17 | return "unknown" 18 | } 19 | -------------------------------------------------------------------------------- /src/testing/et/et_test.go: -------------------------------------------------------------------------------- 1 | package et 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func ExampleHello() { 9 | for i := 0; i < 3; i++ { 10 | fmt.Println("Hello, Golang~") 11 | } 12 | 13 | // Output: Hello, Golang~ 14 | // Hello, Golang~ 15 | // Hello, Golang~ 16 | } 17 | 18 | func TestOne(t *testing.T) { 19 | t.Log("Hi~") 20 | } 21 | -------------------------------------------------------------------------------- /src/testing/rp/rp_test.go: -------------------------------------------------------------------------------- 1 | package rp 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func BenchmarkParallel(b *testing.B) { 9 | b.SetParallelism(3) 10 | fmt.Printf("\nN: %d\n", b.N) 11 | var index int 12 | b.RunParallel(func(pb *testing.PB) { 13 | i := index 14 | index++ 15 | var count int 16 | for pb.Next() { 17 | count++ 18 | } 19 | fmt.Printf("count[%d]: %d \n", i, count) 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /src/webcrawler/analyzer/analyzer.go: -------------------------------------------------------------------------------- 1 | package analyzer 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "logging" 7 | "net/url" 8 | base "webcrawler/base" 9 | mdw "webcrawler/middleware" 10 | ) 11 | 12 | // 日志记录器。 13 | var logger logging.Logger = base.NewLogger() 14 | 15 | // ID生成器。 16 | var analyzerIdGenerator mdw.IdGenerator = mdw.NewIdGenerator() 17 | 18 | // 生成并返回ID。 19 | func genAnalyzerId() uint32 { 20 | return analyzerIdGenerator.GetUint32() 21 | } 22 | 23 | // 分析器的接口类型。 24 | type Analyzer interface { 25 | Id() uint32 // 获得ID。 26 | Analyze( 27 | respParsers []ParseResponse, 28 | resp base.Response) ([]base.Data, []error) // 根据规则分析响应并返回请求和条目。 29 | } 30 | 31 | // 创建分析器。 32 | func NewAnalyzer() Analyzer { 33 | return &myAnalyzer{id: genAnalyzerId()} 34 | } 35 | 36 | // 分析器的实现类型。 37 | type myAnalyzer struct { 38 | id uint32 // ID。 39 | } 40 | 41 | func (analyzer *myAnalyzer) Id() uint32 { 42 | return analyzer.id 43 | } 44 | 45 | func (analyzer *myAnalyzer) Analyze( 46 | respParsers []ParseResponse, 47 | resp base.Response) (dataList []base.Data, errorList []error) { 48 | if respParsers == nil { 49 | err := errors.New("The response parser list is invalid!") 50 | return nil, []error{err} 51 | } 52 | httpResp := resp.HttpResp() 53 | if httpResp == nil { 54 | err := errors.New("The http response is invalid!") 55 | return nil, []error{err} 56 | } 57 | var reqUrl *url.URL = httpResp.Request.URL 58 | logger.Infof("Parse the response (reqUrl=%s)... \n", reqUrl) 59 | respDepth := resp.Depth() 60 | 61 | // 解析HTTP响应。 62 | dataList = make([]base.Data, 0) 63 | errorList = make([]error, 0) 64 | for i, respParser := range respParsers { 65 | if respParser == nil { 66 | err := errors.New(fmt.Sprintf("The document parser [%d] is invalid!", i)) 67 | errorList = append(errorList, err) 68 | continue 69 | } 70 | pDataList, pErrorList := respParser(httpResp, respDepth) 71 | if pDataList != nil { 72 | for _, pData := range pDataList { 73 | dataList = appendDataList(dataList, pData, respDepth) 74 | } 75 | } 76 | if pErrorList != nil { 77 | for _, pError := range pErrorList { 78 | errorList = appendErrorList(errorList, pError) 79 | } 80 | } 81 | } 82 | return dataList, errorList 83 | } 84 | 85 | // 添加请求值或条目值到列表。 86 | func appendDataList(dataList []base.Data, data base.Data, respDepth uint32) []base.Data { 87 | if data == nil { 88 | return dataList 89 | } 90 | req, ok := data.(*base.Request) 91 | if !ok { 92 | return append(dataList, data) 93 | } 94 | newDepth := respDepth + 1 95 | if req.Depth() != newDepth { 96 | req = base.NewRequest(req.HttpReq(), newDepth) 97 | } 98 | return append(dataList, req) 99 | } 100 | 101 | // 添加错误值到列表。 102 | func appendErrorList(errorList []error, err error) []error { 103 | if err == nil { 104 | return errorList 105 | } 106 | return append(errorList, err) 107 | } 108 | -------------------------------------------------------------------------------- /src/webcrawler/analyzer/parser.go: -------------------------------------------------------------------------------- 1 | package analyzer 2 | 3 | import ( 4 | "net/http" 5 | base "webcrawler/base" 6 | ) 7 | 8 | // 被用于解析HTTP响应的函数类型。 9 | type ParseResponse func(httpResp *http.Response, respDepth uint32) ([]base.Data, []error) 10 | -------------------------------------------------------------------------------- /src/webcrawler/analyzer/pool.go: -------------------------------------------------------------------------------- 1 | package analyzer 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | mdw "webcrawler/middleware" 8 | ) 9 | 10 | // 生成分析器的函数类型。 11 | type GenAnalyzer func() Analyzer 12 | 13 | // 分析器池的接口类型。 14 | type AnalyzerPool interface { 15 | Take() (Analyzer, error) // 从池中取出一个分析器。 16 | Return(analyzer Analyzer) error // 把一个分析器归还给池。 17 | Total() uint32 // 获得池的总容量。 18 | Used() uint32 // 获得正在被使用的分析器的数量。 19 | } 20 | 21 | func NewAnalyzerPool( 22 | total uint32, 23 | gen GenAnalyzer) (AnalyzerPool, error) { 24 | etype := reflect.TypeOf(gen()) 25 | genEntity := func() mdw.Entity { 26 | return gen() 27 | } 28 | pool, err := mdw.NewPool(total, etype, genEntity) 29 | if err != nil { 30 | return nil, err 31 | } 32 | dlpool := &myAnalyzerPool{pool: pool, etype: etype} 33 | return dlpool, nil 34 | } 35 | 36 | type myAnalyzerPool struct { 37 | pool mdw.Pool // 实体池。 38 | etype reflect.Type // 池内实体的类型。 39 | } 40 | 41 | func (spdpool *myAnalyzerPool) Take() (Analyzer, error) { 42 | entity, err := spdpool.pool.Take() 43 | if err != nil { 44 | return nil, err 45 | } 46 | analyzer, ok := entity.(Analyzer) 47 | if !ok { 48 | errMsg := fmt.Sprintf("The type of entity is NOT %s!\n", spdpool.etype) 49 | panic(errors.New(errMsg)) 50 | } 51 | return analyzer, nil 52 | } 53 | 54 | func (spdpool *myAnalyzerPool) Return(analyzer Analyzer) error { 55 | return spdpool.pool.Return(analyzer) 56 | } 57 | 58 | func (spdpool *myAnalyzerPool) Total() uint32 { 59 | return spdpool.pool.Total() 60 | } 61 | func (spdpool *myAnalyzerPool) Used() uint32 { 62 | return spdpool.pool.Used() 63 | } 64 | -------------------------------------------------------------------------------- /src/webcrawler/base/argument.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | // 参数容器的接口。 9 | type Args interface { 10 | // 自检参数的有效性，并在必要时返回可以说明问题的错误值。 11 | // 若结果值为nil，则说明未发现问题，否则就意味着自检未通过。 12 | Check() error 13 | // 获得参数容器的字符串表现形式。 14 | String() string 15 | } 16 | 17 | // 通道参数的容器的描述模板。 18 | var channelArgsTemplate string = "{ reqChanLen: %d, respChanLen: %d," + 19 | " itemChanLen: %d, errorChanLen: %d }" 20 | 21 | // 通道参数的容器。 22 | type ChannelArgs struct { 23 | reqChanLen uint // 请求通道的长度。 24 | respChanLen uint // 响应通道的长度。 25 | itemChanLen uint // 条目通道的长度。 26 | errorChanLen uint // 错误通道的长度。 27 | description string // 描述。 28 | } 29 | 30 | // 创建通道参数的容器。 31 | func NewChannelArgs( 32 | reqChanLen uint, 33 | respChanLen uint, 34 | itemChanLen uint, 35 | errorChanLen uint) ChannelArgs { 36 | return ChannelArgs{ 37 | reqChanLen: reqChanLen, 38 | respChanLen: respChanLen, 39 | itemChanLen: itemChanLen, 40 | errorChanLen: errorChanLen, 41 | } 42 | } 43 | 44 | func (args *ChannelArgs) Check() error { 45 | if args.reqChanLen == 0 { 46 | return errors.New("The request channel max length (capacity) can not be 0!\n") 47 | } 48 | if args.respChanLen == 0 { 49 | return errors.New("The response channel max length (capacity) can not be 0!\n") 50 | } 51 | if args.itemChanLen == 0 { 52 | return errors.New("The item channel max length (capacity) can not be 0!\n") 53 | } 54 | if args.errorChanLen == 0 { 55 | return errors.New("The error channel max length (capacity) can not be 0!\n") 56 | } 57 | return nil 58 | } 59 | 60 | func (args *ChannelArgs) String() string { 61 | if args.description == "" { 62 | args.description = 63 | fmt.Sprintf(channelArgsTemplate, 64 | args.reqChanLen, 65 | args.respChanLen, 66 | args.itemChanLen, 67 | args.errorChanLen) 68 | } 69 | return args.description 70 | } 71 | 72 | // 获得请求通道的长度。 73 | func (args *ChannelArgs) ReqChanLen() uint { 74 | return args.reqChanLen 75 | } 76 | 77 | // 获得响应通道的长度。 78 | func (args *ChannelArgs) RespChanLen() uint { 79 | return args.respChanLen 80 | } 81 | 82 | // 获得条目通道的长度。 83 | func (args *ChannelArgs) ItemChanLen() uint { 84 | return args.itemChanLen 85 | } 86 | 87 | // 获得错误通道的长度。 88 | func (args *ChannelArgs) ErrorChanLen() uint { 89 | return args.errorChanLen 90 | } 91 | 92 | // 池基本参数容器的描述模板。 93 | var poolBaseArgsTemplate string = "{ pageDownloaderPoolSize: %d," + 94 | " analyzerPoolSize: %d }" 95 | 96 | // 池基本参数的容器。 97 | type PoolBaseArgs struct { 98 | pageDownloaderPoolSize uint32 // 网页下载器池的尺寸。 99 | analyzerPoolSize uint32 // 分析器池的尺寸。 100 | description string // 描述。 101 | } 102 | 103 | // 创建池基本参数的容器。 104 | func NewPoolBaseArgs( 105 | pageDownloaderPoolSize uint32, 106 | analyzerPoolSize uint32) PoolBaseArgs { 107 | return PoolBaseArgs{ 108 | pageDownloaderPoolSize: pageDownloaderPoolSize, 109 | analyzerPoolSize: analyzerPoolSize, 110 | } 111 | } 112 | 113 | func (args *PoolBaseArgs) Check() error { 114 | if args.pageDownloaderPoolSize == 0 { 115 | return errors.New("The page downloader pool size can not be 0!\n") 116 | } 117 | if args.analyzerPoolSize == 0 { 118 | return errors.New("The analyzer pool size can not be 0!\n") 119 | } 120 | return nil 121 | } 122 | 123 | func (args *PoolBaseArgs) String() string { 124 | if args.description == "" { 125 | args.description = 126 | fmt.Sprintf(poolBaseArgsTemplate, 127 | args.pageDownloaderPoolSize, 128 | args.analyzerPoolSize) 129 | } 130 | return args.description 131 | } 132 | 133 | // 获得网页下载器池的尺寸。 134 | func (args *PoolBaseArgs) PageDownloaderPoolSize() uint32 { 135 | return args.pageDownloaderPoolSize 136 | } 137 | 138 | // 获得分析器池的尺寸。 139 | func (args *PoolBaseArgs) AnalyzerPoolSize() uint32 { 140 | return args.analyzerPoolSize 141 | } 142 | -------------------------------------------------------------------------------- /src/webcrawler/base/data.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | // 数据的接口。 8 | type Data interface { 9 | Valid() bool // 数据是否有效。 10 | } 11 | 12 | // 请求。 13 | type Request struct { 14 | httpReq *http.Request // HTTP请求的指针值。 15 | depth uint32 // 请求的深度。 16 | } 17 | 18 | // 创建新的请求。 19 | func NewRequest(httpReq *http.Request, depth uint32) *Request { 20 | return &Request{httpReq: httpReq, depth: depth} 21 | } 22 | 23 | // 获取HTTP请求。 24 | func (req *Request) HttpReq() *http.Request { 25 | return req.httpReq 26 | } 27 | 28 | // 获取深度值。 29 | func (req *Request) Depth() uint32 { 30 | return req.depth 31 | } 32 | 33 | // 数据是否有效。 34 | func (req *Request) Valid() bool { 35 | return req.httpReq != nil && req.httpReq.URL != nil 36 | } 37 | 38 | // 响应。 39 | type Response struct { 40 | httpResp *http.Response 41 | depth uint32 42 | } 43 | 44 | // 创建新的响应。 45 | func NewResponse(httpResp *http.Response, depth uint32) *Response { 46 | return &Response{httpResp: httpResp, depth: depth} 47 | } 48 | 49 | // 获取HTTP响应。 50 | func (resp *Response) HttpResp() *http.Response { 51 | return resp.httpResp 52 | } 53 | 54 | // 获取深度值。 55 | func (resp *Response) Depth() uint32 { 56 | return resp.depth 57 | } 58 | 59 | // 数据是否有效。 60 | func (resp *Response) Valid() bool { 61 | return resp.httpResp != nil && resp.httpResp.Body != nil 62 | } 63 | 64 | // 条目。 65 | type Item map[string]interface{} 66 | 67 | // 数据是否有效。 68 | func (item Item) Valid() bool { 69 | return item != nil 70 | } 71 | -------------------------------------------------------------------------------- /src/webcrawler/base/error.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | ) 7 | 8 | // 错误类型。 9 | type ErrorType string 10 | 11 | // 错误类型常量。 12 | const ( 13 | DOWNLOADER_ERROR ErrorType = "Downloader Error" 14 | ANALYZER_ERROR ErrorType = "Analyzer Error" 15 | ITEM_PROCESSOR_ERROR ErrorType = "Item Processor Error" 16 | ) 17 | 18 | // 爬虫错误的接口。 19 | type CrawlerError interface { 20 | Type() ErrorType // 获得错误类型。 21 | Error() string // 获得错误提示信息。 22 | } 23 | 24 | // 爬虫错误的实现。 25 | type myCrawlerError struct { 26 | errType ErrorType // 错误类型。 27 | errMsg string // 错误提示信息。 28 | fullErrMsg string // 完整的错误提示信息。 29 | } 30 | 31 | // 创建一个新的爬虫错误。 32 | func NewCrawlerError(errType ErrorType, errMsg string) CrawlerError { 33 | return &myCrawlerError{errType: errType, errMsg: errMsg} 34 | } 35 | 36 | // 获得错误类型。 37 | func (ce *myCrawlerError) Type() ErrorType { 38 | return ce.errType 39 | } 40 | 41 | // 获得错误提示信息。 42 | func (ce *myCrawlerError) Error() string { 43 | if ce.fullErrMsg == "" { 44 | ce.genFullErrMsg() 45 | } 46 | return ce.fullErrMsg 47 | } 48 | 49 | // 生成错误提示信息，并给相应的字段赋值。 50 | func (ce *myCrawlerError) genFullErrMsg() { 51 | var buffer bytes.Buffer 52 | buffer.WriteString("Crawler Error: ") 53 | if ce.errType != "" { 54 | buffer.WriteString(string(ce.errType)) 55 | buffer.WriteString(": ") 56 | } 57 | buffer.WriteString(ce.errMsg) 58 | ce.fullErrMsg = fmt.Sprintf("%s\n", buffer.String()) 59 | return 60 | } 61 | -------------------------------------------------------------------------------- /src/webcrawler/base/logger.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import "logging" 4 | 5 | // 创建日志记录器。 6 | func NewLogger() logging.Logger { 7 | return logging.NewSimpleLogger() 8 | } 9 | -------------------------------------------------------------------------------- /src/webcrawler/demo/demo.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/PuerkitoBio/goquery" 7 | "io" 8 | "logging" 9 | "net/http" 10 | "net/url" 11 | "strings" 12 | "time" 13 | "webcrawler/analyzer" 14 | base "webcrawler/base" 15 | pipeline "webcrawler/itempipeline" 16 | sched "webcrawler/scheduler" 17 | "webcrawler/tool" 18 | ) 19 | 20 | // 日志记录器。 21 | var logger logging.Logger = logging.NewSimpleLogger() 22 | 23 | // 条目处理器。 24 | func processItem(item base.Item) (result base.Item, err error) { 25 | if item == nil { 26 | return nil, errors.New("Invalid item!") 27 | } 28 | // 生成结果 29 | result = make(map[string]interface{}) 30 | for k, v := range item { 31 | result[k] = v 32 | } 33 | if _, ok := result["number"]; !ok { 34 | result["number"] = len(result) 35 | } 36 | time.Sleep(10 * time.Millisecond) 37 | return result, nil 38 | } 39 | 40 | // 响应解析函数。只解析“A”标签。 41 | func parseForATag(httpResp *http.Response, respDepth uint32) ([]base.Data, []error) { 42 | // TODO 支持更多的HTTP响应状态 43 | if httpResp.StatusCode != 200 { 44 | err := errors.New( 45 | fmt.Sprintf("Unsupported status code %d. (httpResponse=%v)", httpResp)) 46 | return nil, []error{err} 47 | } 48 | var reqUrl *url.URL = httpResp.Request.URL 49 | var httpRespBody io.ReadCloser = httpResp.Body 50 | defer func() { 51 | if httpRespBody != nil { 52 | httpRespBody.Close() 53 | } 54 | }() 55 | dataList := make([]base.Data, 0) 56 | errs := make([]error, 0) 57 | // 开始解析 58 | doc, err := goquery.NewDocumentFromReader(httpRespBody) 59 | if err != nil { 60 | errs = append(errs, err) 61 | return dataList, errs 62 | } 63 | // 查找“A”标签并提取链接地址 64 | doc.Find("a").Each(func(index int, sel *goquery.Selection) { 65 | href, exists := sel.Attr("href") 66 | // 前期过滤 67 | if !exists || href == "" || href == "#" || href == "/" { 68 | return 69 | } 70 | href = strings.TrimSpace(href) 71 | lowerHref := strings.ToLower(href) 72 | // 暂不支持对Javascript代码的解析。 73 | if href != "" && !strings.HasPrefix(lowerHref, "javascript") { 74 | aUrl, err := url.Parse(href) 75 | if err != nil { 76 | errs = append(errs, err) 77 | return 78 | } 79 | if !aUrl.IsAbs() { 80 | aUrl = reqUrl.ResolveReference(aUrl) 81 | } 82 | httpReq, err := http.NewRequest("GET", aUrl.String(), nil) 83 | if err != nil { 84 | errs = append(errs, err) 85 | } else { 86 | req := base.NewRequest(httpReq, respDepth) 87 | dataList = append(dataList, req) 88 | } 89 | } 90 | text := strings.TrimSpace(sel.Text()) 91 | if text != "" { 92 | imap := make(map[string]interface{}) 93 | imap["parent_url"] = reqUrl 94 | imap["a.text"] = text 95 | imap["a.index"] = index 96 | item := base.Item(imap) 97 | dataList = append(dataList, &item) 98 | } 99 | }) 100 | return dataList, errs 101 | } 102 | 103 | // 获得响应解析函数的序列。 104 | func getResponseParsers() []analyzer.ParseResponse { 105 | parsers := []analyzer.ParseResponse{ 106 | parseForATag, 107 | } 108 | return parsers 109 | } 110 | 111 | // 获得条目处理器的序列。 112 | func getItemProcessors() []pipeline.ProcessItem { 113 | itemProcessors := []pipeline.ProcessItem{ 114 | processItem, 115 | } 116 | return itemProcessors 117 | } 118 | 119 | // 生成HTTP客户端。 120 | func genHttpClient() *http.Client { 121 | return &http.Client{} 122 | } 123 | 124 | func record(level byte, content string) { 125 | if content == "" { 126 | return 127 | } 128 | switch level { 129 | case 0: 130 | logger.Infoln(content) 131 | case 1: 132 | logger.Warnln(content) 133 | case 2: 134 | logger.Infoln(content) 135 | } 136 | } 137 | 138 | func main() { 139 | // 创建调度器 140 | scheduler := sched.NewScheduler() 141 | 142 | // 准备监控参数 143 | intervalNs := 10 * time.Millisecond 144 | maxIdleCount := uint(1000) 145 | // 开始监控 146 | checkCountChan := tool.Monitoring( 147 | scheduler, 148 | intervalNs, 149 | maxIdleCount, 150 | true, 151 | false, 152 | record) 153 | 154 | // 准备启动参数 155 | channelArgs := base.NewChannelArgs(10, 10, 10, 10) 156 | poolBaseArgs := base.NewPoolBaseArgs(3, 3) 157 | crawlDepth := uint32(1) 158 | httpClientGenerator := genHttpClient 159 | respParsers := getResponseParsers() 160 | itemProcessors := getItemProcessors() 161 | startUrl := "http://www.sogou.com" 162 | firstHttpReq, err := http.NewRequest("GET", startUrl, nil) 163 | if err != nil { 164 | logger.Errorln(err) 165 | return 166 | } 167 | // 开启调度器 168 | scheduler.Start( 169 | channelArgs, 170 | poolBaseArgs, 171 | crawlDepth, 172 | httpClientGenerator, 173 | respParsers, 174 | itemProcessors, 175 | firstHttpReq) 176 | 177 | // 等待监控结束 178 | <-checkCountChan 179 | } 180 | -------------------------------------------------------------------------------- /src/webcrawler/downloader/downloader.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "logging" 5 | "net/http" 6 | base "webcrawler/base" 7 | mdw "webcrawler/middleware" 8 | ) 9 | 10 | // 日志记录器。 11 | var logger logging.Logger = base.NewLogger() 12 | 13 | // ID生成器。 14 | var downloaderIdGenerator mdw.IdGenerator = mdw.NewIdGenerator() 15 | 16 | // 生成并返回ID。 17 | func genDownloaderId() uint32 { 18 | return downloaderIdGenerator.GetUint32() 19 | } 20 | 21 | // 网页下载器的接口类型。 22 | type PageDownloader interface { 23 | Id() uint32 // 获得ID。 24 | Download(req base.Request) (*base.Response, error) // 根据请求下载网页并返回响应。 25 | } 26 | 27 | // 创建网页下载器。 28 | func NewPageDownloader(client *http.Client) PageDownloader { 29 | id := genDownloaderId() 30 | if client == nil { 31 | client = &http.Client{} 32 | } 33 | return &myPageDownloader{ 34 | id: id, 35 | httpClient: *client, 36 | } 37 | } 38 | 39 | // 网页下载器的实现类型。 40 | type myPageDownloader struct { 41 | id uint32 // ID。 42 | httpClient http.Client // HTTP客户端。 43 | } 44 | 45 | func (dl *myPageDownloader) Id() uint32 { 46 | return dl.id 47 | } 48 | 49 | func (dl *myPageDownloader) Download(req base.Request) (*base.Response, error) { 50 | httpReq := req.HttpReq() 51 | logger.Infof("Do the request (url=%s)... \n", httpReq.URL) 52 | httpResp, err := dl.httpClient.Do(httpReq) 53 | if err != nil { 54 | return nil, err 55 | } 56 | return base.NewResponse(httpResp, req.Depth()), nil 57 | } 58 | -------------------------------------------------------------------------------- /src/webcrawler/downloader/pool.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | mdw "webcrawler/middleware" 8 | ) 9 | 10 | // 生成网页下载器的函数类型。 11 | type GenPageDownloader func() PageDownloader 12 | 13 | // 网页下载器池的接口类型。 14 | type PageDownloaderPool interface { 15 | Take() (PageDownloader, error) // 从池中取出一个网页下载器。 16 | Return(dl PageDownloader) error // 把一个网页下载器归还给池。 17 | Total() uint32 // 获得池的总容量。 18 | Used() uint32 // 获得正在被使用的网页下载器的数量。 19 | } 20 | 21 | // 创建网页下载器池。 22 | func NewPageDownloaderPool( 23 | total uint32, 24 | gen GenPageDownloader) (PageDownloaderPool, error) { 25 | etype := reflect.TypeOf(gen()) 26 | genEntity := func() mdw.Entity { 27 | return gen() 28 | } 29 | pool, err := mdw.NewPool(total, etype, genEntity) 30 | if err != nil { 31 | return nil, err 32 | } 33 | dlpool := &myDownloaderPool{pool: pool, etype: etype} 34 | return dlpool, nil 35 | } 36 | 37 | // 网页下载器池的实现类型。 38 | type myDownloaderPool struct { 39 | pool mdw.Pool // 实体池。 40 | etype reflect.Type // 池内实体的类型。 41 | } 42 | 43 | func (dlpool *myDownloaderPool) Take() (PageDownloader, error) { 44 | entity, err := dlpool.pool.Take() 45 | if err != nil { 46 | return nil, err 47 | } 48 | dl, ok := entity.(PageDownloader) 49 | if !ok { 50 | errMsg := fmt.Sprintf("The type of entity is NOT %s!\n", dlpool.etype) 51 | panic(errors.New(errMsg)) 52 | } 53 | return dl, nil 54 | } 55 | 56 | func (dlpool *myDownloaderPool) Return(dl PageDownloader) error { 57 | return dlpool.pool.Return(dl) 58 | } 59 | 60 | func (dlpool *myDownloaderPool) Total() uint32 { 61 | return dlpool.pool.Total() 62 | } 63 | func (dlpool *myDownloaderPool) Used() uint32 { 64 | return dlpool.pool.Used() 65 | } 66 | -------------------------------------------------------------------------------- /src/webcrawler/itempipeline/pipeline.go: -------------------------------------------------------------------------------- 1 | package itemproc 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync/atomic" 7 | base "webcrawler/base" 8 | ) 9 | 10 | // 条目处理管道的接口类型。 11 | type ItemPipeline interface { 12 | // 发送条目。 13 | Send(item base.Item) []error 14 | // FailFast方法会返回一个布尔值。该值表示当前的条目处理管道是否是快速失败的。 15 | // 这里的快速失败是指：只要对某个条目的处理流程在某一个步骤上出错， 16 | // 那么条目处理管道就会忽略掉后续的所有处理步骤并报告错误。 17 | FailFast() bool 18 | // 设置是否快速失败。 19 | SetFailFast(failFast bool) 20 | // 获得已发送、已接受和已处理的条目的计数值。 21 | // 更确切地说，作为结果值的切片总会有三个元素值。这三个值会分别代表前述的三个计数。 22 | Count() []uint64 23 | // 获取正在被处理的条目的数量。 24 | ProcessingNumber() uint64 25 | // 获取摘要信息。 26 | Summary() string 27 | } 28 | 29 | // 创建条目处理管道。 30 | func NewItemPipeline(itemProcessors []ProcessItem) ItemPipeline { 31 | if itemProcessors == nil { 32 | panic(errors.New(fmt.Sprintln("Invalid item processor list!"))) 33 | } 34 | innerItemProcessors := make([]ProcessItem, 0) 35 | for i, ip := range itemProcessors { 36 | if ip == nil { 37 | panic(errors.New(fmt.Sprintf("Invalid item processor[%d]!\n", i))) 38 | } 39 | innerItemProcessors = append(innerItemProcessors, ip) 40 | } 41 | return &myItemPipeline{itemProcessors: innerItemProcessors} 42 | } 43 | 44 | // 条目处理管道的实现类型。 45 | type myItemPipeline struct { 46 | itemProcessors []ProcessItem // 条目处理器的列表。 47 | failFast bool // 表示处理是否需要快速失败的标志位。 48 | sent uint64 // 已被发送的条目的数量。 49 | accepted uint64 // 已被接受的条目的数量。 50 | processed uint64 // 已被处理的条目的数量。 51 | processingNumber uint64 // 正在被处理的条目的数量。 52 | } 53 | 54 | func (ip *myItemPipeline) Send(item base.Item) []error { 55 | atomic.AddUint64(&ip.processingNumber, 1) 56 | defer atomic.AddUint64(&ip.processingNumber, ^uint64(0)) 57 | atomic.AddUint64(&ip.sent, 1) 58 | errs := make([]error, 0) 59 | if item == nil { 60 | errs = append(errs, errors.New("The item is invalid!")) 61 | return errs 62 | } 63 | atomic.AddUint64(&ip.accepted, 1) 64 | var currentItem base.Item = item 65 | for _, itemProcessor := range ip.itemProcessors { 66 | processedItem, err := itemProcessor(currentItem) 67 | if err != nil { 68 | errs = append(errs, err) 69 | if ip.failFast { 70 | break 71 | } 72 | } 73 | if processedItem != nil { 74 | currentItem = processedItem 75 | } 76 | } 77 | atomic.AddUint64(&ip.processed, 1) 78 | return errs 79 | } 80 | 81 | func (ip *myItemPipeline) FailFast() bool { 82 | return ip.failFast 83 | } 84 | 85 | func (ip *myItemPipeline) SetFailFast(failFast bool) { 86 | ip.failFast = failFast 87 | } 88 | 89 | func (ip *myItemPipeline) Count() []uint64 { 90 | counts := make([]uint64, 3) 91 | counts[0] = atomic.LoadUint64(&ip.sent) 92 | counts[1] = atomic.LoadUint64(&ip.accepted) 93 | counts[2] = atomic.LoadUint64(&ip.processed) 94 | return counts 95 | } 96 | 97 | func (ip *myItemPipeline) ProcessingNumber() uint64 { 98 | return atomic.LoadUint64(&ip.processingNumber) 99 | } 100 | 101 | var summaryTemplate = "failFast: %v, processorNumber: %d," + 102 | " sent: %d, accepted: %d, processed: %d, processingNumber: %d" 103 | 104 | func (ip *myItemPipeline) Summary() string { 105 | counts := ip.Count() 106 | summary := fmt.Sprintf(summaryTemplate, 107 | ip.failFast, len(ip.itemProcessors), 108 | counts[0], counts[1], counts[2], ip.ProcessingNumber()) 109 | return summary 110 | } 111 | -------------------------------------------------------------------------------- /src/webcrawler/itempipeline/processor.go: -------------------------------------------------------------------------------- 1 | package itemproc 2 | 3 | import ( 4 | base "webcrawler/base" 5 | ) 6 | 7 | // 被用来处理条目的函数类型。 8 | type ProcessItem func(item base.Item) (result base.Item, err error) 9 | -------------------------------------------------------------------------------- /src/webcrawler/middleware/chanman.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | base "webcrawler/base" 8 | ) 9 | 10 | // 被用来表示通道管理器的状态的类型。 11 | type ChannelManagerStatus uint8 12 | 13 | const ( 14 | CHANNEL_MANAGER_STATUS_UNINITIALIZED ChannelManagerStatus = 0 // 未初始化状态。 15 | CHANNEL_MANAGER_STATUS_INITIALIZED ChannelManagerStatus = 1 // 已初始化状态。 16 | CHANNEL_MANAGER_STATUS_CLOSED ChannelManagerStatus = 2 // 已关闭状态。 17 | ) 18 | 19 | // 表示状态代码与状态名称之间的映射关系的字典。 20 | var statusNameMap = map[ChannelManagerStatus]string{ 21 | CHANNEL_MANAGER_STATUS_UNINITIALIZED: "uninitialized", 22 | CHANNEL_MANAGER_STATUS_INITIALIZED: "initialized", 23 | CHANNEL_MANAGER_STATUS_CLOSED: "closed", 24 | } 25 | 26 | // 通道管理器的接口类型。 27 | type ChannelManager interface { 28 | // 初始化通道管理器。 29 | // 参数channelArgs代表通道参数的容器。 30 | // 参数reset指明是否重新初始化通道管理器。 31 | Init(channelArgs base.ChannelArgs, reset bool) bool 32 | // 关闭通道管理器。 33 | Close() bool 34 | // 获取请求传输通道。 35 | ReqChan() (chan base.Request, error) 36 | // 获取响应传输通道。 37 | RespChan() (chan base.Response, error) 38 | // 获取条目传输通道。 39 | ItemChan() (chan base.Item, error) 40 | // 获取错误传输通道。 41 | ErrorChan() (chan error, error) 42 | // 获取通道管理器的状态。 43 | Status() ChannelManagerStatus 44 | // 获取摘要信息。 45 | Summary() string 46 | } 47 | 48 | // 创建通道管理器。 49 | func NewChannelManager(channelArgs base.ChannelArgs) ChannelManager { 50 | chanman := &myChannelManager{} 51 | chanman.Init(channelArgs, true) 52 | return chanman 53 | } 54 | 55 | // 通道管理器的实现类型。 56 | type myChannelManager struct { 57 | channelArgs base.ChannelArgs // 通道参数的容器。 58 | reqCh chan base.Request // 请求通道。 59 | respCh chan base.Response // 响应通道。 60 | itemCh chan base.Item // 条目通道。 61 | errorCh chan error // 错误通道。 62 | status ChannelManagerStatus // 通道管理器的状态。 63 | rwmutex sync.RWMutex // 读写锁。 64 | } 65 | 66 | func (chanman *myChannelManager) Init(channelArgs base.ChannelArgs, reset bool) bool { 67 | if err := channelArgs.Check(); err != nil { 68 | panic(err) 69 | } 70 | chanman.rwmutex.Lock() 71 | defer chanman.rwmutex.Unlock() 72 | if chanman.status == CHANNEL_MANAGER_STATUS_INITIALIZED && !reset { 73 | return false 74 | } 75 | chanman.channelArgs = channelArgs 76 | chanman.reqCh = make(chan base.Request, channelArgs.ReqChanLen()) 77 | chanman.respCh = make(chan base.Response, channelArgs.RespChanLen()) 78 | chanman.itemCh = make(chan base.Item, channelArgs.ItemChanLen()) 79 | chanman.errorCh = make(chan error, channelArgs.ErrorChanLen()) 80 | chanman.status = CHANNEL_MANAGER_STATUS_INITIALIZED 81 | return true 82 | } 83 | 84 | func (chanman *myChannelManager) Close() bool { 85 | chanman.rwmutex.Lock() 86 | defer chanman.rwmutex.Unlock() 87 | if chanman.status != CHANNEL_MANAGER_STATUS_INITIALIZED { 88 | return false 89 | } 90 | close(chanman.reqCh) 91 | close(chanman.respCh) 92 | close(chanman.itemCh) 93 | close(chanman.errorCh) 94 | chanman.status = CHANNEL_MANAGER_STATUS_CLOSED 95 | return true 96 | } 97 | 98 | func (chanman *myChannelManager) ReqChan() (chan base.Request, error) { 99 | chanman.rwmutex.RLock() 100 | defer chanman.rwmutex.RUnlock() 101 | if err := chanman.checkStatus(); err != nil { 102 | return nil, err 103 | } 104 | return chanman.reqCh, nil 105 | } 106 | 107 | func (chanman *myChannelManager) RespChan() (chan base.Response, error) { 108 | chanman.rwmutex.RLock() 109 | defer chanman.rwmutex.RUnlock() 110 | if err := chanman.checkStatus(); err != nil { 111 | return nil, err 112 | } 113 | return chanman.respCh, nil 114 | } 115 | 116 | func (chanman *myChannelManager) ItemChan() (chan base.Item, error) { 117 | chanman.rwmutex.RLock() 118 | defer chanman.rwmutex.RUnlock() 119 | if err := chanman.checkStatus(); err != nil { 120 | return nil, err 121 | } 122 | return chanman.itemCh, nil 123 | } 124 | 125 | func (chanman *myChannelManager) ErrorChan() (chan error, error) { 126 | chanman.rwmutex.RLock() 127 | defer chanman.rwmutex.RUnlock() 128 | if err := chanman.checkStatus(); err != nil { 129 | return nil, err 130 | } 131 | return chanman.errorCh, nil 132 | } 133 | 134 | // 检查状态。在获取通道的时候，通道管理器应处于已初始化状态。 135 | // 如果通道管理器未处于已初始化状态，那么本方法将会返回一个非nil的错误值。 136 | func (chanman *myChannelManager) checkStatus() error { 137 | if chanman.status == CHANNEL_MANAGER_STATUS_INITIALIZED { 138 | return nil 139 | } 140 | statusName, ok := statusNameMap[chanman.status] 141 | if !ok { 142 | statusName = fmt.Sprintf("%d", chanman.status) 143 | } 144 | errMsg := 145 | fmt.Sprintf("The undesirable status of channel manager: %s!\n", 146 | statusName) 147 | return errors.New(errMsg) 148 | } 149 | 150 | func (chanman *myChannelManager) Status() ChannelManagerStatus { 151 | return chanman.status 152 | } 153 | 154 | var chanmanSummaryTemplate = "status: %s, " + 155 | "requestChannel: %d/%d, " + 156 | "responseChannel: %d/%d, " + 157 | "itemChannel: %d/%d, " + 158 | "errorChannel: %d/%d" 159 | 160 | func (chanman *myChannelManager) Summary() string { 161 | summary := fmt.Sprintf(chanmanSummaryTemplate, 162 | statusNameMap[chanman.status], 163 | len(chanman.reqCh), cap(chanman.reqCh), 164 | len(chanman.respCh), cap(chanman.respCh), 165 | len(chanman.itemCh), cap(chanman.itemCh), 166 | len(chanman.errorCh), cap(chanman.errorCh)) 167 | return summary 168 | } 169 | -------------------------------------------------------------------------------- /src/webcrawler/middleware/id.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "math" 5 | "sync" 6 | ) 7 | 8 | // ID生成器的接口类型。 9 | type IdGenerator interface { 10 | GetUint32() uint32 // 获得一个uint32类型的ID。 11 | } 12 | 13 | // 创建ID生成器。 14 | func NewIdGenerator() IdGenerator { 15 | return &cyclicIdGenerator{} 16 | } 17 | 18 | // ID生成器的实现类型。 19 | type cyclicIdGenerator struct { 20 | sn uint32 // 当前的ID。 21 | ended bool // 前一个ID是否已经为其类型所能表示的最大值。 22 | mutex sync.Mutex // 互斥锁。 23 | } 24 | 25 | func (gen *cyclicIdGenerator) GetUint32() uint32 { 26 | gen.mutex.Lock() 27 | defer gen.mutex.Unlock() 28 | if gen.ended { 29 | defer func() { gen.ended = false }() 30 | gen.sn = 0 31 | return gen.sn 32 | } 33 | id := gen.sn 34 | if id < math.MaxUint32 { 35 | gen.sn++ 36 | } else { 37 | gen.ended = true 38 | } 39 | return id 40 | } 41 | 42 | // ID生成器的接口类型2。 43 | type IdGenerator2 interface { 44 | GetUint64() uint64 // 获得一个uint64类型的ID。 45 | } 46 | 47 | // 创建ID生成器2。 48 | func NewIdGenerator2() IdGenerator2 { 49 | return &cyclicIdGenerator2{} 50 | } 51 | 52 | // ID生成器的实现类型2。 53 | type cyclicIdGenerator2 struct { 54 | base cyclicIdGenerator // 基本的ID生成器。 55 | cycleCount uint64 // 基于uint32类型的取值范围的周期计数。 56 | } 57 | 58 | func (gen *cyclicIdGenerator2) GetUint64() uint64 { 59 | var id64 uint64 60 | if gen.cycleCount%2 == 1 { 61 | id64 += math.MaxUint32 62 | } 63 | id32 := gen.base.GetUint32() 64 | if id32 == math.MaxUint32 { 65 | gen.cycleCount++ 66 | } 67 | id64 += uint64(id32) 68 | return id64 69 | } 70 | -------------------------------------------------------------------------------- /src/webcrawler/middleware/pool.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | "sync" 8 | ) 9 | 10 | // 实体的接口类型。 11 | type Entity interface { 12 | Id() uint32 // ID的获取方法。 13 | } 14 | 15 | // 实体池的接口类型。 16 | type Pool interface { 17 | Take() (Entity, error) // 取出实体 18 | Return(entity Entity) error // 归还实体。 19 | Total() uint32 // 实体池的容量。 20 | Used() uint32 // 实体池中已被使用的实体的数量。 21 | } 22 | 23 | // 创建实体池。 24 | func NewPool( 25 | total uint32, 26 | entityType reflect.Type, 27 | genEntity func() Entity) (Pool, error) { 28 | if total == 0 { 29 | errMsg := 30 | fmt.Sprintf("The pool can not be initialized! (total=%d)\n", total) 31 | return nil, errors.New(errMsg) 32 | } 33 | size := int(total) 34 | container := make(chan Entity, size) 35 | idContainer := make(map[uint32]bool) 36 | for i := 0; i < size; i++ { 37 | newEntity := genEntity() 38 | if entityType != reflect.TypeOf(newEntity) { 39 | errMsg := 40 | fmt.Sprintf("The type of result of function genEntity() is NOT %s!\n", entityType) 41 | return nil, errors.New(errMsg) 42 | } 43 | container <- newEntity 44 | idContainer[newEntity.Id()] = true 45 | } 46 | pool := &myPool{ 47 | total: total, 48 | etype: entityType, 49 | genEntity: genEntity, 50 | container: container, 51 | idContainer: idContainer, 52 | } 53 | return pool, nil 54 | } 55 | 56 | // 实体池的实现类型。 57 | type myPool struct { 58 | total uint32 // 池的总容量。 59 | etype reflect.Type // 池中实体的类型。 60 | genEntity func() Entity // 池中实体的生成函数。 61 | container chan Entity // 实体容器。 62 | idContainer map[uint32]bool // 实体ID的容器。 63 | mutex sync.Mutex // 针对实体ID容器操作的互斥锁。 64 | } 65 | 66 | func (pool *myPool) Take() (Entity, error) { 67 | entity, ok := <-pool.container 68 | if !ok { 69 | return nil, errors.New("The inner container is invalid!") 70 | } 71 | pool.mutex.Lock() 72 | defer pool.mutex.Unlock() 73 | pool.idContainer[entity.Id()] = false 74 | return entity, nil 75 | } 76 | 77 | func (pool *myPool) Return(entity Entity) error { 78 | if entity == nil { 79 | return errors.New("The returning entity is invalid!") 80 | } 81 | if pool.etype != reflect.TypeOf(entity) { 82 | errMsg := fmt.Sprintf("The type of returning entity is NOT %s!\n", pool.etype) 83 | return errors.New(errMsg) 84 | } 85 | entityId := entity.Id() 86 | casResult := pool.compareAndSetForIdContainer(entityId, false, true) 87 | if casResult == 1 { 88 | pool.container <- entity 89 | return nil 90 | } else if casResult == 0 { 91 | errMsg := fmt.Sprintf("The entity (id=%d) is already in the pool!\n", entityId) 92 | return errors.New(errMsg) 93 | } else { 94 | errMsg := fmt.Sprintf("The entity (id=%d) is illegal!\n", entityId) 95 | return errors.New(errMsg) 96 | } 97 | } 98 | 99 | // 比较并设置实体ID容器中与给定实体ID对应的键值对的元素值。 100 | // 结果值： 101 | // -1：表示键值对不存在。 102 | // 0：表示操作失败。 103 | // 1：表示操作成功。 104 | func (pool *myPool) compareAndSetForIdContainer( 105 | entityId uint32, oldValue bool, newValue bool) int8 { 106 | pool.mutex.Lock() 107 | defer pool.mutex.Unlock() 108 | v, ok := pool.idContainer[entityId] 109 | if !ok { 110 | return -1 111 | } 112 | if v != oldValue { 113 | return 0 114 | } 115 | pool.idContainer[entityId] = newValue 116 | return 1 117 | } 118 | 119 | func (pool *myPool) Total() uint32 { 120 | return pool.total 121 | } 122 | 123 | func (pool *myPool) Used() uint32 { 124 | return pool.total - uint32(len(pool.container)) 125 | } 126 | -------------------------------------------------------------------------------- /src/webcrawler/middleware/stopsign.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | // 停止信号的接口类型。 9 | type StopSign interface { 10 | // 置位停止信号。相当于发出停止信号。 11 | // 如果先前已发出过停止信号，那么该方法会返回false。 12 | Sign() bool 13 | // 判断停止信号是否已被发出。 14 | Signed() bool 15 | // 重置停止信号。相当于收回停止信号，并清除所有的停止信号处理记录。 16 | Reset() 17 | // 处理停止信号。 18 | // 参数code应该代表停止信号处理方的代号。该代号会出现在停止信号的处理记录中。 19 | Deal(code string) 20 | // 获取某一个停止信号处理方的处理计数。该处理计数会从相应的停止信号处理记录中获得。 21 | DealCount(code string) uint32 22 | // 获取停止信号被处理的总计数。 23 | DealTotal() uint32 24 | // 获取摘要信息。其中应该包含所有的停止信号处理记录。 25 | Summary() string 26 | } 27 | 28 | // 创建停止信号。 29 | func NewStopSign() StopSign { 30 | ss := &myStopSign{ 31 | dealCountMap: make(map[string]uint32), 32 | } 33 | return ss 34 | } 35 | 36 | // 停止信号的实现类型。 37 | type myStopSign struct { 38 | rwmutex sync.RWMutex // 读写锁。 39 | signed bool // 表示信号是否已发出的标志位。 40 | dealCountMap map[string]uint32 // 处理计数的字典。 41 | } 42 | 43 | func (ss *myStopSign) Sign() bool { 44 | ss.rwmutex.Lock() 45 | defer ss.rwmutex.Unlock() 46 | if ss.signed { 47 | return false 48 | } 49 | ss.signed = true 50 | return true 51 | } 52 | 53 | func (ss *myStopSign) Signed() bool { 54 | return ss.signed 55 | } 56 | 57 | func (ss *myStopSign) Reset() { 58 | ss.rwmutex.Lock() 59 | defer ss.rwmutex.Unlock() 60 | ss.signed = false 61 | ss.dealCountMap = make(map[string]uint32) 62 | } 63 | 64 | func (ss *myStopSign) Deal(code string) { 65 | ss.rwmutex.Lock() 66 | defer ss.rwmutex.Unlock() 67 | if !ss.signed { 68 | return 69 | } 70 | if _, ok := ss.dealCountMap[code]; !ok { 71 | ss.dealCountMap[code] = 1 72 | } else { 73 | ss.dealCountMap[code] += 1 74 | } 75 | } 76 | 77 | func (ss *myStopSign) DealCount(code string) uint32 { 78 | ss.rwmutex.RLock() 79 | defer ss.rwmutex.Unlock() 80 | return ss.dealCountMap[code] 81 | } 82 | 83 | func (ss *myStopSign) DealTotal() uint32 { 84 | ss.rwmutex.RLock() 85 | defer ss.rwmutex.Unlock() 86 | var total uint32 87 | for _, v := range ss.dealCountMap { 88 | total += v 89 | } 90 | return total 91 | } 92 | 93 | func (ss *myStopSign) Summary() string { 94 | if ss.signed { 95 | return fmt.Sprintf("signed: true, dealCount: %v", ss.dealCountMap) 96 | } else { 97 | return "signed: false" 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/webcrawler/scheduler/cache.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | base "webcrawler/base" 7 | ) 8 | 9 | // 状态字典。 10 | var statusMap = map[byte]string{ 11 | 0: "running", 12 | 1: "closed", 13 | } 14 | 15 | // 请求缓存的接口类型。 16 | type requestCache interface { 17 | // 将请求放入请求缓存。 18 | put(req *base.Request) bool 19 | // 从请求缓存获取最早被放入且仍在其中的请求。 20 | get() *base.Request 21 | // 获得请求缓存的容量。 22 | capacity() int 23 | // 获得请求缓存的实时长度，即：其中的请求的即时数量。 24 | length() int 25 | // 关闭请求缓存。 26 | close() 27 | // 获取请求缓存的摘要信息。 28 | summary() string 29 | } 30 | 31 | // 创建请求缓存。 32 | func newRequestCache() requestCache { 33 | rc := &reqCacheBySlice{ 34 | cache: make([]*base.Request, 0), 35 | } 36 | return rc 37 | } 38 | 39 | // 请求缓存的实现类型。 40 | type reqCacheBySlice struct { 41 | cache []*base.Request // 请求的存储介质。 42 | mutex sync.Mutex // 互斥锁。 43 | status byte // 缓存状态。0表示正在运行，1表示已关闭。 44 | } 45 | 46 | func (rcache *reqCacheBySlice) put(req *base.Request) bool { 47 | if req == nil { 48 | return false 49 | } 50 | if rcache.status == 1 { 51 | return false 52 | } 53 | rcache.mutex.Lock() 54 | defer rcache.mutex.Unlock() 55 | rcache.cache = append(rcache.cache, req) 56 | return true 57 | } 58 | 59 | func (rcache *reqCacheBySlice) get() *base.Request { 60 | if rcache.length() == 0 { 61 | return nil 62 | } 63 | if rcache.status == 1 { 64 | return nil 65 | } 66 | rcache.mutex.Lock() 67 | defer rcache.mutex.Unlock() 68 | req := rcache.cache[0] 69 | rcache.cache = rcache.cache[1:] 70 | return req 71 | } 72 | 73 | func (rcache *reqCacheBySlice) capacity() int { 74 | return cap(rcache.cache) 75 | } 76 | 77 | func (rcache *reqCacheBySlice) length() int { 78 | return len(rcache.cache) 79 | } 80 | 81 | func (rcache *reqCacheBySlice) close() { 82 | if rcache.status == 1 { 83 | return 84 | } 85 | rcache.status = 1 86 | } 87 | 88 | // 摘要信息模板。 89 | var summaryTemplate = "status: %s, " + "length: %d, " + "capacity: %d" 90 | 91 | func (rcache *reqCacheBySlice) summary() string { 92 | summary := fmt.Sprintf(summaryTemplate, 93 | statusMap[rcache.status], 94 | rcache.length(), 95 | rcache.capacity()) 96 | return summary 97 | } 98 | -------------------------------------------------------------------------------- /src/webcrawler/scheduler/helper.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "regexp" 7 | "strings" 8 | anlz "webcrawler/analyzer" 9 | base "webcrawler/base" 10 | dl "webcrawler/downloader" 11 | ipl "webcrawler/itempipeline" 12 | mdw "webcrawler/middleware" 13 | ) 14 | 15 | func generateChannelManager(channelArgs base.ChannelArgs) mdw.ChannelManager { 16 | return mdw.NewChannelManager(channelArgs) 17 | } 18 | 19 | func generatePageDownloaderPool( 20 | poolSize uint32, 21 | httpClientGenerator GenHttpClient) (dl.PageDownloaderPool, error) { 22 | dlPool, err := dl.NewPageDownloaderPool( 23 | poolSize, 24 | func() dl.PageDownloader { 25 | return dl.NewPageDownloader(httpClientGenerator()) 26 | }, 27 | ) 28 | if err != nil { 29 | return nil, err 30 | } 31 | return dlPool, nil 32 | } 33 | 34 | func generateAnalyzerPool(poolSize uint32) (anlz.AnalyzerPool, error) { 35 | analyzerPool, err := anlz.NewAnalyzerPool( 36 | poolSize, 37 | func() anlz.Analyzer { 38 | return anlz.NewAnalyzer() 39 | }, 40 | ) 41 | if err != nil { 42 | return nil, err 43 | } 44 | return analyzerPool, nil 45 | } 46 | 47 | func generateItemPipeline(itemProcessors []ipl.ProcessItem) ipl.ItemPipeline { 48 | return ipl.NewItemPipeline(itemProcessors) 49 | } 50 | 51 | // 生成组件实例代号。 52 | func generateCode(prefix string, id uint32) string { 53 | return fmt.Sprintf("%s-%d", prefix, id) 54 | } 55 | 56 | // 解析组件实例代号。 57 | func parseCode(code string) []string { 58 | result := make([]string, 2) 59 | var codePrefix string 60 | var id string 61 | index := strings.Index(code, "-") 62 | if index > 0 { 63 | codePrefix = code[:index] 64 | id = code[index+1:] 65 | } else { 66 | codePrefix = code 67 | } 68 | result[0] = codePrefix 69 | result[1] = id 70 | return result 71 | } 72 | 73 | var regexpForIp = regexp.MustCompile(`((?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d))`) 74 | 75 | var regexpForDomains = []*regexp.Regexp{ 76 | // *.xx or *.xxx.xx 77 | regexp.MustCompile(`\.(com|com\.\w{2})$`), 78 | regexp.MustCompile(`\.(gov|gov\.\w{2})$`), 79 | regexp.MustCompile(`\.(net|net\.\w{2})$`), 80 | regexp.MustCompile(`\.(org|org\.\w{2})$`), 81 | // *.xx 82 | regexp.MustCompile(`\.me$`), 83 | regexp.MustCompile(`\.biz$`), 84 | regexp.MustCompile(`\.info$`), 85 | regexp.MustCompile(`\.name$`), 86 | regexp.MustCompile(`\.mobi$`), 87 | regexp.MustCompile(`\.so$`), 88 | regexp.MustCompile(`\.asia$`), 89 | regexp.MustCompile(`\.tel$`), 90 | regexp.MustCompile(`\.tv$`), 91 | regexp.MustCompile(`\.cc$`), 92 | regexp.MustCompile(`\.co$`), 93 | regexp.MustCompile(`\.\w{2}$`), 94 | } 95 | 96 | func getPrimaryDomain(host string) (string, error) { 97 | host = strings.TrimSpace(host) 98 | if host == "" { 99 | return "", errors.New("The host is empty!") 100 | } 101 | if regexpForIp.MatchString(host) { 102 | return host, nil 103 | } 104 | var suffixIndex int 105 | for _, re := range regexpForDomains { 106 | pos := re.FindStringIndex(host) 107 | if pos != nil { 108 | suffixIndex = pos[0] 109 | break 110 | } 111 | } 112 | if suffixIndex > 0 { 113 | var pdIndex int 114 | firstPart := host[:suffixIndex] 115 | index := strings.LastIndex(firstPart, ".") 116 | if index < 0 { 117 | pdIndex = 0 118 | } else { 119 | pdIndex = index + 1 120 | } 121 | return host[pdIndex:], nil 122 | } else { 123 | return "", errors.New("Unrecognized host!") 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/webcrawler/scheduler/summary.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | base "webcrawler/base" 7 | ) 8 | 9 | // 调度器摘要信息的接口类型。 10 | type SchedSummary interface { 11 | String() string // 获得摘要信息的一般表示。 12 | Detail() string // 获取摘要信息的详细表示。 13 | Same(other SchedSummary) bool // 判断是否与另一份摘要信息相同。 14 | } 15 | 16 | // 创建调度器摘要信息。 17 | func NewSchedSummary(sched *myScheduler, prefix string) SchedSummary { 18 | if sched == nil { 19 | return nil 20 | } 21 | urlCount := len(sched.urlMap) 22 | var urlDetail string 23 | if urlCount > 0 { 24 | var buffer bytes.Buffer 25 | buffer.WriteByte('\n') 26 | for k, _ := range sched.urlMap { 27 | buffer.WriteString(prefix) 28 | buffer.WriteString(prefix) 29 | buffer.WriteString(k) 30 | buffer.WriteByte('\n') 31 | } 32 | urlDetail = buffer.String() 33 | } else { 34 | urlDetail = "\n" 35 | } 36 | return &mySchedSummary{ 37 | prefix: prefix, 38 | running: sched.running, 39 | channelArgs: sched.channelArgs, 40 | poolBaseArgs: sched.poolBaseArgs, 41 | crawlDepth: sched.crawlDepth, 42 | chanmanSummary: sched.chanman.Summary(), 43 | reqCacheSummary: sched.reqCache.summary(), 44 | dlPoolLen: sched.dlpool.Used(), 45 | dlPoolCap: sched.dlpool.Total(), 46 | analyzerPoolLen: sched.analyzerPool.Used(), 47 | analyzerPoolCap: sched.analyzerPool.Total(), 48 | itemPipelineSummary: sched.itemPipeline.Summary(), 49 | urlCount: urlCount, 50 | urlDetail: urlDetail, 51 | stopSignSummary: sched.stopSign.Summary(), 52 | } 53 | } 54 | 55 | // 调度器摘要信息的实现类型。 56 | type mySchedSummary struct { 57 | prefix string // 前缀。 58 | running uint32 // 运行标记。 59 | channelArgs base.ChannelArgs // 通道参数的容器。 60 | poolBaseArgs base.PoolBaseArgs // 池基本参数的容器。 61 | crawlDepth uint32 // 爬取的最大深度。 62 | chanmanSummary string // 通道管理器的摘要信息。 63 | reqCacheSummary string // 请求缓存的摘要信息。 64 | dlPoolLen uint32 // 网页下载器池的长度。 65 | dlPoolCap uint32 // 网页下载器池的容量。 66 | analyzerPoolLen uint32 // 分析器池的长度。 67 | analyzerPoolCap uint32 // 分析器池的容量。 68 | itemPipelineSummary string // 条目处理管道的摘要信息。 69 | urlCount int // 已请求的URL的计数。 70 | urlDetail string // 已请求的URL的详细信息。 71 | stopSignSummary string // 停止信号的摘要信息。 72 | } 73 | 74 | func (ss *mySchedSummary) String() string { 75 | return ss.getSummary(false) 76 | } 77 | 78 | func (ss *mySchedSummary) Detail() string { 79 | return ss.getSummary(true) 80 | } 81 | 82 | // 获取摘要信息。 83 | func (ss *mySchedSummary) getSummary(detail bool) string { 84 | prefix := ss.prefix 85 | template := prefix + "Running: %v \n" + 86 | prefix + "Channel args: %s \n" + 87 | prefix + "Pool base args: %s \n" + 88 | prefix + "Crawl depth: %d \n" + 89 | prefix + "Channels manager: %s \n" + 90 | prefix + "Request cache: %s\n" + 91 | prefix + "Downloader pool: %d/%d\n" + 92 | prefix + "Analyzer pool: %d/%d\n" + 93 | prefix + "Item pipeline: %s\n" + 94 | prefix + "Urls(%d): %s" + 95 | prefix + "Stop sign: %s\n" 96 | return fmt.Sprintf(template, 97 | func() bool { 98 | return ss.running == 1 99 | }(), 100 | ss.channelArgs.String(), 101 | ss.poolBaseArgs.String(), 102 | ss.crawlDepth, 103 | ss.chanmanSummary, 104 | ss.reqCacheSummary, 105 | ss.dlPoolLen, ss.dlPoolCap, 106 | ss.analyzerPoolLen, ss.analyzerPoolCap, 107 | ss.itemPipelineSummary, 108 | ss.urlCount, 109 | func() string { 110 | if detail { 111 | return ss.urlDetail 112 | } else { 113 | return "\n" 114 | } 115 | }(), 116 | ss.stopSignSummary) 117 | } 118 | 119 | func (ss *mySchedSummary) Same(other SchedSummary) bool { 120 | if other == nil { 121 | return false 122 | } 123 | otherSs, ok := interface{}(other).(*mySchedSummary) 124 | if !ok { 125 | return false 126 | } 127 | if ss.running != otherSs.running || 128 | ss.crawlDepth != otherSs.crawlDepth || 129 | ss.dlPoolLen != otherSs.dlPoolLen || 130 | ss.dlPoolCap != otherSs.dlPoolCap || 131 | ss.analyzerPoolLen != otherSs.analyzerPoolLen || 132 | ss.analyzerPoolCap != otherSs.analyzerPoolCap || 133 | ss.urlCount != otherSs.urlCount || 134 | ss.stopSignSummary != otherSs.stopSignSummary || 135 | ss.reqCacheSummary != otherSs.reqCacheSummary || 136 | ss.poolBaseArgs.String() != otherSs.poolBaseArgs.String() || 137 | ss.channelArgs.String() != otherSs.channelArgs.String() || 138 | ss.itemPipelineSummary != otherSs.itemPipelineSummary || 139 | ss.chanmanSummary != otherSs.chanmanSummary { 140 | return false 141 | } else { 142 | return true 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/webcrawler/tool/cookie/cookiejar.go: -------------------------------------------------------------------------------- 1 | package cookie 2 | 3 | import ( 4 | "code.google.com/p/go.net/publicsuffix" 5 | "net/http" 6 | "net/http/cookiejar" 7 | ) 8 | 9 | // 创建 http.CookieJar 类型的值。 10 | func NewCookiejar() http.CookieJar { 11 | options := &cookiejar.Options{PublicSuffixList: &myPublicSuffixList{}} 12 | cj, _ := cookiejar.New(options) 13 | return cj 14 | } 15 | 16 | // cookiejar.PublicSuffixList 接口的实现类型。 17 | type myPublicSuffixList struct{} 18 | 19 | func (psl *myPublicSuffixList) PublicSuffix(domain string) string { 20 | suffix, _ := publicsuffix.PublicSuffix(domain) 21 | return suffix 22 | } 23 | 24 | func (psl *myPublicSuffixList) String() string { 25 | return "Web crawler - public suffix list (rev 1.0) power by 'code.google.com/p/go.net/publicsuffix'" 26 | } 27 | -------------------------------------------------------------------------------- /src/webcrawler/tool/monitor.go: -------------------------------------------------------------------------------- 1 | package tool 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "runtime" 7 | "time" 8 | sched "webcrawler/scheduler" 9 | ) 10 | 11 | // 摘要信息的模板。 12 | var summaryForMonitoring = "Monitor - Collected information[%d]:\n" + 13 | " Goroutine number: %d\n" + 14 | " Scheduler:\n%s" + 15 | " Escaped time: %s\n" 16 | 17 | // 已达到最大空闲计数的消息模板。 18 | var msgReachMaxIdleCount = "The scheduler has been idle for a period of time" + 19 | " (about %s)." + 20 | " Now consider what stop it." 21 | 22 | // 停止调度器的消息模板。 23 | var msgStopScheduler = "Stop scheduler...%s." 24 | 25 | // 日志记录函数的类型。 26 | // 参数level代表日志级别。级别设定：0：普通；1：警告；2：错误。 27 | type Record func(level byte, content string) 28 | 29 | // 调度器监控函数。 30 | // 参数scheduler代表作为监控目标的调度器。 31 | // 参数intervalNs代表检查间隔时间，单位：纳秒。 32 | // 参数maxIdleCount代表最大空闲计数。 33 | // 参数autoStop被用来指示该方法是否在调度器空闲一段时间（即持续空闲时间，由intervalNs * maxIdleCount得出）之后自行停止调度器。 34 | // 参数detailSummary被用来表示是否需要详细的摘要信息。 35 | // 参数record代表日志记录函数。 36 | // 当监控结束之后，该方法会会向作为唯一返回值的通道发送一个代表了空闲状态检查次数的数值。 37 | func Monitoring( 38 | scheduler sched.Scheduler, 39 | intervalNs time.Duration, 40 | maxIdleCount uint, 41 | autoStop bool, 42 | detailSummary bool, 43 | record Record) <-chan uint64 { 44 | if scheduler == nil { // 调度器不能不可用！ 45 | panic(errors.New("The scheduler is invalid!")) 46 | } 47 | // 防止过小的参数值对爬取流程的影响 48 | if intervalNs < time.Millisecond { 49 | intervalNs = time.Millisecond 50 | } 51 | if maxIdleCount < 1000 { 52 | maxIdleCount = 1000 53 | } 54 | // 监控停止通知器 55 | stopNotifier := make(chan byte, 1) 56 | // 接收和报告错误 57 | reportError(scheduler, record, stopNotifier) 58 | // 记录摘要信息 59 | recordSummary(scheduler, detailSummary, record, stopNotifier) 60 | // 检查计数通道 61 | checkCountChan := make(chan uint64, 2) 62 | // 检查空闲状态 63 | checkStatus(scheduler, 64 | intervalNs, 65 | maxIdleCount, 66 | autoStop, 67 | checkCountChan, 68 | record, 69 | stopNotifier) 70 | return checkCountChan 71 | } 72 | 73 | // 检查状态，并在满足持续空闲时间的条件时采取必要措施。 74 | func checkStatus( 75 | scheduler sched.Scheduler, 76 | intervalNs time.Duration, 77 | maxIdleCount uint, 78 | autoStop bool, 79 | checkCountChan chan<- uint64, 80 | record Record, 81 | stopNotifier chan<- byte) { 82 | var checkCount uint64 83 | go func() { 84 | defer func() { 85 | stopNotifier <- 1 86 | stopNotifier <- 2 87 | checkCountChan <- checkCount 88 | }() 89 | // 等待调度器开启 90 | waitForSchedulerStart(scheduler) 91 | // 准备 92 | var idleCount uint 93 | var firstIdleTime time.Time 94 | for { 95 | // 检查调度器的空闲状态 96 | if scheduler.Idle() { 97 | idleCount++ 98 | if idleCount == 1 { 99 | firstIdleTime = time.Now() 100 | } 101 | if idleCount >= maxIdleCount { 102 | msg := 103 | fmt.Sprintf(msgReachMaxIdleCount, time.Since(firstIdleTime).String()) 104 | record(0, msg) 105 | // 再次检查调度器的空闲状态，确保它已经可以被停止 106 | if scheduler.Idle() { 107 | if autoStop { 108 | var result string 109 | if scheduler.Stop() { 110 | result = "success" 111 | } else { 112 | result = "failing" 113 | } 114 | msg = fmt.Sprintf(msgStopScheduler, result) 115 | record(0, msg) 116 | } 117 | break 118 | } else { 119 | if idleCount > 0 { 120 | idleCount = 0 121 | } 122 | } 123 | } 124 | } else { 125 | if idleCount > 0 { 126 | idleCount = 0 127 | } 128 | } 129 | checkCount++ 130 | time.Sleep(intervalNs) 131 | } 132 | }() 133 | } 134 | 135 | // 记录摘要信息。 136 | func recordSummary( 137 | scheduler sched.Scheduler, 138 | detailSummary bool, 139 | record Record, 140 | stopNotifier <-chan byte) { 141 | go func() { 142 | // 等待调度器开启 143 | waitForSchedulerStart(scheduler) 144 | // 准备 145 | var prevSchedSummary sched.SchedSummary 146 | var prevNumGoroutine int 147 | var recordCount uint64 = 1 148 | startTime := time.Now() 149 | for { 150 | // 查看监控停止通知器 151 | select { 152 | case <-stopNotifier: 153 | return 154 | default: 155 | } 156 | // 获取摘要信息的各组成部分 157 | currNumGoroutine := runtime.NumGoroutine() 158 | currSchedSummary := scheduler.Summary(" ") 159 | // 比对前后两份摘要信息的一致性。只有不一致时才会予以记录。 160 | if currNumGoroutine != prevNumGoroutine || 161 | !currSchedSummary.Same(prevSchedSummary) { 162 | schedSummaryStr := func() string { 163 | if detailSummary { 164 | return currSchedSummary.Detail() 165 | } else { 166 | return currSchedSummary.String() 167 | } 168 | }() 169 | // 记录摘要信息 170 | info := fmt.Sprintf(summaryForMonitoring, 171 | recordCount, 172 | currNumGoroutine, 173 | schedSummaryStr, 174 | time.Since(startTime).String(), 175 | ) 176 | record(0, info) 177 | prevNumGoroutine = currNumGoroutine 178 | prevSchedSummary = currSchedSummary 179 | recordCount++ 180 | } 181 | time.Sleep(time.Microsecond) 182 | } 183 | }() 184 | } 185 | 186 | // 接收和报告错误。 187 | func reportError( 188 | scheduler sched.Scheduler, 189 | record Record, 190 | stopNotifier <-chan byte) { 191 | go func() { 192 | // 等待调度器开启 193 | waitForSchedulerStart(scheduler) 194 | for { 195 | // 查看监控停止通知器 196 | select { 197 | case <-stopNotifier: 198 | return 199 | default: 200 | } 201 | errorChan := scheduler.ErrorChan() 202 | if errorChan == nil { 203 | return 204 | } 205 | err := <-errorChan 206 | if err != nil { 207 | errMsg := fmt.Sprintf("Error (received from error channel): %s", err) 208 | record(2, errMsg) 209 | } 210 | time.Sleep(time.Microsecond) 211 | } 212 | }() 213 | } 214 | 215 | // 等待调度器开启。 216 | func waitForSchedulerStart(scheduler sched.Scheduler) { 217 | for !scheduler.Running() { 218 | time.Sleep(time.Microsecond) 219 | } 220 | } 221 | --------------------------------------------------------------------------------