├── Chapter01 ├── addInt.go ├── addInt_test.go ├── docker │ ├── Dockerfile │ └── main.go ├── nil_test.go ├── variadic.go └── variadic_test.go ├── Chapter02 ├── correctSerialGoroutines.go ├── goroutineHalt.go ├── goroutineRecover.go ├── parallel.go ├── serial.go └── serialGoroutines.go ├── Chapter03 ├── buffchan.go ├── cashier.go ├── closed.go ├── elems.go ├── multiplexing.go ├── naiveMultiplexing.go ├── simchan.go ├── unichans.go ├── unichans2.go ├── wichan.go └── wochan.go ├── Chapter04 ├── helloServer.go ├── restClient.go └── restServer │ ├── books-handler │ ├── actions.go │ ├── common.go │ └── handler.go │ └── main.go ├── Chapter05 ├── goophr │ ├── concierge │ │ ├── Dockerfile │ │ ├── Makefile │ │ ├── api │ │ │ ├── feeder.go │ │ │ └── query.go │ │ ├── common │ │ │ ├── helpers.go │ │ │ ├── state.go │ │ │ └── test_helpers.go │ │ └── main.go │ ├── docker-compose.yaml │ └── librarian │ │ ├── Dockerfile │ │ ├── Makefile │ │ ├── api │ │ ├── index.go │ │ └── query.go │ │ ├── common │ │ ├── helpers.go │ │ ├── state.go │ │ └── test_helpers.go │ │ └── main.go └── openapi │ ├── books.yaml │ ├── concierge.yaml │ └── librarian.yaml ├── Chapter06 ├── goophr │ └── concierge │ │ ├── Dockerfile │ │ ├── api │ │ ├── feeder.go │ │ ├── feeder_test.go │ │ └── query.go │ │ ├── common │ │ ├── helpers.go │ │ ├── state.go │ │ └── test_helpers.go │ │ └── main.go └── openapi │ └── concierge.yaml ├── Chapter07 ├── feeder.go └── goophr │ └── librarian │ ├── Dockerfile │ ├── api │ ├── index.go │ └── query.go │ ├── common │ ├── helpers.go │ ├── state.go │ └── test_helpers.go │ └── main.go ├── Chapter08 ├── goophr │ ├── concierge │ │ ├── Dockerfile │ │ ├── api │ │ │ ├── feeder.go │ │ │ ├── feeder_test.go │ │ │ └── query.go │ │ ├── common │ │ │ └── helpers.go │ │ └── main.go │ ├── docker-compose.yaml │ ├── librarian │ │ ├── Dockerfile │ │ ├── api │ │ │ ├── index.go │ │ │ └── query.go │ │ ├── common │ │ │ └── helpers.go │ │ └── main.go │ └── simple-server │ │ ├── Dockerfile │ │ └── main.go └── secure │ ├── secure.go │ └── secure_test.go └── README.md /Chapter01/addInt.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func addInt(numbers ...int) int { 4 | sum := 0 5 | for _, num := range numbers { 6 | sum += num 7 | } 8 | return sum 9 | } 10 | -------------------------------------------------------------------------------- /Chapter01/addInt_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestAddInt(t *testing.T) { 8 | testCases := []struct { 9 | Name string 10 | Values []int 11 | Expected int 12 | }{ 13 | {"addInt() -> 0", []int{}, 0}, 14 | {"addInt([]int{10, 20, 100}) -> 130", []int{10, 20, 100}, 130}, 15 | } 16 | 17 | for _, tc := range testCases { 18 | t.Run(tc.Name, func(t *testing.T) { 19 | sum := addInt(tc.Values...) 20 | if sum != tc.Expected { 21 | t.Errorf("%d != %d", sum, tc.Expected) 22 | } else { 23 | t.Logf("%d == %d", sum, tc.Expected) 24 | } 25 | }) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /Chapter01/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.9.1 2 | # The base image we want to use to build our docker image from. 3 | # Since this image is specialized for golang it will have GOPATH = /go 4 | 5 | 6 | ADD . /go/src/hello 7 | # We copy files & folders from our system onto the docker image 8 | 9 | RUN go install hello 10 | # Next we can create an executable binary for our project with the command, `go install` 11 | 12 | ENV NAME Bob 13 | 14 | ENTRYPOINT /go/bin/hello 15 | # Command to execute when we start the container 16 | 17 | -------------------------------------------------------------------------------- /Chapter01/docker/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | fmt.Println(os.Getenv("NAME") + " is your uncle.") 10 | } 11 | -------------------------------------------------------------------------------- /Chapter01/nil_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "net/http/httptest" 8 | "testing" 9 | ) 10 | 11 | const passMark = "\u2713" 12 | const failMark = "\u2717" 13 | 14 | func assertResponseEqual(t *testing.T, expected string, actual string) { 15 | t.Helper() // comment this line to see tests fail due to `if expected != actual` 16 | if expected != actual { 17 | t.Errorf("%s != %s %s", expected, actual, failMark) 18 | } else { 19 | t.Logf("%s == %s %s", expected, actual, passMark) 20 | } 21 | } 22 | 23 | func TestServer(t *testing.T) { 24 | testServer := httptest.NewServer( 25 | http.HandlerFunc( 26 | func(w http.ResponseWriter, r *http.Request) { 27 | path := r.RequestURI 28 | if path == "/1" { 29 | w.Write([]byte("Got 1.")) 30 | } else { 31 | w.Write([]byte("Got None.")) 32 | } 33 | })) 34 | defer testServer.Close() 35 | 36 | for _, testCase := range []struct { 37 | Name string 38 | Path string 39 | Expected string 40 | }{ 41 | {"Request correct URL", "/1", "Got 1."}, 42 | {"Request incorrect URL", "/12345", "Got None."}, 43 | } { 44 | t.Run(testCase.Name, func(t *testing.T) { 45 | res, err := http.Get(testServer.URL + testCase.Path) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | 50 | actual, err := ioutil.ReadAll(res.Body) 51 | res.Body.Close() 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | assertResponseEqual(t, testCase.Expected, fmt.Sprintf("%s", actual)) 56 | }) 57 | } 58 | t.Run("Fail for no reason", func(t *testing.T) { 59 | assertResponseEqual(t, "+", "-") 60 | }) 61 | } 62 | -------------------------------------------------------------------------------- /Chapter01/variadic.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func simpleVariadicToSlice(numbers ...int) []int { 4 | return numbers 5 | } 6 | 7 | func mixedVariadicToSlice(name string, numbers ...int) (string, []int) { 8 | return name, numbers 9 | } 10 | 11 | // Does not work. 12 | // func badVariadic(name ...string, numbers ...int) {} 13 | -------------------------------------------------------------------------------- /Chapter01/variadic_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func TestSimpleVariadicToSlice(t *testing.T) { 6 | // Test for no arguments 7 | if val := simpleVariadicToSlice(); val != nil { 8 | t.Error("value should be nil", nil) 9 | } else { 10 | t.Log("simpleVariadicToSlice() -> nil") 11 | } 12 | 13 | // Test for random set of values 14 | vals := simpleVariadicToSlice(1, 2, 3) 15 | expected := []int{1, 2, 3} 16 | isErr := false 17 | for i := 0; i < 3; i++ { 18 | if vals[i] != expected[i] { 19 | isErr = true 20 | break 21 | } 22 | } 23 | if isErr { 24 | t.Error("value should be []int{1, 2, 3}", vals) 25 | } else { 26 | t.Log("simpleVariadicToSlice(1, 2, 3) -> []int{1, 2, 3}") 27 | } 28 | 29 | // Test for a slice 30 | vals = simpleVariadicToSlice(expected...) 31 | isErr = false 32 | for i := 0; i < 3; i++ { 33 | if vals[i] != expected[i] { 34 | isErr = true 35 | break 36 | } 37 | } 38 | if isErr { 39 | t.Error("value should be []int{1, 2, 3}", vals) 40 | } else { 41 | t.Log("simpleVariadicToSlice([]int{1, 2, 3}...) -> []int{1, 2, 3}") 42 | } 43 | } 44 | 45 | func TestMixedVariadicToSlice(t *testing.T) { 46 | // Test for simple argument & no variadic arguments 47 | name, numbers := mixedVariadicToSlice("Bob") 48 | if name == "Bob" && numbers == nil { 49 | t.Log("Received as expected: Bob, ") 50 | } else { 51 | t.Errorf("Received unexpected values: %s, %s", name, numbers) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Chapter02/correctSerialGoroutines.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | // Simple individual tasks 9 | func makeHotelReservation(wg *sync.WaitGroup) { 10 | fmt.Println("Done making hotel reservation.") 11 | wg.Done() 12 | } 13 | func bookFlightTickets(wg *sync.WaitGroup) { 14 | fmt.Println("Done booking flight tickets.") 15 | wg.Done() 16 | } 17 | func orderADress(wg *sync.WaitGroup) { 18 | fmt.Println("Done ordering a dress.") 19 | wg.Done() 20 | } 21 | func payCreditCardBills(wg *sync.WaitGroup) { 22 | fmt.Println("Done paying Credit Card bills.") 23 | wg.Done() 24 | } 25 | 26 | // Tasks that will be executed in parts 27 | 28 | // Writing Mail 29 | func writeAMail(wg *sync.WaitGroup) { 30 | fmt.Println("Wrote 1/3rd of the mail.") 31 | go continueWritingMail1(wg) 32 | } 33 | func continueWritingMail1(wg *sync.WaitGroup) { 34 | fmt.Println("Wrote 2/3rds of the mail.") 35 | go continueWritingMail2(wg) 36 | } 37 | func continueWritingMail2(wg *sync.WaitGroup) { 38 | fmt.Println("Done writing the mail.") 39 | wg.Done() 40 | } 41 | 42 | // Listening to Audio Book 43 | func listenToAudioBook(wg *sync.WaitGroup) { 44 | fmt.Println("Listened to 10 minutes of audio book.") 45 | go continueListeningToAudioBook(wg) 46 | } 47 | func continueListeningToAudioBook(wg *sync.WaitGroup) { 48 | fmt.Println("Done listening to audio book.") 49 | wg.Done() 50 | } 51 | 52 | // All the tasks we want to complete in the day. 53 | // Note that we do not include the sub tasks here. 54 | var listOfTasks = []func(*sync.WaitGroup){ 55 | makeHotelReservation, bookFlightTickets, orderADress, 56 | payCreditCardBills, writeAMail, listenToAudioBook, 57 | } 58 | 59 | func main() { 60 | var waitGroup sync.WaitGroup 61 | // Set number of effective goroutines we want to wait upon 62 | waitGroup.Add(len(listOfTasks)) 63 | 64 | for _, task := range listOfTasks { 65 | // Pass reference to WaitGroup instance 66 | // Each of the tasks should call on WaitGroup.Done() 67 | go task(&waitGroup) // Achieving maximum concurrency 68 | } 69 | 70 | // Wait until all goroutines have completed execution. 71 | waitGroup.Wait() 72 | } 73 | -------------------------------------------------------------------------------- /Chapter02/goroutineHalt.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | func simpleFunc(index int, wg *sync.WaitGroup) { 9 | // This line should fail with Divide By Zero when index = 10 10 | fmt.Println("Attempting x/(x-10) where x = ", index, " answer is : ", index/(index-10)) 11 | wg.Done() 12 | } 13 | 14 | func main() { 15 | var wg sync.WaitGroup 16 | wg.Add(40) 17 | for i := 0; i < 40; i += 1 { 18 | go func(j int) { 19 | simpleFunc(j, &wg) 20 | }(i) 21 | } 22 | 23 | wg.Wait() 24 | } 25 | -------------------------------------------------------------------------------- /Chapter02/goroutineRecover.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | func simpleFunc(index int, wg *sync.WaitGroup) { 9 | // functions with defer keyword are executed at the end of the function 10 | // regardless of whether the function was executed successfully or not. 11 | defer func() { 12 | if r := recover(); r != nil { 13 | fmt.Println("Recovered from", r) 14 | } 15 | }() 16 | 17 | // We have changed the order of when wg.Done is called because 18 | // we should call upon wg.Done even if the following line fails. 19 | // Whether a defer function exists or not is dependant on whether it is registered 20 | // before or after the failing line of code. 21 | defer wg.Done() 22 | // This line should fail with Divide By Zero when index = 10 23 | fmt.Println("Attempting x/(x-10) where x = ", index, " answer is : ", index/(index-10)) 24 | } 25 | 26 | func main() { 27 | var wg sync.WaitGroup 28 | wg.Add(40) 29 | for i := 0; i < 40; i += 1 { 30 | go func(j int) { 31 | simpleFunc(j, &wg) 32 | }(i) 33 | } 34 | 35 | wg.Wait() 36 | } 37 | -------------------------------------------------------------------------------- /Chapter02/parallel.go: -------------------------------------------------------------------------------- 1 | // parallel.go 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | func printTime(msg string) { 12 | fmt.Println(msg, time.Now().Format("15:04:05")) 13 | } 14 | 15 | // Task that will be done over time 16 | func writeMail1(wg *sync.WaitGroup) { 17 | printTime("Done writing mail #1.") 18 | wg.Done() 19 | } 20 | func writeMail2(wg *sync.WaitGroup) { 21 | printTime("Done writing mail #2.") 22 | wg.Done() 23 | } 24 | func writeMail3(wg *sync.WaitGroup) { 25 | printTime("Done writing mail #3.") 26 | wg.Done() 27 | } 28 | 29 | // Task done in parallel 30 | func listenForever() { 31 | for { 32 | printTime("Listening...") 33 | } 34 | } 35 | 36 | func main() { 37 | var waitGroup sync.WaitGroup 38 | waitGroup.Add(3) 39 | 40 | go listenForever() 41 | 42 | // Give some time for listenForever to start 43 | time.Sleep(time.Nanosecond * 10) 44 | 45 | // Let's start writing the mails 46 | go writeMail1(&waitGroup) 47 | go writeMail2(&waitGroup) 48 | go writeMail3(&waitGroup) 49 | 50 | waitGroup.Wait() 51 | } 52 | -------------------------------------------------------------------------------- /Chapter02/serial.go: -------------------------------------------------------------------------------- 1 | // serial.go 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | ) 8 | 9 | // Simple individual tasks 10 | 11 | func makeHotelReservation() { 12 | fmt.Println("Done making hotel reservation.") 13 | } 14 | func bookFlightTickets() { 15 | fmt.Println("Done booking flight tickets.") 16 | } 17 | func orderADress() { 18 | fmt.Println("Done ordering a dress.") 19 | } 20 | func payCreditCardBills() { 21 | fmt.Println("Done paying Credit Card bills.") 22 | } 23 | 24 | // Tasks that will be executed in parts 25 | 26 | // Writing Mail 27 | func writeAMail() { 28 | fmt.Println("Wrote 1/3rd of the mail.") 29 | continueWritingMail1() 30 | } 31 | func continueWritingMail1() { 32 | fmt.Println("Wrote 2/3rds of the mail.") 33 | continueWritingMail2() 34 | } 35 | func continueWritingMail2() { 36 | fmt.Println("Done writing the mail.") 37 | } 38 | 39 | // Listening to Audio Book 40 | func listenToAudioBook() { 41 | fmt.Println("Listened to 10 minutes of audio book.") 42 | continueListeningToAudioBook() 43 | } 44 | func continueListeningToAudioBook() { 45 | fmt.Println("Done listening to audio book.") 46 | } 47 | 48 | // All the tasks we want to complete in the day. 49 | // Note that we do not include the sub tasks here. 50 | var listOfTasks []func() = []func(){ 51 | makeHotelReservation, bookFlightTickets, orderADress, 52 | payCreditCardBills, writeAMail, listenToAudioBook, 53 | } 54 | 55 | func main() { 56 | for _, task := range listOfTasks { 57 | task() 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Chapter02/serialGoroutines.go: -------------------------------------------------------------------------------- 1 | // serialGoroutines.go 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | ) 8 | 9 | // Simple individual tasks 10 | 11 | func makeHotelReservation() { 12 | fmt.Println("Done making hotel reservation.") 13 | } 14 | func bookFlightTickets() { 15 | fmt.Println("Done booking flight tickets.") 16 | } 17 | func orderADress() { 18 | fmt.Println("Done ordering a dress.") 19 | } 20 | func payCreditCardBills() { 21 | fmt.Println("Done paying Credit Card bills.") 22 | } 23 | 24 | /******************************************************************** 25 | We start by making Writing Mail & Listening Audo Book concurrent. 26 | *********************************************************************/ 27 | 28 | // Tasks that will be executed in parts 29 | 30 | // Writing Mail 31 | func writeAMail() { 32 | fmt.Println("Wrote 1/3rd of the mail.") 33 | go continueWritingMail1() // Notice the addition of `go` keyword. 34 | } 35 | func continueWritingMail1() { 36 | fmt.Println("Wrote 2/3rds of the mail.") 37 | go continueWritingMail2() // Notice the addition of `go` keyword. 38 | } 39 | func continueWritingMail2() { 40 | fmt.Println("Done writing the mail.") 41 | } 42 | 43 | // Listening to Audio Book 44 | func listenToAudioBook() { 45 | fmt.Println("Listened to 10 minutes of audio book.") 46 | go continueListeningToAudioBook() // Notice the addition of `go` keyword. 47 | } 48 | func continueListeningToAudioBook() { 49 | fmt.Println("Done listening to audio book.") 50 | } 51 | 52 | // All the tasks we want to complete in the day. 53 | // Note that we do not include the sub tasks here. 54 | var listOfTasks = []func(){ 55 | makeHotelReservation, bookFlightTickets, orderADress, 56 | payCreditCardBills, writeAMail, listenToAudioBook, 57 | } 58 | 59 | func main() { 60 | for _, task := range listOfTasks { 61 | task() 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /Chapter03/buffchan.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | func recv(ch <-chan int, wg *sync.WaitGroup) { 9 | fmt.Println("Receiving", <-ch) 10 | wg.Done() 11 | } 12 | 13 | func send(ch chan<- int, wg *sync.WaitGroup) { 14 | fmt.Println("Sending...") 15 | ch <- 100 16 | fmt.Println("Sent") 17 | wg.Done() 18 | } 19 | 20 | func main() { 21 | var wg sync.WaitGroup 22 | wg.Add(2) 23 | 24 | // Using a buffered channel. 25 | ch := make(chan int, 10) 26 | go recv(ch, &wg) 27 | go send(ch, &wg) 28 | 29 | wg.Wait() 30 | } 31 | -------------------------------------------------------------------------------- /Chapter03/cashier.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | func main() { 9 | var wg sync.WaitGroup 10 | // ordersProcessed & cashier are declared in main function 11 | // so that cashier has access to shared state variable `ordersProcessed`. 12 | // If we were to declare the variable inside the `cashier` function, 13 | // then it's value would be set to zero with every function call. 14 | ordersProcessed := 0 15 | cashier := func(orderNum int) { 16 | if ordersProcessed < 10 { 17 | // Cashier is ready to serve! 18 | fmt.Println("Processing order", orderNum) 19 | ordersProcessed++ 20 | } else { 21 | // Cashier has reached the max capacity of processing orders. 22 | fmt.Println("I am tired! I want to take rest!", orderNum) 23 | } 24 | wg.Done() 25 | } 26 | 27 | for i := 0; i < 30; i++ { 28 | // Note that instead of wg.Add(60), we are instead adding 1 29 | // per each loop iteration. Both are valid ways to add to WaitGroup as long as we can ensure the right number of calls. 30 | wg.Add(1) 31 | go func(orderNum int) { 32 | // Making an order 33 | cashier(orderNum) 34 | }(i) 35 | 36 | } 37 | wg.Wait() 38 | } 39 | -------------------------------------------------------------------------------- /Chapter03/closed.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | type msg struct { 6 | ID int 7 | value string 8 | } 9 | 10 | func handleIntChan(intChan <-chan int, done chan<- int) { 11 | // Even though there are only 4 elements being sent via channel, we retrieve 6 values. 12 | for i := 0; i < 6; i++ { 13 | fmt.Println(<-intChan) 14 | } 15 | done <- 0 16 | } 17 | 18 | func handleMsgChan(msgChan <-chan msg, done chan<- int) { 19 | // We retrieve 6 values of element type struct `msg`. 20 | // Given that there are only 4 values in the buffered channel, 21 | // the rest should be zero value of struct `msg`. 22 | for i := 0; i < 6; i++ { 23 | fmt.Println(fmt.Sprintf("%#v", <-msgChan)) 24 | } 25 | done <- 0 26 | } 27 | 28 | func main() { 29 | intChan := make(chan int) 30 | done := make(chan int) 31 | 32 | go func() { 33 | intChan <- 9 34 | intChan <- 2 35 | intChan <- 3 36 | intChan <- 7 37 | close(intChan) 38 | }() 39 | go handleIntChan(intChan, done) 40 | 41 | msgChan := make(chan msg, 5) 42 | go func() { 43 | for i := 1; i < 5; i++ { 44 | msgChan <- msg{ 45 | ID: i, 46 | value: fmt.Sprintf("VALUE-%v", i), 47 | } 48 | } 49 | close(msgChan) 50 | }() 51 | go handleMsgChan(msgChan, done) 52 | 53 | // We wait on the two channel handler goroutines to complete. 54 | <-done 55 | <-done 56 | 57 | // Since intChan is closed, this will cause a panic to occur. 58 | intChan <- 100 59 | } 60 | -------------------------------------------------------------------------------- /Chapter03/elems.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | func main() { 6 | // Let's create three simple functions that take an int argument 7 | fcn1 := func(i int) { 8 | fmt.Println("fcn1", i) 9 | } 10 | fcn2 := func(i int) { 11 | fmt.Println("fcn2", i*2) 12 | } 13 | fcn3 := func(i int) { 14 | fmt.Println("fcn3", i*3) 15 | } 16 | 17 | ch := make(chan func(int)) // Channel that sends & receives functions that take an int argument 18 | done := make(chan bool) // A Channel whose element type is a boolean value. 19 | 20 | // Launch a goroutine to work with the channels ch & done. 21 | go func() { 22 | // We accept all incoming functions on Channel ch and call the functions with value 10. 23 | for fcn := range ch { 24 | fcn(10) 25 | } 26 | // Once the loop terminates, we print Exiting and send true to done Channel. 27 | fmt.Println("Exiting") 28 | done <- true 29 | }() 30 | 31 | // Sending functions to channel ch 32 | ch <- fcn1 33 | ch <- fcn2 34 | ch <- fcn3 35 | 36 | // Close the channel once we are done sending it data. 37 | close(ch) 38 | 39 | // Wait on the launched goroutine to end. 40 | <-done 41 | } 42 | -------------------------------------------------------------------------------- /Chapter03/multiplexing.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | func main() { 8 | ch1 := make(chan int) 9 | ch2 := make(chan string) 10 | ch3 := make(chan int, 3) 11 | done := make(chan bool) 12 | completed := make(chan bool) 13 | 14 | ch3 <- 1 15 | ch3 <- 2 16 | ch3 <- 3 17 | go func() { 18 | for { 19 | 20 | select { 21 | case <-ch1: 22 | fmt.Println("Received data from ch1") 23 | case val := <-ch2: 24 | fmt.Println(val) 25 | case c := <-ch3: 26 | fmt.Println(c) 27 | case <-done: 28 | fmt.Println("exiting...") 29 | completed <- true 30 | return 31 | } 32 | } 33 | }() 34 | 35 | ch1 <- 100 36 | ch2 <- "ch2 msg" 37 | // Uncomment us to avoid leaking the `select` goroutine! 38 | //close(done) 39 | //<-completed 40 | } 41 | -------------------------------------------------------------------------------- /Chapter03/naiveMultiplexing.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | func main() { 6 | channels := [5](chan int){ 7 | make(chan int), 8 | make(chan int), 9 | make(chan int), 10 | make(chan int), 11 | make(chan int), 12 | } 13 | 14 | go func() { 15 | // Starting to wait on channels 16 | for _, chX := range channels { 17 | fmt.Println("Receiving from", <- chX) 18 | } 19 | }() 20 | 21 | for i := 1; i < 6; i++ { 22 | fmt.Println("Sending on channel:", i) 23 | channels[i] <- 1 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Chapter03/simchan.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | // helloChan waits on a channel until it gets some data and then prints the value. 6 | func helloChan(ch <-chan string) { 7 | val := <-ch 8 | fmt.Println("Hello, ", val) 9 | } 10 | 11 | func main() { 12 | // Creating a channel 13 | ch := make(chan string) 14 | 15 | // A Goroutine that receives data from a channel 16 | go helloChan(ch) 17 | 18 | // Sending data to a channel. 19 | ch <- "Bob" 20 | } 21 | -------------------------------------------------------------------------------- /Chapter03/unichans.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | func recv(ch <-chan int, wg *sync.WaitGroup) { 9 | fmt.Println("Receiving", <-ch) 10 | wg.Done() 11 | } 12 | 13 | func send(ch chan<- int, wg *sync.WaitGroup) { 14 | fmt.Println("Sending...") 15 | ch <- 100 16 | fmt.Println("Sent") 17 | wg.Done() 18 | } 19 | 20 | func main() { 21 | var wg sync.WaitGroup 22 | wg.Add(2) 23 | 24 | ch := make(chan int) 25 | go recv(ch, &wg) 26 | go send(ch, &wg) 27 | 28 | wg.Wait() 29 | } 30 | -------------------------------------------------------------------------------- /Chapter03/unichans2.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | func recv(ch <-chan int, wg *sync.WaitGroup) { 9 | fmt.Println("Receiving", <-ch) 10 | fmt.Println("Trying to send") // signalling that we are going to send over channel. 11 | ch <- 13 // Sending over channel 12 | wg.Done() 13 | } 14 | 15 | func send(ch chan<- int, wg *sync.WaitGroup) { 16 | fmt.Println("Sending...") 17 | ch <- 100 18 | fmt.Println("Sent") 19 | wg.Done() 20 | } 21 | 22 | func main() { 23 | var wg sync.WaitGroup 24 | wg.Add(2) 25 | 26 | ch := make(chan int, 10) 27 | go recv(ch, &wg) 28 | go send(ch, &wg) 29 | 30 | wg.Wait() 31 | } 32 | -------------------------------------------------------------------------------- /Chapter03/wichan.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | func cashier(cashierID int, orderChannel <-chan int, wg *sync.WaitGroup) { 9 | // Process orders upto limit. 10 | for ordersProcessed := 0; ordersProcessed < 10; ordersProcessed++ { 11 | // Retrieve order from orderChannel 12 | orderNum := <-orderChannel 13 | 14 | // Cashier is ready to serve! 15 | fmt.Println("Cashier ", cashierID, "Processing order", orderNum, "Orders Processed", ordersProcessed) 16 | wg.Done() 17 | } 18 | } 19 | 20 | func main() { 21 | var wg sync.WaitGroup 22 | wg.Add(30) 23 | ordersChannel := make(chan int) 24 | 25 | for i := 0; i < 3; i++ { 26 | // Start the three cashiers 27 | func(i int) { 28 | go cashier(i, ordersChannel, &wg) 29 | }(i) 30 | } 31 | 32 | // Start adding orders to be processed. 33 | for i := 0; i < 30; i++ { 34 | ordersChannel <- i 35 | } 36 | wg.Wait() 37 | } 38 | -------------------------------------------------------------------------------- /Chapter03/wochan.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | func createCashier(cashierID int, wg *sync.WaitGroup) func(int) { 9 | ordersProcessed := 0 10 | return func(orderNum int) { 11 | if ordersProcessed < 10 { 12 | // Cashier is ready to serve! 13 | //fmt.Println("Cashier ", cashierID, "Processing order", orderNum, "Orders Processed", ordersProcessed) 14 | fmt.Println(cashierID, "->", ordersProcessed) 15 | ordersProcessed++ 16 | } else { 17 | // Cashier has reached the max capacity of processing orders. 18 | fmt.Println("Cashier ", cashierID, "I am tired! I want to take rest!", orderNum) 19 | } 20 | wg.Done() 21 | } 22 | } 23 | 24 | func main() { 25 | cashierIndex := 0 26 | var wg sync.WaitGroup 27 | 28 | // cashier{1,2,3} 29 | cashiers := []func(int){} 30 | for i := 1; i <= 3; i++ { 31 | cashiers = append(cashiers, createCashier(i, &wg)) 32 | } 33 | 34 | for i := 0; i < 30; i++ { 35 | wg.Add(1) 36 | 37 | cashierIndex = cashierIndex % 3 38 | 39 | func(cashier func(int), i int) { 40 | // Making an order 41 | go cashier(i) 42 | }(cashiers[cashierIndex], i) 43 | 44 | cashierIndex++ 45 | } 46 | wg.Wait() 47 | } 48 | -------------------------------------------------------------------------------- /Chapter04/helloServer.go: -------------------------------------------------------------------------------- 1 | // helloServer.go 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | "log" 8 | "net/http" 9 | ) 10 | 11 | func helloWorldHandler(w http.ResponseWriter, r *http.Request) { 12 | msg := fmt.Sprintf("Received request [%s] for path: [%s]", r.Method, r.URL.Path) 13 | log.Println(msg) 14 | 15 | response := fmt.Sprintf("Hello, World! at Path: %s", r.URL.Path) 16 | fmt.Fprintf(w, response) 17 | } 18 | 19 | func main() { 20 | http.HandleFunc("/", helloWorldHandler) // Catch all Path 21 | 22 | log.Println("Starting server at port :8080...") 23 | http.ListenAndServe(":8080", nil) 24 | } 25 | -------------------------------------------------------------------------------- /Chapter04/restClient.go: -------------------------------------------------------------------------------- 1 | // restClient.go 2 | 3 | package main 4 | 5 | import ( 6 | "bytes" 7 | "encoding/json" 8 | "fmt" 9 | "io/ioutil" 10 | "net/http" 11 | ) 12 | 13 | type bookResource struct { 14 | Id string `json:"id"` 15 | Title string `json:"title"` 16 | Link string `json:"link"` 17 | } 18 | 19 | func main() { 20 | // GET 21 | fmt.Println("Making GET call.") 22 | // It is possible that we might have error while making an HTTP request 23 | // due to too many redirects or HTTP protocol error. We should check for this eventuality. 24 | resp, err := http.Get("http://localhost:8080/api/books") 25 | if err != nil { 26 | fmt.Println("Error while making GET call.", err) 27 | return 28 | } 29 | 30 | fmt.Printf("%+v\n\n", resp) 31 | 32 | // The response body is a data stream from the server we got the response back from. 33 | // This data stream is not in a useable format yet. 34 | // We need to read it from the server and convert it into a byte stream. 35 | body, _ := ioutil.ReadAll(resp.Body) 36 | defer resp.Body.Close() 37 | 38 | var books []bookResource 39 | json.Unmarshal(body, &books) 40 | 41 | fmt.Println(books) 42 | fmt.Println("\n") 43 | 44 | // POST 45 | payload, _ := json.Marshal(bookResource{ 46 | Title: "New Book", 47 | Link: "http://new-book.com", 48 | }) 49 | 50 | fmt.Println("Making POST call.") 51 | resp, err = http.Post( 52 | "http://localhost:8080/api/books/", 53 | "application/json", 54 | bytes.NewBuffer(payload), 55 | ) 56 | if err != nil { 57 | fmt.Println(err) 58 | } 59 | 60 | fmt.Printf("%+v\n\n", resp) 61 | 62 | body, _ = ioutil.ReadAll(resp.Body) 63 | defer resp.Body.Close() 64 | 65 | var book bookResource 66 | json.Unmarshal(body, &book) 67 | 68 | fmt.Println(book) 69 | 70 | fmt.Println("\n") 71 | } 72 | -------------------------------------------------------------------------------- /Chapter04/restServer/books-handler/actions.go: -------------------------------------------------------------------------------- 1 | // restServer/books-handler/actions.go 2 | 3 | package booksHandler 4 | 5 | import ( 6 | "net/http" 7 | ) 8 | 9 | // actOn{GET, POST, DELETE, PUT} functions return Response based on specific Request type. 10 | 11 | func actOnGET(books map[string]bookResource, act Action) { 12 | // These initialized values cover the case: 13 | // Request asked for an id that doesn't exist. 14 | status := http.StatusNotFound 15 | bookResult := []bookResource{} 16 | 17 | if act.Id == "" { 18 | 19 | // Request asked for all books. 20 | status = http.StatusOK 21 | for _, book := range books { 22 | bookResult = append(bookResult, book) 23 | } 24 | } else if book, exists := books[act.Id]; exists { 25 | 26 | // Request asked for a specific book and the id exists. 27 | status = http.StatusOK 28 | bookResult = []bookResource{book} 29 | } 30 | 31 | act.RetChan <- response{ 32 | StatusCode: status, 33 | Books: bookResult, 34 | } 35 | } 36 | 37 | func actOnDELETE(books map[string]bookResource, act Action) { 38 | book, exists := books[act.Id] 39 | delete(books, act.Id) 40 | 41 | if !exists { 42 | book = bookResource{} 43 | } 44 | 45 | // Return the deleted book if it exists else return an empty book. 46 | act.RetChan <- response{ 47 | StatusCode: http.StatusOK, 48 | Books: []bookResource{book}, 49 | } 50 | } 51 | 52 | func actOnPUT(books map[string]bookResource, act Action) { 53 | // These initialized values cover the case: 54 | // Request asked for an id that doesn't exist. 55 | status := http.StatusNotFound 56 | bookResult := []bookResource{} 57 | 58 | // If the id exists, update its values with the values from the payload. 59 | if book, exists := books[act.Id]; exists { 60 | book.Link = act.Payload.Link 61 | book.Title = act.Payload.Title 62 | books[act.Id] = book 63 | 64 | status = http.StatusOK 65 | bookResult = []bookResource{books[act.Id]} 66 | } 67 | 68 | // Return status and updated resource. 69 | act.RetChan <- response{ 70 | StatusCode: status, 71 | Books: bookResult, 72 | } 73 | 74 | } 75 | 76 | func actOnPOST(books map[string]bookResource, act Action, newID string) { 77 | // Add the new book to `books`. 78 | books[newID] = bookResource{ 79 | Id: newID, 80 | Link: act.Payload.Link, 81 | Title: act.Payload.Title, 82 | } 83 | 84 | act.RetChan <- response{ 85 | StatusCode: http.StatusCreated, 86 | Books: []bookResource{books[newID]}, 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /Chapter04/restServer/books-handler/common.go: -------------------------------------------------------------------------------- 1 | // restServer/books-handler/common.go 2 | 3 | package booksHandler 4 | 5 | import ( 6 | "encoding/json" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | ) 11 | 12 | // bookResource is used to hold all data needed to represent a Book resource in the books map. 13 | type bookResource struct { 14 | Id string `json:"id"` 15 | Title string `json:"title"` 16 | Link string `json:"link"` 17 | } 18 | 19 | // requestPayload is used to parse request's Payload. We ignore Id field for simplicity. 20 | type requestPayload struct { 21 | Title string `json:"title"` 22 | Link string `json:"link"` 23 | } 24 | 25 | // response struct consists of all the information required to create the correct HTTP response. 26 | type response struct { 27 | StatusCode int 28 | Books []bookResource 29 | } 30 | 31 | // Action struct is used to send data to the goroutine managing the state (map) of books. 32 | // RetChan allows us to send data back to the Handler function so that we can complete the HTTP request. 33 | type Action struct { 34 | Id string 35 | Type string 36 | Payload requestPayload 37 | RetChan chan<- response 38 | } 39 | 40 | // GetBooks is used to get the initial state of books represented by a map. 41 | func GetBooks() map[string]bookResource { 42 | books := map[string]bookResource{} 43 | for i := 1; i < 6; i++ { 44 | id := fmt.Sprintf("%d", i) 45 | books[id] = bookResource{ 46 | Id: id, 47 | Title: fmt.Sprintf("Book-%s", id), 48 | Link: fmt.Sprintf("http://link-to-book%s.com", id), 49 | } 50 | } 51 | return books 52 | } 53 | 54 | // MakeHandler shows a common pattern used reduce duplicated code. 55 | func MakeHandler(fn func(http.ResponseWriter, *http.Request, string, string, chan<- Action), 56 | endpoint string, actionCh chan<- Action) http.HandlerFunc { 57 | 58 | return func(w http.ResponseWriter, r *http.Request) { 59 | path := r.URL.Path 60 | method := r.Method 61 | 62 | msg := fmt.Sprintf("Received request [%s] for path: [%s]", method, path) 63 | log.Println(msg) 64 | 65 | id := path[len(endpoint):] 66 | log.Println("ID is ", id) 67 | fn(w, r, id, method, actionCh) 68 | } 69 | } 70 | 71 | // writeResponse uses the pattern similar to MakeHandler. 72 | func writeResponse(w http.ResponseWriter, resp response) { 73 | var err error 74 | var serializedPayload []byte 75 | 76 | if len(resp.Books) == 1 { 77 | serializedPayload, err = json.Marshal(resp.Books[0]) 78 | } else { 79 | serializedPayload, err = json.Marshal(resp.Books) 80 | } 81 | 82 | if err != nil { 83 | writeError(w, http.StatusInternalServerError) 84 | fmt.Println("Error while serializing payload: ", err) 85 | } else { 86 | w.Header().Set("Content-Type", "application/json") 87 | w.WriteHeader(resp.StatusCode) 88 | w.Write(serializedPayload) 89 | } 90 | } 91 | 92 | // writeError allows us to return error message in JSON format. 93 | func writeError(w http.ResponseWriter, statusCode int) { 94 | jsonMsg := struct { 95 | Msg string `json:"msg"` 96 | Code int `json:"code"` 97 | }{ 98 | Code: statusCode, 99 | Msg: http.StatusText(statusCode), 100 | } 101 | 102 | if serializedPayload, err := json.Marshal(jsonMsg); err != nil { 103 | http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 104 | fmt.Println("Error while serializing payload: ", err) 105 | } else { 106 | w.Header().Set("Content-Type", "application/json") 107 | w.WriteHeader(statusCode) 108 | w.Write(serializedPayload) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /Chapter04/restServer/books-handler/handler.go: -------------------------------------------------------------------------------- 1 | // restServer/books-handler/handler.go 2 | 3 | package booksHandler 4 | 5 | import ( 6 | "encoding/json" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | ) 12 | 13 | // StartBooksManager starts a goroutine that changes the state of books (map). 14 | // Primary reason to use a goroutine instead of directly manipulating the books map is to ensure 15 | // that we do not have multiple requests changing books' state simultaneously. 16 | func StartBooksManager(books map[string]bookResource, actionCh <-chan Action) { 17 | newID := len(books) 18 | for { 19 | select { 20 | case act := <-actionCh: 21 | switch act.Type { 22 | case "GET": 23 | actOnGET(books, act) 24 | case "POST": 25 | newID++ 26 | newBookID := fmt.Sprintf("%d", newID) 27 | actOnPOST(books, act, newBookID) 28 | case "PUT": 29 | actOnPUT(books, act) 30 | case "DELETE": 31 | actOnDELETE(books, act) 32 | } 33 | } 34 | } 35 | } 36 | 37 | /* BookHandler is responsible for ensuring that we process only the valid HTTP Requests. 38 | 39 | * GET -> id: Any 40 | 41 | * POST -> id: No 42 | * -> payload: Required 43 | 44 | * PUT -> id: Any 45 | * -> payload: Required 46 | 47 | * DELETE -> id: Any 48 | */ 49 | func BookHandler(w http.ResponseWriter, r *http.Request, id string, method string, actionCh chan<- Action) { 50 | 51 | // Ensure that id is set only for valid requests 52 | isGet := method == "GET" 53 | idIsSetForPost := method == "POST" && id != "" 54 | isPutOrPost := method == "PUT" || method == "POST" 55 | idIsSetForDelPut := (method == "DELETE" || method == "PUT") && id != "" 56 | if !isGet && !(idIsSetForPost || idIsSetForDelPut || isPutOrPost) { 57 | writeError(w, http.StatusMethodNotAllowed) 58 | return 59 | } 60 | 61 | respCh := make(chan response) 62 | act := Action{ 63 | Id: id, 64 | Type: method, 65 | RetChan: respCh, 66 | } 67 | 68 | // PUT & POST require a properly formed JSON payload 69 | if isPutOrPost { 70 | var reqPayload requestPayload 71 | body, _ := ioutil.ReadAll(r.Body) 72 | defer r.Body.Close() 73 | 74 | if err := json.Unmarshal(body, &reqPayload); err != nil { 75 | writeError(w, http.StatusBadRequest) 76 | return 77 | } 78 | 79 | act.Payload = reqPayload 80 | } 81 | 82 | // We have all the data required to process the Request. 83 | // Time to update the state of books. 84 | actionCh <- act 85 | 86 | // Wait for respCh to return data after updating the state of books. 87 | // For all successful Actions, the HTTP status code will either be 200 or 201. 88 | // Any other status code means that there was an issue with the request. 89 | var resp response 90 | if resp = <-respCh; resp.StatusCode > http.StatusCreated { 91 | writeError(w, resp.StatusCode) 92 | return 93 | } 94 | 95 | // We should only log the delete resource and not send it back to user 96 | if method == "DELETE" { 97 | log.Println(fmt.Sprintf("Resource ID %s deleted: %+v", id, resp.Books)) 98 | resp = response{ 99 | StatusCode: http.StatusOK, 100 | Books: []bookResource{}, 101 | } 102 | } 103 | 104 | writeResponse(w, resp) 105 | } 106 | -------------------------------------------------------------------------------- /Chapter04/restServer/main.go: -------------------------------------------------------------------------------- 1 | // restServer/main.go 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | "log" 8 | "net/http" 9 | 10 | booksHandler "github.com/last-ent/distributed-go/chapter4/restServer/books-handler" 11 | ) 12 | 13 | func main() { 14 | // Get state (map) for books available on REST server. 15 | books := booksHandler.GetBooks() 16 | log.Println(fmt.Sprintf("%+v", books)) 17 | 18 | actionCh := make(chan booksHandler.Action) 19 | 20 | // Start goroutine responsible for handling interaction with the books map 21 | go booksHandler.StartBooksManager(books, actionCh) 22 | 23 | http.HandleFunc("/api/books/", booksHandler.MakeHandler(booksHandler.BookHandler, "/api/books/", actionCh)) 24 | 25 | log.Println("Starting server at port 8080...") 26 | http.ListenAndServe(":8080", nil) 27 | } 28 | -------------------------------------------------------------------------------- /Chapter05/goophr/concierge/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.10 2 | 3 | 4 | ADD . /go/src/github.com/last-ent/distributed-go/chapter5/goophr/concierge 5 | 6 | WORKDIR /go/src/github.com/last-ent/distributed-go/chapter5/goophr/concierge 7 | 8 | RUN go install github.com/last-ent/distributed-go/chapter5/goophr/concierge 9 | 10 | ENTRYPOINT /go/bin/concierge 11 | 12 | EXPOSE 9000 13 | -------------------------------------------------------------------------------- /Chapter05/goophr/concierge/Makefile: -------------------------------------------------------------------------------- 1 | 2 | IMAGE = goophr-concierge 3 | GOPKGS = $(shell go list ./...) 4 | GOOS = linux 5 | GOARCH = amd64 6 | CGO_ENABLED = 0 7 | BUILD_PATH = ./build 8 | BINARY_NAME = "exec-bin" 9 | SRC_ROOT = "." 10 | 11 | 12 | test: lint 13 | @echo "RUNNING TESTS\n" 14 | go get ./... 15 | go test --cover -v $(GOPKGS) 16 | 17 | lint: 18 | @echo "COMMENCING LINT CHECKS." 19 | go fmt ./... 20 | golint ./... 21 | go vet ./... 22 | 23 | run.docker: test build.linux 24 | docker build -t $(IMAGE) . 25 | docker run --rm -it --name=$(IMAGE) $(IMAGE) 26 | 27 | build.linux: 28 | @echo "BUILDING LINUX BINARY" 29 | GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO_ENABLED) \ 30 | go build -o $(BUILD_PATH)/lin-$(BINARY_NAME) $(SRC_ROOT) 31 | 32 | build.bin: 33 | @echo "BUILDING BINARY" 34 | go build -o $(BUILD_PATH)/$(BINARY_NAME) $(SRC_ROOT) 35 | 36 | run: build.bin 37 | ./$(BUILD_PATH)/$(BINARY_NAME) 38 | 39 | -------------------------------------------------------------------------------- /Chapter05/goophr/concierge/api/feeder.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | // Logic related to /feeder 4 | -------------------------------------------------------------------------------- /Chapter05/goophr/concierge/api/query.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | // Logic related to /query 4 | -------------------------------------------------------------------------------- /Chapter05/goophr/concierge/common/helpers.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // Generic logic used by concierge 4 | -------------------------------------------------------------------------------- /Chapter05/goophr/concierge/common/state.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // Logic required to manage state for concierge 4 | -------------------------------------------------------------------------------- /Chapter05/goophr/concierge/common/test_helpers.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "testing" 4 | 5 | const checkMark = "\u2713" 6 | const ballotX = "\u2717" 7 | const prefix = "\t\t - " 8 | 9 | // OnTestSuccess is for pass. 10 | func OnTestSuccess(t *testing.T, msg string) { 11 | t.Log(prefix+msg, checkMark) 12 | } 13 | 14 | //OnTestError is for fail. 15 | func OnTestError(t *testing.T, msg string) { 16 | t.Error(prefix+msg, ballotX) 17 | } 18 | 19 | //OnTestUnexpectedError is for unexpected fail. 20 | func OnTestUnexpectedError(t *testing.T, err error) { 21 | OnTestError(t, "Unexpected Error:\n"+err.Error()) 22 | } 23 | -------------------------------------------------------------------------------- /Chapter05/goophr/concierge/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | func main() { 6 | fmt.Println("Hello from Concierge!") 7 | } 8 | -------------------------------------------------------------------------------- /Chapter05/goophr/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | concierge: 5 | build: concierge/. 6 | ports: 7 | - "6060:9000" 8 | a_m_librarian: 9 | build: librarian/. 10 | ports: 11 | - "7070:9000" 12 | n_z_librarian: 13 | build: librarian/. 14 | ports: 15 | - "8080:9000" 16 | others_librarian: 17 | build: librarian/. 18 | ports: 19 | - "9090:9000" 20 | -------------------------------------------------------------------------------- /Chapter05/goophr/librarian/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.10 2 | 3 | 4 | ADD . /go/src/github.com/last-ent/distributed-go/chapter5/goophr/librarian 5 | 6 | RUN go install github.com/last-ent/distributed-go/chapter5/goophr/librarian 7 | 8 | ENTRYPOINT /go/bin/librarian 9 | 10 | EXPOSE 9000 11 | -------------------------------------------------------------------------------- /Chapter05/goophr/librarian/Makefile: -------------------------------------------------------------------------------- 1 | 2 | IMAGE = goophr-librarian 3 | GOPKGS = $(shell go list ./...) 4 | GOOS = linux 5 | GOARCH = amd64 6 | CGO_ENABLED = 0 7 | BUILD_PATH = ./build 8 | BINARY_NAME = "exec-bin" 9 | SRC_ROOT = "." 10 | 11 | 12 | test: lint 13 | @echo "RUNNING TESTS\n" 14 | go get ./... 15 | go test --cover -v $(GOPKGS) 16 | 17 | lint: 18 | @echo "COMMENCING LINT CHECKS." 19 | go fmt ./... 20 | golint ./... 21 | go vet ./... 22 | 23 | run.docker: test build.linux 24 | docker build -t $(IMAGE) . 25 | docker run --rm -it --name=$(IMAGE) $(IMAGE) 26 | 27 | build.linux: 28 | @echo "BUILDING LINUX BINARY" 29 | GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO_ENABLED) \ 30 | go build -o $(BUILD_PATH)/lin-$(BINARY_NAME) $(SRC_ROOT) 31 | 32 | build.bin: 33 | @echo "BUILDING BINARY" 34 | go build -o $(BUILD_PATH)/$(BINARY_NAME) $(SRC_ROOT) 35 | 36 | run: build.bin 37 | ./$(BUILD_PATH)/$(BINARY_NAME) 38 | 39 | -------------------------------------------------------------------------------- /Chapter05/goophr/librarian/api/index.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | // Logic related to /index 4 | -------------------------------------------------------------------------------- /Chapter05/goophr/librarian/api/query.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | // Logic related to /query 4 | -------------------------------------------------------------------------------- /Chapter05/goophr/librarian/common/helpers.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // Generic logic used by librarian 4 | -------------------------------------------------------------------------------- /Chapter05/goophr/librarian/common/state.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // Logic required to manage state for Librarian 4 | -------------------------------------------------------------------------------- /Chapter05/goophr/librarian/common/test_helpers.go: -------------------------------------------------------------------------------- 1 | 2 | package common 3 | 4 | import "testing" 5 | 6 | const checkMark = "\u2713" 7 | const ballotX = "\u2717" 8 | const prefix = "\t\t - " 9 | 10 | // OnTestSuccess is for pass. 11 | func OnTestSuccess(t *testing.T, msg string){ 12 | t.Log(prefix+msg, checkMark) 13 | } 14 | 15 | //OnTestError is for fail. 16 | func OnTestError(t *testing.T, msg string) { 17 | t.Error(prefix+msg, ballotX) 18 | } 19 | 20 | //OnTestUnexpectedError is for unexpected fail. 21 | func OnTestUnexpectedError(t *testing.T, err error) { 22 | OnTestError(t, "Unexpected Error:\n"+err.Error()) 23 | } 24 | 25 | -------------------------------------------------------------------------------- /Chapter05/goophr/librarian/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | func main() { 8 | fmt.Println("Hello from Librarian!") 9 | } 10 | -------------------------------------------------------------------------------- /Chapter05/openapi/books.yaml: -------------------------------------------------------------------------------- 1 | # openapi/books.yaml 2 | 3 | openapi: 3.0.0 4 | servers: 5 | - url: /api 6 | info: 7 | title: Books API 8 | version: '1.0' 9 | description: > 10 | API responsible for adding, reading and updating list of books. 11 | paths: 12 | /books: 13 | get: 14 | description: | 15 | Get list of all books 16 | responses: 17 | '200': 18 | description: | 19 | Request successfully returned list of all books 20 | content: 21 | application/json: 22 | schema: 23 | $ref: '#/components/schemas/response' 24 | /books/{id}: 25 | get: 26 | description: | 27 | Get a particular books with ID `id` 28 | responses: 29 | '200': 30 | description: | 31 | Request was successfully completed. 32 | content: 33 | application/json: 34 | schema: 35 | $ref: '#/components/schemas/document' 36 | parameters: 37 | - in: query 38 | name: id 39 | schema: 40 | type: integer 41 | description: Book ID of the book to get. 42 | post: 43 | description: | 44 | Get a particular books with ID `id` 45 | responses: 46 | '200': 47 | description: | 48 | Request was successfully completed. 49 | content: 50 | application/json: 51 | schema: 52 | $ref: '#/components/schemas/payload' 53 | requestBody: 54 | content: 55 | application/json: 56 | schema: 57 | $ref: '#/components/schemas/document' 58 | put: 59 | description: | 60 | Update the data of a Book with ID `id` with the payload sent in the request body. 61 | responses: 62 | '200': 63 | description: | 64 | Request was successfully completed. 65 | content: 66 | application/json: 67 | schema: 68 | $ref: '#/components/schemas/payload' 69 | requestBody: 70 | content: 71 | application/json: 72 | schema: 73 | $ref: '#/components/schemas/document' 74 | delete: 75 | description: | 76 | Get a particular books with ID `id` 77 | responses: 78 | '200': 79 | description: | 80 | Request was successfully completed. 81 | parameters: 82 | - in: query 83 | name: id 84 | schema: 85 | type: integer 86 | description: Book ID of the book to get. 87 | components: 88 | schemas: 89 | response: 90 | type: array 91 | items: 92 | $ref: '#/components/schemas/document' 93 | 94 | document: 95 | type: object 96 | required: 97 | - title 98 | - link 99 | properties: 100 | id: 101 | type: integer 102 | description: Book ID 103 | title: 104 | type: string 105 | description: Title of the book 106 | link: 107 | type: string 108 | description: Link to the book 109 | 110 | payload: 111 | type: object 112 | required: 113 | - title 114 | - link 115 | properties: 116 | title: 117 | type: string 118 | description: Title of the book 119 | link: 120 | type: string 121 | description: Link to the book 122 | -------------------------------------------------------------------------------- /Chapter05/openapi/concierge.yaml: -------------------------------------------------------------------------------- 1 | # openapi/concierge.yaml 2 | 3 | openapi: 3.0.0 4 | servers: 5 | - url: /api 6 | info: 7 | title: Goophr Concierge API 8 | version: '1.0' 9 | description: > 10 | API responsible for responding to user input and communicating with Goophr 11 | Librarian. 12 | paths: 13 | /feeder: 14 | post: 15 | description: | 16 | Register new document to be indexed. 17 | responses: 18 | '200': 19 | description: | 20 | Request was successfully completed. 21 | content: 22 | application/json: 23 | schema: 24 | $ref: '#/components/schemas/response' 25 | '400': 26 | description: > 27 | Request was not processed because payload was incomplete or 28 | incorrect. 29 | content: 30 | application/json: 31 | schema: 32 | $ref: '#/components/schemas/response' 33 | requestBody: 34 | content: 35 | application/json: 36 | schema: 37 | $ref: '#/components/schemas/document' 38 | required: true 39 | /query: 40 | post: 41 | description: | 42 | Search query 43 | responses: 44 | '200': 45 | description: | 46 | Response consists of links to document 47 | content: 48 | application/json: 49 | schema: 50 | type: array 51 | items: 52 | $ref: '#/components/schemas/document' 53 | requestBody: 54 | content: 55 | application/json: 56 | schema: 57 | type: array 58 | items: 59 | type: string 60 | required: true 61 | components: 62 | schemas: 63 | response: 64 | type: object 65 | properties: 66 | code: 67 | type: integer 68 | description: Status code to send in response 69 | msg: 70 | type: string 71 | description: Message to send in response 72 | document: 73 | type: object 74 | required: 75 | - title 76 | - link 77 | properties: 78 | title: 79 | type: string 80 | description: Title of the document 81 | link: 82 | type: string 83 | description: Link to the document 84 | -------------------------------------------------------------------------------- /Chapter05/openapi/librarian.yaml: -------------------------------------------------------------------------------- 1 | # openapi/librarian.yaml 2 | 3 | openapi: 3.0.0 4 | servers: 5 | - url: /api 6 | info: 7 | title: Goophr Librarian API 8 | version: '1.0' 9 | description: | 10 | API responsible for indexing & communicating with Goophr Concierge. 11 | paths: 12 | /index: 13 | post: 14 | description: | 15 | Add terms to index. 16 | responses: 17 | '200': 18 | description: | 19 | Terms were successfully added to the index. 20 | '400': 21 | description: > 22 | Request was not processed because payload was incomplete or 23 | incorrect. 24 | content: 25 | application/json: 26 | schema: 27 | $ref: '#/components/schemas/error' 28 | requestBody: 29 | content: 30 | application/json: 31 | schema: 32 | $ref: '#/components/schemas/terms' 33 | description: | 34 | List of terms to be added to the index. 35 | required: true 36 | /query: 37 | post: 38 | description: | 39 | Search for all terms in the payload. 40 | responses: 41 | '200': 42 | description: | 43 | Returns a list of all the terms along with their frequency, 44 | documents the terms appear in and link to the said documents. 45 | content: 46 | application/json: 47 | schema: 48 | $ref: '#/components/schemas/results' 49 | '400': 50 | description: > 51 | Request was not processed because payload was incomplete or 52 | incorrect. 53 | content: 54 | application/json: 55 | schema: 56 | $ref: '#/components/schemas/error' 57 | parameters: [] 58 | components: 59 | schemas: 60 | error: 61 | type: object 62 | properties: 63 | msg: 64 | type: string 65 | term: 66 | type: object 67 | required: 68 | - title 69 | - token 70 | - doc_id 71 | - line_index 72 | - token_index 73 | properties: 74 | title: 75 | description: | 76 | Title of the document to which the term belongs. 77 | type: string 78 | token: 79 | description: | 80 | The term to be added to the index. 81 | type: string 82 | doc_id: 83 | description: | 84 | The unique hash for each document. 85 | type: string 86 | line_index: 87 | description: | 88 | Line index at which the term occurs in the document. 89 | type: integer 90 | token_index: 91 | description: | 92 | Position of the term in the document. 93 | type: integer 94 | terms: 95 | type: object 96 | properties: 97 | code: 98 | type: integer 99 | data: 100 | type: array 101 | items: 102 | $ref: '#/components/schemas/term' 103 | results: 104 | type: object 105 | properties: 106 | count: 107 | type: integer 108 | data: 109 | type: array 110 | items: 111 | $ref: '#/components/schemas/result' 112 | result: 113 | type: object 114 | properties: 115 | doc_id: 116 | type: string 117 | score: 118 | type: integer 119 | -------------------------------------------------------------------------------- /Chapter06/goophr/concierge/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.10 2 | 3 | 4 | ADD . /go/src/github.com/last-ent/distributed-go/chapter5/goophr/concierge 5 | 6 | WORKDIR /go/src/github.com/last-ent/distributed-go/chapter5/goophr/concierge 7 | 8 | RUN go install github.com/last-ent/distributed-go/chapter5/goophr/concierge 9 | 10 | ENTRYPOINT /go/bin/concierge 11 | 12 | EXPOSE 9000 13 | -------------------------------------------------------------------------------- /Chapter06/goophr/concierge/api/feeder.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "crypto/sha1" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/last-ent/distributed-go/chapter6/goophr/concierge/common" 13 | ) 14 | 15 | type payload struct { 16 | URL string `json:"url"` 17 | Title string `json:"title"` 18 | } 19 | 20 | type document struct { 21 | Doc string `json:"-"` 22 | Title string `json:"title"` 23 | DocID string `json:"DocID"` 24 | } 25 | 26 | type token struct { 27 | Line string `json:"-"` 28 | Token string `json:"token"` 29 | Title string `json:"title"` 30 | DocID string `json:"doc_id"` 31 | LIndex int `json:"line_index"` 32 | Index int `json:"token_index"` 33 | } 34 | 35 | type dMsg struct { 36 | DocID string 37 | Ch chan document 38 | } 39 | 40 | type lMsg struct { 41 | LIndex int 42 | DocID string 43 | Ch chan string 44 | } 45 | 46 | type lMeta struct { 47 | LIndex int 48 | DocID string 49 | Line string 50 | } 51 | 52 | type dAllMsg struct { 53 | Ch chan []document 54 | } 55 | 56 | // done signals all listening goroutines to stop. 57 | var done chan bool 58 | 59 | // dGetCh is used to retrieve a single document from store. 60 | var dGetCh chan dMsg 61 | 62 | // lGetCh is used to retrieve a single line from store. 63 | var lGetCh chan lMsg 64 | 65 | // lStoreCh is used to put a line into store. 66 | var lStoreCh chan lMeta 67 | 68 | // iAddCh is used to add token to index (Librarian). 69 | var iAddCh chan token 70 | 71 | // dStoreCh is used to put a document into store. 72 | var dStoreCh chan document 73 | 74 | // dProcessCh is used to process a document and convert it to tokens. 75 | var dProcessCh chan document 76 | 77 | // dGetAllCh is used to retrieve all documents in store. 78 | var dGetAllCh chan dAllMsg 79 | 80 | // pProcessCh is used to process the /feeder's payload and start the indexing process. 81 | var pProcessCh chan payload 82 | 83 | // StartFeederSystem initializes all channels and starts all goroutines. 84 | // We are using a standard function instead of `init()` 85 | // because we don't want the channels & goroutines to be initialized during testing. 86 | // Unless explicitly required by a particular test. 87 | func StartFeederSystem() { 88 | done = make(chan bool) 89 | 90 | dGetCh = make(chan dMsg, 8) 91 | dGetAllCh = make(chan dAllMsg) 92 | 93 | iAddCh = make(chan token, 8) 94 | pProcessCh = make(chan payload, 8) 95 | 96 | dStoreCh = make(chan document, 8) 97 | dProcessCh = make(chan document, 8) 98 | lGetCh = make(chan lMsg) 99 | lStoreCh = make(chan lMeta, 8) 100 | 101 | for i := 0; i < 4; i++ { 102 | go indexAdder(iAddCh, done) 103 | go docProcessor(pProcessCh, dStoreCh, dProcessCh, done) 104 | go indexProcessor(dProcessCh, lStoreCh, iAddCh, done) 105 | } 106 | 107 | go docStore(dStoreCh, dGetCh, dGetAllCh, done) 108 | go lineStore(lStoreCh, lGetCh, done) 109 | } 110 | 111 | // indexAdder adds token to index (Librarian). 112 | func indexAdder(ch chan token, done chan bool) { 113 | for { 114 | select { 115 | case tok := <-ch: 116 | fmt.Println("adding to librarian:", tok.Token) 117 | 118 | case <-done: 119 | common.Log("Exiting indexAdder.") 120 | return 121 | } 122 | } 123 | } 124 | 125 | // lineStore maintains a catalog of all lines for all documents being indexed. 126 | func lineStore(ch chan lMeta, callback chan lMsg, done chan bool) { 127 | store := map[string]string{} 128 | for { 129 | select { 130 | case line := <-ch: 131 | id := fmt.Sprintf("%s-%d", line.DocID, line.LIndex) 132 | store[id] = line.Line 133 | 134 | case ch := <-callback: 135 | line := "" 136 | id := fmt.Sprintf("%s-%d", ch.DocID, ch.LIndex) 137 | if l, exists := store[id]; exists { 138 | line = l 139 | } 140 | ch.Ch <- line 141 | case <-done: 142 | common.Log("Exiting docStore.") 143 | return 144 | } 145 | } 146 | } 147 | 148 | // indexProcessor is responsible for converting a document into tokens for indexing. 149 | func indexProcessor(ch chan document, lStoreCh chan lMeta, iAddCh chan token, done chan bool) { 150 | for { 151 | select { 152 | case doc := <-ch: 153 | docLines := strings.Split(doc.Doc, "\n") 154 | 155 | lin := 0 156 | for _, line := range docLines { 157 | if strings.TrimSpace(line) == "" { 158 | continue 159 | } 160 | 161 | lStoreCh <- lMeta{ 162 | LIndex: lin, 163 | Line: line, 164 | DocID: doc.DocID, 165 | } 166 | 167 | index := 0 168 | words := strings.Fields(line) 169 | for _, word := range words { 170 | if tok, valid := common.SimplifyToken(word); valid { 171 | iAddCh <- token{ 172 | Token: tok, 173 | LIndex: lin, 174 | Line: line, 175 | Index: index, 176 | DocID: doc.DocID, 177 | Title: doc.Title, 178 | } 179 | index++ 180 | } 181 | } 182 | lin++ 183 | } 184 | 185 | case <-done: 186 | common.Log("Exiting indexProcessor.") 187 | return 188 | } 189 | } 190 | } 191 | 192 | // docStore maintains a catalog of all documents being indexed. 193 | func docStore(add chan document, get chan dMsg, dGetAllCh chan dAllMsg, done chan bool) { 194 | store := map[string]document{} 195 | 196 | for { 197 | select { 198 | case doc := <-add: 199 | store[doc.DocID] = doc 200 | case m := <-get: 201 | m.Ch <- store[m.DocID] 202 | case ch := <-dGetAllCh: 203 | docs := []document{} 204 | for _, doc := range store { 205 | docs = append(docs, doc) 206 | } 207 | ch.Ch <- docs 208 | case <-done: 209 | common.Log("Exiting docStore.") 210 | return 211 | } 212 | } 213 | } 214 | 215 | // docProcessor processes new document payloads. 216 | func docProcessor(in chan payload, dStoreCh chan document, dProcessCh chan document, done chan bool) { 217 | for { 218 | select { 219 | case newDoc := <-in: 220 | var err error 221 | doc := "" 222 | 223 | if doc, err = getFile(newDoc.URL); err != nil { 224 | common.Warn(err.Error()) 225 | continue 226 | } 227 | 228 | titleID := getTitleHash(newDoc.Title) 229 | msg := document{ 230 | Doc: doc, 231 | DocID: titleID, 232 | Title: newDoc.Title, 233 | } 234 | 235 | dStoreCh <- msg 236 | dProcessCh <- msg 237 | case <-done: 238 | common.Log("Exiting docProcessor.") 239 | return 240 | } 241 | } 242 | } 243 | 244 | // getTitleHash returns a new hash ID everytime it is called. 245 | // Based on: https://gobyexample.com/sha1-hashes 246 | func getTitleHash(title string) string { 247 | hash := sha1.New() 248 | title = strings.ToLower(title) 249 | 250 | str := fmt.Sprintf("%s-%s", time.Now(), title) 251 | hash.Write([]byte(str)) 252 | 253 | hByte := hash.Sum(nil) 254 | 255 | return fmt.Sprintf("%x", hByte) 256 | } 257 | 258 | // getFile returns file content after retrieving it from URL. 259 | func getFile(URL string) (string, error) { 260 | var res *http.Response 261 | var err error 262 | 263 | if res, err = http.Get(URL); err != nil { 264 | errMsg := fmt.Errorf("Unable to retrieve URL: %s.\nError: %s", URL, err) 265 | 266 | return "", errMsg 267 | 268 | } 269 | if res.StatusCode > 200 { 270 | errMsg := fmt.Errorf("Unable to retrieve URL: %s.\nStatus Code: %d", URL, res.StatusCode) 271 | 272 | return "", errMsg 273 | } 274 | 275 | body, err := ioutil.ReadAll(res.Body) 276 | defer res.Body.Close() 277 | 278 | if err != nil { 279 | errMsg := fmt.Errorf("Error while reading response: URL: %s.\nError: %s", URL, res.StatusCode, err.Error()) 280 | 281 | return "", errMsg 282 | } 283 | 284 | return string(body), nil 285 | } 286 | 287 | // FeedHandler start processing the payload which contains the file to index. 288 | func FeedHandler(w http.ResponseWriter, r *http.Request) { 289 | if r.Method == "GET" { 290 | ch := make(chan []document) 291 | dGetAllCh <- dAllMsg{Ch: ch} 292 | docs := <-ch 293 | close(ch) 294 | 295 | if serializedPayload, err := json.Marshal(docs); err == nil { 296 | w.Write(serializedPayload) 297 | } else { 298 | common.Warn("Unable to serialize all docs: " + err.Error()) 299 | w.WriteHeader(http.StatusInternalServerError) 300 | w.Write([]byte(`{"code": 500, "msg": "Error occurred while trying to retrieve documents."}`)) 301 | } 302 | return 303 | } else if r.Method != "POST" { 304 | w.WriteHeader(http.StatusMethodNotAllowed) 305 | w.Write([]byte(`{"code": 405, "msg": "Method Not Allowed."}`)) 306 | return 307 | } 308 | 309 | decoder := json.NewDecoder(r.Body) 310 | defer r.Body.Close() 311 | 312 | var newDoc payload 313 | decoder.Decode(&newDoc) 314 | pProcessCh <- newDoc 315 | 316 | w.Write([]byte(`{"code": 200, "msg": "Request is being processed."}`)) 317 | } 318 | -------------------------------------------------------------------------------- /Chapter06/goophr/concierge/api/feeder_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | ) 9 | 10 | func TestGetTitleHash(t *testing.T) { 11 | 12 | h1 := getTitleHash("A-Title") 13 | h2 := getTitleHash("Diff Title") 14 | hDup := getTitleHash("A-Title") 15 | 16 | for _, tc := range []struct { 17 | name string 18 | hashes []string 19 | expected bool 20 | }{ 21 | {"Different Titles", []string{h1, h2}, false}, 22 | {"Duplicate Titles", []string{h1, hDup}, false}, 23 | {"Same hashes", []string{h2, h2}, true}, 24 | } { 25 | t.Run(tc.name, func(t *testing.T) { 26 | actual := tc.hashes[0] == tc.hashes[1] 27 | if actual != tc.expected { 28 | t.Error(actual, tc.expected, tc.hashes) 29 | } 30 | }) 31 | } 32 | } 33 | 34 | func TestGetFile(t *testing.T) { 35 | doc := "Server returned text!" 36 | testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 37 | w.Write([]byte(doc)) 38 | })) 39 | defer testServer.Close() 40 | 41 | rDoc, err := getFile(testServer.URL) 42 | if err != nil { 43 | t.Error("Error while retrieving document", err) 44 | } 45 | if doc != rDoc { 46 | t.Error(doc, "!=", rDoc) 47 | } 48 | } 49 | 50 | func TestIndexProcessor(t *testing.T) { 51 | ch1 := make(chan document, 1) 52 | ch2 := make(chan lMeta, 1) 53 | ch3 := make(chan token, 3) 54 | done := make(chan bool) 55 | 56 | go indexProcessor(ch1, ch2, ch3, done) 57 | 58 | ch1 <- document{ 59 | DocID: "a-hash", 60 | Title: "a-title", 61 | Doc: "Golang Programming rocks!", 62 | } 63 | 64 | for i, tc := range []string{ 65 | "golang", "programming", "rocks", 66 | } { 67 | t.Run(fmt.Sprintf("Testing if '%s' is returned. at index: %d", tc, i+1), func(t *testing.T) { 68 | tok := <-ch3 69 | if tok.Token != tc { 70 | t.Error(tok.Token, "!=", tc) 71 | } 72 | if tok.Index != i { 73 | t.Error(tok.Index, "!=", i) 74 | } 75 | }) 76 | } 77 | close(done) 78 | 79 | } 80 | -------------------------------------------------------------------------------- /Chapter06/goophr/concierge/api/query.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | 7 | "github.com/last-ent/distributed-go/chapter6/goophr/concierge/common" 8 | ) 9 | 10 | func QueryHandler(w http.ResponseWriter, r *http.Request) { 11 | if r.Method != "POST" { 12 | w.WriteHeader(http.StatusMethodNotAllowed) 13 | w.Write([]byte(`{"code": 405, "msg": "Method Not Allowed."}`)) 14 | return 15 | } 16 | 17 | decoder := json.NewDecoder(r.Body) 18 | defer r.Body.Close() 19 | 20 | var searchTerms []string 21 | if err := decoder.Decode(&searchTerms); err != nil { 22 | common.Warn("Unable to parse request." + err.Error()) 23 | 24 | w.WriteHeader(http.StatusBadRequest) 25 | w.Write([]byte(`{"code": 400, "msg": "Unable to parse payload."}`)) 26 | return 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Chapter06/goophr/concierge/common/helpers.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "regexp" 7 | "strings" 8 | ) 9 | 10 | func Log(msg string) { 11 | log.Println("INFO - ", msg) 12 | } 13 | 14 | func Warn(msg string) { 15 | log.Println("---------------------------") 16 | log.Println(fmt.Sprintf("WARN: %s", msg)) 17 | log.Println("---------------------------") 18 | } 19 | 20 | var punctuations = regexp.MustCompile(`^\p{P}+|\p{P}+$`) 21 | 22 | var stopWords = []string{ 23 | "a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "aren't", "as", "at", 24 | "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "can't", "cannot", "could", 25 | "couldn't", "did", "didn't", "do", "does", "doesn't", "doing", "don't", "down", "during", "each", "few", "for", 26 | "from", "further", "had", "hadn't", "has", "hasn't", "have", "haven't", "having", "he", "he'd", "he'll", "he's", 27 | "her", "here", "here's", "hers", "herself", "him", "himself", "his", "how", "how's", "i", "i'd", "i'll", "i'm", 28 | "i've", "if", "in", "into", "is", "isn't", "it", "it's", "its", "itself", "let's", "me", "more", "most", "mustn't", 29 | "my", "myself", "no", "nor", "not", "of", "off", "on", "once", "only", "or", "other", "ought", "our", "ours", 30 | "ourselves", "out", "over", "own", "same", "shan't", "she", "she'd", "she'll", "she's", "should", "shouldn't", 31 | "so", "some", "such", "than", "that", "that's", "the", "their", "theirs", "them", "themselves", "then", "there", 32 | "there's", "these", "they", "they'd", "they'll", "they're", "they've", "this", "those", "through", "to", "too", 33 | "under", "until", "up", "very", "was", "wasn't", "we", "we'd", "we'll", "we're", "we've", "were", "weren't", "what", 34 | "what's", "when", "when's", "where", "where's", "which", "while", "who", "who's", "whom", "why", "why's", "with", 35 | "won't", "would", "wouldn't", "you", "you'd", "you'll", "you're", "you've", "your", "yours", "yourself", "yourselves"} 36 | 37 | // SimplifyToken is responsible to normalizing a string token and 38 | // also checks whether the token should be indexed or not. 39 | func SimplifyToken(token string) (string, bool) { 40 | simpleToken := strings.ToLower(punctuations.ReplaceAllString(token, "")) 41 | 42 | for _, stopWord := range stopWords { 43 | if stopWord == simpleToken { 44 | return "", false 45 | } 46 | } 47 | 48 | return simpleToken, true 49 | } 50 | -------------------------------------------------------------------------------- /Chapter06/goophr/concierge/common/state.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // Logic required to manage state for concierge 4 | -------------------------------------------------------------------------------- /Chapter06/goophr/concierge/common/test_helpers.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "testing" 4 | 5 | const checkMark = "\u2713" 6 | const ballotX = "\u2717" 7 | const prefix = "\t\t - " 8 | 9 | // OnTestSuccess is for pass. 10 | func OnTestSuccess(t *testing.T, msg string) { 11 | t.Log(prefix+msg, checkMark) 12 | } 13 | 14 | //OnTestError is for fail. 15 | func OnTestError(t *testing.T, msg string) { 16 | t.Error(prefix+msg, ballotX) 17 | } 18 | 19 | //OnTestUnexpectedError is for unexpected fail. 20 | func OnTestUnexpectedError(t *testing.T, err error) { 21 | OnTestError(t, "Unexpected Error:\n"+err.Error()) 22 | } 23 | -------------------------------------------------------------------------------- /Chapter06/goophr/concierge/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/last-ent/distributed-go/chapter6/goophr/concierge/api" 7 | "github.com/last-ent/distributed-go/chapter6/goophr/concierge/common" 8 | ) 9 | 10 | func main() { 11 | common.Log("Adding API handlers...") 12 | http.HandleFunc("/api/feeder", api.FeedHandler) 13 | 14 | common.Log("Starting feeder...") 15 | api.StartFeederSystem() 16 | 17 | common.Log("Starting Goophr Concierge server on port :8080...") 18 | http.ListenAndServe(":8080", nil) 19 | } 20 | -------------------------------------------------------------------------------- /Chapter06/openapi/concierge.yaml: -------------------------------------------------------------------------------- 1 | # openapi/concierge.yaml 2 | 3 | openapi: 3.0.0 4 | servers: 5 | - url: /api 6 | info: 7 | title: Goophr Concierge API 8 | version: '1.0' 9 | description: > 10 | API responsible for responding to user input and communicating with Goophr 11 | Librarian. 12 | paths: 13 | /feeder: 14 | post: 15 | description: | 16 | Register new document to be indexed. 17 | responses: 18 | '200': 19 | description: | 20 | Request was successfully completed. 21 | content: 22 | application/json: 23 | schema: 24 | $ref: '#/components/schemas/response' 25 | '400': 26 | description: > 27 | Request was not processed because payload was incomplete or 28 | incorrect. 29 | content: 30 | application/json: 31 | schema: 32 | $ref: '#/components/schemas/response' 33 | requestBody: 34 | content: 35 | application/json: 36 | schema: 37 | $ref: '#/components/schemas/document' 38 | required: true 39 | /query: 40 | post: 41 | description: | 42 | Search query 43 | responses: 44 | '200': 45 | description: | 46 | Response consists of links to document 47 | content: 48 | application/json: 49 | schema: 50 | type: array 51 | items: 52 | $ref: '#/components/schemas/document' 53 | requestBody: 54 | content: 55 | application/json: 56 | schema: 57 | type: array 58 | items: 59 | type: string 60 | required: true 61 | components: 62 | schemas: 63 | response: 64 | type: object 65 | properties: 66 | code: 67 | type: integer 68 | description: Status code to send in response 69 | msg: 70 | type: string 71 | description: Message to send in response 72 | document: 73 | type: object 74 | required: 75 | - title 76 | - link 77 | properties: 78 | title: 79 | type: string 80 | description: Title of the document 81 | link: 82 | type: string 83 | description: Link to the document 84 | -------------------------------------------------------------------------------- /Chapter07/feeder.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | ) 10 | 11 | type tPayload struct { 12 | Token string `json:"token"` 13 | Title string `json:"title"` 14 | DocID string `json:"doc_id"` 15 | LIndex int `json:"line_index"` 16 | Index int `json:"token_index"` 17 | } 18 | 19 | type msgS struct { 20 | Code int `json:"code"` 21 | Msg string `json:"msg"` 22 | } 23 | 24 | func main() { 25 | // Searching for "apple" should return Book 1 at the top of search results. 26 | // Searching for "cake" should return Book 3 at the top. 27 | for bookX, terms := range map[string][]string{ 28 | "Book 1": []string{"apple", "apple", "cat", "zebra"}, 29 | "Book 2": []string{"banana", "cake", "zebra"}, 30 | "Book 3": []string{"apple", "cake", "cake", "whale"}, 31 | } { 32 | for lin, term := range terms { 33 | payload, _ := json.Marshal(tPayload{ 34 | Token: term, 35 | Title: bookX + term, 36 | DocID: bookX, 37 | LIndex: lin, 38 | }) 39 | resp, err := http.Post( 40 | "http://localhost:9090/api/index", 41 | "application/json", 42 | bytes.NewBuffer(payload), 43 | ) 44 | if err != nil { 45 | panic(err) 46 | } 47 | body, _ := ioutil.ReadAll(resp.Body) 48 | defer resp.Body.Close() 49 | 50 | var msg msgS 51 | json.Unmarshal(body, &msg) 52 | log.Println(msg) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Chapter07/goophr/librarian/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.10 2 | 3 | 4 | ADD . /go/src/github.com/last-ent/distributed-go/chapter5/goophr/librarian 5 | 6 | RUN go install github.com/last-ent/distributed-go/chapter5/goophr/librarian 7 | 8 | ENTRYPOINT /go/bin/librarian 9 | 10 | EXPOSE 9000 11 | -------------------------------------------------------------------------------- /Chapter07/goophr/librarian/api/index.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | ) 10 | 11 | // tPayload is used to parse the JSON payload consisting of Token data. 12 | type tPayload struct { 13 | Token string `json:"token"` 14 | Title string `json:"title"` 15 | DocID string `json:"doc_id"` 16 | LIndex int `json:"line_index"` 17 | Index int `json:"token_index"` 18 | } 19 | 20 | type tIndex struct { 21 | Index int 22 | LIndex int 23 | } 24 | 25 | func (ti *tIndex) String() string { 26 | return fmt.Sprintf("i: %d, li: %d", ti.Index, ti.LIndex) 27 | } 28 | 29 | type tIndices []tIndex 30 | 31 | // document - key in Indices represent Line Index. 32 | type document struct { 33 | Count int 34 | DocID string 35 | Title string 36 | Indices map[int]tIndices 37 | } 38 | 39 | func (d *document) String() string { 40 | str := fmt.Sprintf("%s (%s): %d\n", d.Title, d.DocID, d.Count) 41 | var buffer bytes.Buffer 42 | 43 | for lin, tis := range d.Indices { 44 | var lBuffer bytes.Buffer 45 | for _, ti := range tis { 46 | lBuffer.WriteString(fmt.Sprintf("%s ", ti.String())) 47 | } 48 | buffer.WriteString(fmt.Sprintf("@%d -> %s\n", lin, lBuffer.String())) 49 | } 50 | return str + buffer.String() 51 | } 52 | 53 | // documentCatalog - key represents DocID. 54 | type documentCatalog map[string]*document 55 | 56 | func (dc *documentCatalog) String() string { 57 | return fmt.Sprintf("%#v", dc) 58 | } 59 | 60 | // tCatalog - key in map represents Token. 61 | type tCatalog map[string]documentCatalog 62 | 63 | func (tc *tCatalog) String() string { 64 | return fmt.Sprintf("%#v", tc) 65 | } 66 | 67 | type tcCallback struct { 68 | Token string 69 | Ch chan tcMsg 70 | } 71 | 72 | type tcMsg struct { 73 | Token string 74 | DC documentCatalog 75 | } 76 | 77 | // pProcessCh is used to process /index's payload and start process to add the token to catalog (tCatalog). 78 | var pProcessCh chan tPayload 79 | 80 | // tcGet is used to retrieve a token's catalog (documentCatalog). 81 | var tcGet chan tcCallback 82 | 83 | func StartIndexSystem() { 84 | pProcessCh = make(chan tPayload, 100) 85 | tcGet = make(chan tcCallback, 20) 86 | go tIndexer(pProcessCh, tcGet) 87 | } 88 | 89 | // tIndexer maintains a catalog of all tokens along with where they occur within documents. 90 | func tIndexer(ch chan tPayload, callback chan tcCallback) { 91 | store := tCatalog{} 92 | for { 93 | select { 94 | case msg := <-callback: 95 | dc := store[msg.Token] 96 | msg.Ch <- tcMsg{ 97 | DC: dc, 98 | Token: msg.Token, 99 | } 100 | 101 | case pd := <-ch: 102 | dc, exists := store[pd.Token] 103 | if !exists { 104 | dc = documentCatalog{} 105 | store[pd.Token] = dc 106 | } 107 | 108 | doc, exists := dc[pd.DocID] 109 | if !exists { 110 | doc = &document{ 111 | DocID: pd.DocID, 112 | Title: pd.Title, 113 | Indices: map[int]tIndices{}, 114 | } 115 | dc[pd.DocID] = doc 116 | } 117 | 118 | tin := tIndex{ 119 | Index: pd.Index, 120 | LIndex: pd.LIndex, 121 | } 122 | doc.Indices[tin.LIndex] = append(doc.Indices[tin.LIndex], tin) 123 | doc.Count++ 124 | } 125 | } 126 | } 127 | 128 | func IndexHandler(w http.ResponseWriter, r *http.Request) { 129 | if r.Method != "POST" { 130 | w.WriteHeader(http.StatusMethodNotAllowed) 131 | w.Write([]byte(`{"code": 405, "msg": "Method Not Allowed."}`)) 132 | return 133 | } 134 | 135 | decoder := json.NewDecoder(r.Body) 136 | defer r.Body.Close() 137 | 138 | var tp tPayload 139 | decoder.Decode(&tp) 140 | log.Printf("Token received%#v\n", tp) 141 | 142 | pProcessCh <- tp 143 | 144 | w.Write([]byte(`{"code": 200, "msg": "Tokens are being added to index."}`)) 145 | } 146 | -------------------------------------------------------------------------------- /Chapter07/goophr/librarian/api/query.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "sort" 7 | 8 | "github.com/last-ent/distributed-go/chapter7/goophr/librarian/common" 9 | ) 10 | 11 | type docResult struct { 12 | DocID string `json:"doc_id"` 13 | Score int `json:"doc_score"` 14 | Indices tIndices `json:"token_indices"` 15 | } 16 | 17 | type result struct { 18 | Count int `json:"count"` 19 | Data []docResult `json:"data"` 20 | } 21 | 22 | // getResults returns unsorted search results & a map of documents containing tokens. 23 | func getResults(out chan tcMsg, count int) tCatalog { 24 | tc := tCatalog{} 25 | for i := 0; i < count; i++ { 26 | dc := <-out 27 | tc[dc.Token] = dc.DC 28 | } 29 | close(out) 30 | 31 | return tc 32 | } 33 | 34 | func getFScores(docIDScore map[string]int) (map[int][]string, []int) { 35 | // fScore maps frequency score to set of documents. 36 | fScore := map[int][]string{} 37 | 38 | fSorted := []int{} 39 | 40 | for dID, score := range docIDScore { 41 | fs := fScore[score] 42 | fScore[score] = append(fs, dID) 43 | fSorted = append(fSorted, score) 44 | } 45 | 46 | sort.Sort(sort.Reverse(sort.IntSlice(fSorted))) 47 | 48 | return fScore, fSorted 49 | } 50 | 51 | func getDocMaps(tc tCatalog) (map[string]int, map[string]tIndices) { 52 | // docIDScore maps DocIDs to occurences of all tokens. 53 | // key: DocID. 54 | // val: Sum of all occurences of tokens so far. 55 | docIDScore := map[string]int{} 56 | docIndices := map[string]tIndices{} 57 | 58 | // for each token's catalog 59 | for _, dc := range tc { 60 | // for each document registered under the token 61 | for dID, doc := range dc { 62 | // add to docID score 63 | var tokIndices tIndices 64 | for _, tList := range doc.Indices { 65 | tokIndices = append(tokIndices, tList...) 66 | } 67 | docIDScore[dID] += doc.Count 68 | 69 | dti := docIndices[dID] 70 | docIndices[dID] = append(dti, tokIndices...) 71 | } 72 | } 73 | 74 | return docIDScore, docIndices 75 | } 76 | 77 | func sortResults(tc tCatalog) []docResult { 78 | docIDScore, docIndices := getDocMaps(tc) 79 | fScore, fSorted := getFScores(docIDScore) 80 | 81 | results := []docResult{} 82 | addedDocs := map[string]bool{} 83 | 84 | for _, score := range fSorted { 85 | for _, docID := range fScore[score] { 86 | if _, exists := addedDocs[docID]; exists { 87 | continue 88 | } 89 | results = append(results, docResult{ 90 | DocID: docID, 91 | Score: score, 92 | Indices: docIndices[docID], 93 | }) 94 | addedDocs[docID] = false 95 | } 96 | } 97 | return results 98 | } 99 | 100 | // getSearchResults returns a list of documents. 101 | // They are listed in descending order of occurences. 102 | func getSearchResults(sts []string) []docResult { 103 | 104 | callback := make(chan tcMsg) 105 | 106 | for _, st := range sts { 107 | go func(term string) { 108 | tcGet <- tcCallback{ 109 | Token: term, 110 | Ch: callback, 111 | } 112 | }(st) 113 | } 114 | 115 | cts := getResults(callback, len(sts)) 116 | results := sortResults(cts) 117 | return results 118 | } 119 | 120 | func QueryHandler(w http.ResponseWriter, r *http.Request) { 121 | if r.Method != "POST" { 122 | w.WriteHeader(http.StatusMethodNotAllowed) 123 | w.Write([]byte(`{"code": 405, "msg": "Method Not Allowed."}`)) 124 | return 125 | } 126 | 127 | decoder := json.NewDecoder(r.Body) 128 | defer r.Body.Close() 129 | 130 | var searchTerms []string 131 | decoder.Decode(&searchTerms) 132 | 133 | results := getSearchResults(searchTerms) 134 | 135 | payload := result{ 136 | Count: len(results), 137 | Data: results, 138 | } 139 | 140 | if serializedPayload, err := json.Marshal(payload); err == nil { 141 | w.Header().Add("Content-Type", "application/json") 142 | w.Write(serializedPayload) 143 | } else { 144 | common.Warn("Unable to serialize all docs: " + err.Error()) 145 | w.WriteHeader(http.StatusInternalServerError) 146 | w.Write([]byte(`{"code": 500, "msg": "Error occurred while trying to retrieve documents."}`)) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /Chapter07/goophr/librarian/common/helpers.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | ) 7 | 8 | func Log(msg string) { 9 | log.Println("INFO - ", msg) 10 | } 11 | 12 | func Warn(msg string) { 13 | log.Println("---------------------------") 14 | log.Println(fmt.Sprintf("WARN: %s", msg)) 15 | log.Println("---------------------------") 16 | } 17 | -------------------------------------------------------------------------------- /Chapter07/goophr/librarian/common/state.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // Logic required to manage state for Librarian 4 | -------------------------------------------------------------------------------- /Chapter07/goophr/librarian/common/test_helpers.go: -------------------------------------------------------------------------------- 1 | 2 | package common 3 | 4 | import "testing" 5 | 6 | const checkMark = "\u2713" 7 | const ballotX = "\u2717" 8 | const prefix = "\t\t - " 9 | 10 | // OnTestSuccess is for pass. 11 | func OnTestSuccess(t *testing.T, msg string){ 12 | t.Log(prefix+msg, checkMark) 13 | } 14 | 15 | //OnTestError is for fail. 16 | func OnTestError(t *testing.T, msg string) { 17 | t.Error(prefix+msg, ballotX) 18 | } 19 | 20 | //OnTestUnexpectedError is for unexpected fail. 21 | func OnTestUnexpectedError(t *testing.T, err error) { 22 | OnTestError(t, "Unexpected Error:\n"+err.Error()) 23 | } 24 | 25 | -------------------------------------------------------------------------------- /Chapter07/goophr/librarian/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/last-ent/distributed-go/chapter7/goophr/librarian/api" 7 | "github.com/last-ent/distributed-go/chapter7/goophr/librarian/common" 8 | ) 9 | 10 | func main() { 11 | common.Log("Adding API handlers...") 12 | http.HandleFunc("/api/index", api.IndexHandler) 13 | http.HandleFunc("/api/query", api.QueryHandler) 14 | 15 | common.Log("Starting index...") 16 | api.StartIndexSystem() 17 | 18 | common.Log("Starting Goophr Librarian server on port :9090...") 19 | http.ListenAndServe(":9090", nil) 20 | } 21 | -------------------------------------------------------------------------------- /Chapter08/goophr/concierge/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.10 2 | 3 | 4 | ADD . /go/src/github.com/last-ent/distributed-go/chapter8/goophr/concierge 5 | 6 | WORKDIR /go/src/github.com/last-ent/distributed-go/chapter8/goophr/concierge 7 | 8 | RUN go install github.com/last-ent/distributed-go/chapter8/goophr/concierge 9 | 10 | ENTRYPOINT /go/bin/concierge 11 | 12 | -------------------------------------------------------------------------------- /Chapter08/goophr/concierge/api/feeder.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha1" 6 | "encoding/json" 7 | "fmt" 8 | "io/ioutil" 9 | "net/http" 10 | "strings" 11 | "time" 12 | 13 | "github.com/last-ent/distributed-go/chapter8/goophr/concierge/common" 14 | ) 15 | 16 | type payload struct { 17 | URL string `json:"url"` 18 | Title string `json:"title"` 19 | } 20 | 21 | type document struct { 22 | Doc string `json:"-"` 23 | Title string `json:"title"` 24 | DocID string `json:"-"` 25 | URL string `json:"url"` 26 | } 27 | 28 | type token struct { 29 | Line string `json:"-"` 30 | Token string `json:"token"` 31 | Title string `json:"title"` 32 | DocID string `json:"doc_id"` 33 | LIndex int `json:"line_index"` 34 | Index int `json:"token_index"` 35 | } 36 | 37 | type dMsg struct { 38 | DocID string 39 | Ch chan document 40 | } 41 | 42 | type lMsg struct { 43 | LIndex int 44 | DocID string 45 | Ch chan string 46 | } 47 | 48 | type lMeta struct { 49 | LIndex int 50 | DocID string 51 | Line string 52 | } 53 | 54 | type dAllMsg struct { 55 | Ch chan []document 56 | } 57 | 58 | // done signals all listening goroutines to stop. 59 | var done chan bool 60 | 61 | // dGetCh is used to retrieve a single document from store. 62 | var dGetCh chan dMsg 63 | 64 | // lGetCh is used to retrieve a single line from store. 65 | var lGetCh chan lMsg 66 | 67 | // lStoreCh is used to put a line into store. 68 | var lStoreCh chan lMeta 69 | 70 | // iAddCh is used to add token to index (Librarian). 71 | var iAddCh chan token 72 | 73 | // dStoreCh is used to put a document into store. 74 | var dStoreCh chan document 75 | 76 | // dProcessCh is used to process a document and convert it to tokens. 77 | var dProcessCh chan document 78 | 79 | // dGetAllCh is used to retrieve all documents in store. 80 | var dGetAllCh chan dAllMsg 81 | 82 | // pProcessCh is used to process the /feeder's payload and start the indexing process. 83 | var pProcessCh chan payload 84 | 85 | // StartFeederSystem initializes all channels and starts all goroutines. 86 | // We are using a standard function instead of `init()` 87 | // because we don't want the channels & goroutines to be initialized during testing. 88 | // Unless explicitly required by a particular test. 89 | func StartFeederSystem() { 90 | done = make(chan bool) 91 | 92 | dGetCh = make(chan dMsg, 8) 93 | dGetAllCh = make(chan dAllMsg) 94 | 95 | iAddCh = make(chan token, 8) 96 | pProcessCh = make(chan payload, 8) 97 | 98 | dStoreCh = make(chan document, 8) 99 | dProcessCh = make(chan document, 8) 100 | lGetCh = make(chan lMsg) 101 | lStoreCh = make(chan lMeta, 8) 102 | 103 | for i := 0; i < 4; i++ { 104 | go indexAdder(iAddCh, done) 105 | go docProcessor(pProcessCh, dStoreCh, dProcessCh, done) 106 | go indexProcessor(dProcessCh, lStoreCh, iAddCh, done) 107 | } 108 | 109 | go docStore(dStoreCh, dGetCh, dGetAllCh, done) 110 | go lineStore(lStoreCh, lGetCh, done) 111 | } 112 | 113 | func getTokIndexer(tok string) string { 114 | t := tok[0] 115 | indexerURL := librarianEndpoints["*"] 116 | 117 | if t >= 'a' && t <= 'm' { 118 | indexerURL = librarianEndpoints["a-m"] 119 | } else if t >= 'n' && t <= 'z' { 120 | indexerURL = librarianEndpoints["n-z"] 121 | } 122 | 123 | return indexerURL + "/index" 124 | } 125 | 126 | // indexAdder adds token to index (Librarian). 127 | func indexAdder(ch chan token, done chan bool) { 128 | for { 129 | select { 130 | case tok := <-ch: 131 | fmt.Println("adding to librarian:", tok.Token) 132 | serializedPayload, err := json.Marshal(tok) 133 | if err != nil { 134 | common.Warn("Unable to serialize the payload. Error:" + err.Error()) 135 | } 136 | 137 | indexerURL := getTokIndexer(tok.Token) 138 | resp, err := http.Post(indexerURL, "application/json", bytes.NewBuffer(serializedPayload)) 139 | if err != nil { 140 | common.Warn("Error while posting to Librarian. Error:" + err.Error()) 141 | continue 142 | } 143 | 144 | rawMsg, err := ioutil.ReadAll(resp.Body) 145 | defer resp.Body.Close() 146 | if err != nil { 147 | common.Warn("Error while reading response body:" + err.Error()) 148 | continue 149 | } 150 | 151 | msg := string(rawMsg) 152 | if resp.StatusCode > 200 { 153 | common.Warn("Error while posting to Librarian. Msg:" + msg) 154 | } else { 155 | common.Log("Request was posted to Librairan. Msg:" + msg) 156 | } 157 | 158 | case <-done: 159 | common.Log("Exiting indexAdder.") 160 | return 161 | } 162 | } 163 | } 164 | 165 | // lineStore maintains a catalog of all lines for all documents being indexed. 166 | func lineStore(ch chan lMeta, callback chan lMsg, done chan bool) { 167 | store := map[string]string{} 168 | for { 169 | select { 170 | case line := <-ch: 171 | id := fmt.Sprintf("%s-%d", line.DocID, line.LIndex) 172 | store[id] = line.Line 173 | 174 | case ch := <-callback: 175 | line := "" 176 | id := fmt.Sprintf("%s-%d", ch.DocID, ch.LIndex) 177 | if l, exists := store[id]; exists { 178 | line = l 179 | } 180 | ch.Ch <- line 181 | case <-done: 182 | common.Log("Exiting docStore.") 183 | return 184 | } 185 | } 186 | } 187 | 188 | // indexProcessor is responsible for converting a document into tokens for indexing. 189 | func indexProcessor(ch chan document, lStoreCh chan lMeta, iAddCh chan token, done chan bool) { 190 | for { 191 | select { 192 | case doc := <-ch: 193 | docLines := strings.Split(doc.Doc, "\n") 194 | 195 | lin := 0 196 | for _, line := range docLines { 197 | if strings.TrimSpace(line) == "" { 198 | continue 199 | } 200 | 201 | lStoreCh <- lMeta{ 202 | LIndex: lin, 203 | Line: line, 204 | DocID: doc.DocID, 205 | } 206 | 207 | index := 0 208 | words := strings.Fields(line) 209 | for _, word := range words { 210 | if tok, valid := common.SimplifyToken(word); valid { 211 | iAddCh <- token{ 212 | Token: tok, 213 | LIndex: lin, 214 | Line: line, 215 | Index: index, 216 | DocID: doc.DocID, 217 | Title: doc.Title, 218 | } 219 | index++ 220 | } 221 | } 222 | lin++ 223 | } 224 | 225 | case <-done: 226 | common.Log("Exiting indexProcessor.") 227 | return 228 | } 229 | } 230 | } 231 | 232 | // docStore maintains a catalog of all documents being indexed. 233 | func docStore(add chan document, get chan dMsg, dGetAllCh chan dAllMsg, done chan bool) { 234 | store := map[string]document{} 235 | 236 | for { 237 | select { 238 | case doc := <-add: 239 | store[doc.DocID] = doc 240 | case m := <-get: 241 | m.Ch <- store[m.DocID] 242 | case ch := <-dGetAllCh: 243 | docs := []document{} 244 | for _, doc := range store { 245 | docs = append(docs, doc) 246 | } 247 | ch.Ch <- docs 248 | case <-done: 249 | common.Log("Exiting docStore.") 250 | return 251 | } 252 | } 253 | } 254 | 255 | // docProcessor processes new document payloads. 256 | func docProcessor(in chan payload, dStoreCh chan document, dProcessCh chan document, done chan bool) { 257 | for { 258 | select { 259 | case newDoc := <-in: 260 | var err error 261 | doc := "" 262 | 263 | if doc, err = getFile(newDoc.URL); err != nil { 264 | common.Warn(err.Error()) 265 | continue 266 | } 267 | 268 | titleID := getTitleHash(newDoc.Title) 269 | msg := document{ 270 | Doc: doc, 271 | DocID: titleID, 272 | URL: newDoc.URL, 273 | Title: newDoc.Title, 274 | } 275 | 276 | dStoreCh <- msg 277 | dProcessCh <- msg 278 | case <-done: 279 | common.Log("Exiting docProcessor.") 280 | return 281 | } 282 | } 283 | } 284 | 285 | // getTitleHash returns a new hash ID everytime it is called. 286 | // Based on: https://gobyexample.com/sha1-hashes 287 | func getTitleHash(title string) string { 288 | hash := sha1.New() 289 | title = strings.ToLower(title) 290 | 291 | str := fmt.Sprintf("%s-%s", time.Now(), title) 292 | hash.Write([]byte(str)) 293 | 294 | hByte := hash.Sum(nil) 295 | 296 | return fmt.Sprintf("%x", hByte) 297 | } 298 | 299 | // getFile returns file content after retrieving it from URL. 300 | func getFile(URL string) (string, error) { 301 | var res *http.Response 302 | var err error 303 | 304 | if res, err = http.Get(URL); err != nil { 305 | errMsg := fmt.Errorf("Unable to retrieve URL: %s.\nError: %s", URL, err) 306 | 307 | return "", errMsg 308 | 309 | } 310 | if res.StatusCode > 200 { 311 | errMsg := fmt.Errorf("Unable to retrieve URL: %s.\nStatus Code: %d", URL, res.StatusCode) 312 | 313 | return "", errMsg 314 | } 315 | 316 | body, err := ioutil.ReadAll(res.Body) 317 | defer res.Body.Close() 318 | 319 | if err != nil { 320 | errMsg := fmt.Errorf("Error while reading response: URL: %s.\nError: %s", URL, res.StatusCode, err.Error()) 321 | 322 | return "", errMsg 323 | } 324 | 325 | return string(body), nil 326 | } 327 | 328 | // FeedHandler start processing the payload which contains the file to index. 329 | func FeedHandler(w http.ResponseWriter, r *http.Request) { 330 | if r.Method == "GET" { 331 | ch := make(chan []document) 332 | dGetAllCh <- dAllMsg{Ch: ch} 333 | docs := <-ch 334 | close(ch) 335 | 336 | if serializedPayload, err := json.Marshal(docs); err == nil { 337 | w.Write(serializedPayload) 338 | } else { 339 | common.Warn("Unable to serialize all docs: " + err.Error()) 340 | w.WriteHeader(http.StatusInternalServerError) 341 | w.Write([]byte(`{"code": 500, "msg": "Error occurred while trying to retrieve documents."}`)) 342 | } 343 | return 344 | } else if r.Method != "POST" { 345 | w.WriteHeader(http.StatusMethodNotAllowed) 346 | w.Write([]byte(`{"code": 405, "msg": "Method Not Allowed."}`)) 347 | return 348 | } 349 | 350 | decoder := json.NewDecoder(r.Body) 351 | defer r.Body.Close() 352 | 353 | var newDoc payload 354 | decoder.Decode(&newDoc) 355 | pProcessCh <- newDoc 356 | 357 | w.Write([]byte(`{"code": 200, "msg": "Request is being processed."}`)) 358 | } 359 | -------------------------------------------------------------------------------- /Chapter08/goophr/concierge/api/feeder_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | ) 9 | 10 | func TestGetTitleHash(t *testing.T) { 11 | 12 | h1 := getTitleHash("A-Title") 13 | h2 := getTitleHash("Diff Title") 14 | hDup := getTitleHash("A-Title") 15 | 16 | for _, tc := range []struct { 17 | name string 18 | hashes []string 19 | expected bool 20 | }{ 21 | {"Different Titles", []string{h1, h2}, false}, 22 | {"Duplicate Titles", []string{h1, hDup}, false}, 23 | {"Same hashes", []string{h2, h2}, true}, 24 | } { 25 | t.Run(tc.name, func(t *testing.T) { 26 | actual := tc.hashes[0] == tc.hashes[1] 27 | if actual != tc.expected { 28 | t.Error(actual, tc.expected, tc.hashes) 29 | } 30 | }) 31 | } 32 | } 33 | 34 | func TestGetFile(t *testing.T) { 35 | doc := "Server returned text!" 36 | testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 37 | w.Write([]byte(doc)) 38 | })) 39 | defer testServer.Close() 40 | 41 | rDoc, err := getFile(testServer.URL) 42 | if err != nil { 43 | t.Error("Error while retrieving document", err) 44 | } 45 | if doc != rDoc { 46 | t.Error(doc, "!=", rDoc) 47 | } 48 | } 49 | 50 | func TestIndexProcessor(t *testing.T) { 51 | ch1 := make(chan document, 1) 52 | ch2 := make(chan lMeta, 1) 53 | ch3 := make(chan token, 3) 54 | done := make(chan bool) 55 | 56 | go indexProcessor(ch1, ch2, ch3, done) 57 | 58 | ch1 <- document{ 59 | DocID: "a-hash", 60 | Title: "a-title", 61 | Doc: "Golang Programming rocks!", 62 | } 63 | 64 | for i, tc := range []string{ 65 | "golang", "programming", "rocks", 66 | } { 67 | t.Run(fmt.Sprintf("Testing if '%s' is returned. at index: %d", tc, i+1), func(t *testing.T) { 68 | tok := <-ch3 69 | if tok.Token != tc { 70 | t.Error(tok.Token, "!=", tc) 71 | } 72 | if tok.Index != i+1 { 73 | t.Error(tok.Index, "!=", i+1) 74 | } 75 | }) 76 | } 77 | close(done) 78 | 79 | } 80 | -------------------------------------------------------------------------------- /Chapter08/goophr/concierge/api/query.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "os" 12 | "sort" 13 | 14 | "github.com/last-ent/distributed-go/chapter8/goophr/concierge/common" 15 | ) 16 | 17 | var librarianEndpoints = map[string]string{} 18 | 19 | func init() { 20 | librarianEndpoints["a-m"] = os.Getenv("LIB_A_M") 21 | librarianEndpoints["n-z"] = os.Getenv("LIB_N_Z") 22 | librarianEndpoints["*"] = os.Getenv("LIB_OTHERS") 23 | } 24 | 25 | type docs struct { 26 | DocID string `json:"doc_id"` 27 | Score int `json:"doc_score"` 28 | } 29 | 30 | type queryResult struct { 31 | Count int `json:"count"` 32 | Data []docs `json:"data"` 33 | } 34 | 35 | func queryLibrarian(endpoint string, stBytes io.Reader, ch chan<- queryResult) { 36 | resp, err := http.Post( 37 | endpoint+"/query", 38 | "application/json", 39 | stBytes, 40 | ) 41 | if err != nil { 42 | common.Warn(fmt.Sprintf("%s -> %+v", endpoint, err)) 43 | ch <- queryResult{} 44 | return 45 | } 46 | body, _ := ioutil.ReadAll(resp.Body) 47 | defer resp.Body.Close() 48 | 49 | var qr queryResult 50 | json.Unmarshal(body, &qr) 51 | log.Println(fmt.Sprintf("%s -> %#v", endpoint, qr)) 52 | ch <- qr 53 | } 54 | 55 | func getResultsMap(ch <-chan queryResult) map[string]int { 56 | results := []docs{} 57 | for range librarianEndpoints { 58 | if result := <-ch; result.Count > 0 { 59 | results = append(results, result.Data...) 60 | } 61 | } 62 | 63 | resultsMap := map[string]int{} 64 | for _, doc := range results { 65 | docID := doc.DocID 66 | score := doc.Score 67 | if _, exists := resultsMap[docID]; !exists { 68 | resultsMap[docID] = 0 69 | } 70 | resultsMap[docID] = resultsMap[docID] + score 71 | } 72 | 73 | return resultsMap 74 | } 75 | 76 | func QueryHandler(w http.ResponseWriter, r *http.Request) { 77 | if r.Method != "POST" { 78 | w.WriteHeader(http.StatusMethodNotAllowed) 79 | w.Write([]byte(`{"code": 405, "msg": "Method Not Allowed."}`)) 80 | return 81 | } 82 | 83 | decoder := json.NewDecoder(r.Body) 84 | defer r.Body.Close() 85 | 86 | var searchTerms []string 87 | if err := decoder.Decode(&searchTerms); err != nil { 88 | common.Warn("Unable to parse request." + err.Error()) 89 | 90 | w.WriteHeader(http.StatusBadRequest) 91 | w.Write([]byte(`{"code": 400, "msg": "Unable to parse payload."}`)) 92 | return 93 | } 94 | 95 | st, err := json.Marshal(searchTerms) 96 | if err != nil { 97 | panic(err) 98 | } 99 | stBytes := bytes.NewBuffer(st) 100 | 101 | resultsCh := make(chan queryResult) 102 | 103 | for _, le := range librarianEndpoints { 104 | func(endpoint string) { 105 | go queryLibrarian(endpoint, stBytes, resultsCh) 106 | }(le) 107 | } 108 | 109 | resultsMap := getResultsMap(resultsCh) 110 | close(resultsCh) 111 | 112 | sortedResults := sortResults(resultsMap) 113 | 114 | payload, _ := json.Marshal(sortedResults) 115 | w.Header().Add("Content-Type", "application/json") 116 | w.Write(payload) 117 | 118 | fmt.Printf("%#v\n", sortedResults)) 119 | } 120 | 121 | func sortResults(rm map[string]int) []document { 122 | scoreMap := map[int][]document{} 123 | ch := make(chan document) 124 | 125 | for docID, score := range rm { 126 | if _, exists := scoreMap[score]; !exists { 127 | scoreMap[score] = []document{} 128 | } 129 | 130 | dGetCh <- dMsg{ 131 | DocID: docID, 132 | Ch: ch, 133 | } 134 | doc := <-ch 135 | 136 | scoreMap[score] = append(scoreMap[score], doc) 137 | } 138 | 139 | close(ch) 140 | 141 | scores := []int{} 142 | for score := range scoreMap { 143 | scores = append(scores, score) 144 | } 145 | sort.Sort(sort.Reverse(sort.IntSlice(scores))) 146 | 147 | sortedResults := []document{} 148 | for _, score := range scores { 149 | resDocs := scoreMap[score] 150 | sortedResults = append(sortedResults, resDocs...) 151 | } 152 | 153 | return sortedResults 154 | } 155 | -------------------------------------------------------------------------------- /Chapter08/goophr/concierge/common/helpers.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "regexp" 7 | "strings" 8 | ) 9 | 10 | func Log(msg string) { 11 | log.Println("INFO - ", msg) 12 | } 13 | 14 | func Warn(msg string) { 15 | log.Println("---------------------------") 16 | log.Println(fmt.Sprintf("WARN: %s", msg)) 17 | log.Println("---------------------------") 18 | } 19 | 20 | var punctuations = regexp.MustCompile(`^\p{P}+|\p{P}+$`) 21 | 22 | var stopWords = []string{ 23 | "a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "aren't", "as", "at", 24 | "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "can't", "cannot", "could", 25 | "couldn't", "did", "didn't", "do", "does", "doesn't", "doing", "don't", "down", "during", "each", "few", "for", 26 | "from", "further", "had", "hadn't", "has", "hasn't", "have", "haven't", "having", "he", "he'd", "he'll", "he's", 27 | "her", "here", "here's", "hers", "herself", "him", "himself", "his", "how", "how's", "i", "i'd", "i'll", "i'm", 28 | "i've", "if", "in", "into", "is", "isn't", "it", "it's", "its", "itself", "let's", "me", "more", "most", "mustn't", 29 | "my", "myself", "no", "nor", "not", "of", "off", "on", "once", "only", "or", "other", "ought", "our", "ours", 30 | "ourselves", "out", "over", "own", "same", "shan't", "she", "she'd", "she'll", "she's", "should", "shouldn't", 31 | "so", "some", "such", "than", "that", "that's", "the", "their", "theirs", "them", "themselves", "then", "there", 32 | "there's", "these", "they", "they'd", "they'll", "they're", "they've", "this", "those", "through", "to", "too", 33 | "under", "until", "up", "very", "was", "wasn't", "we", "we'd", "we'll", "we're", "we've", "were", "weren't", "what", 34 | "what's", "when", "when's", "where", "where's", "which", "while", "who", "who's", "whom", "why", "why's", "with", 35 | "won't", "would", "wouldn't", "you", "you'd", "you'll", "you're", "you've", "your", "yours", "yourself", "yourselves"} 36 | 37 | // SimplifyToken is responsible to normalizing a string token and 38 | // also checks whether the token should be indexed or not. 39 | func SimplifyToken(token string) (string, bool) { 40 | simpleToken := strings.ToLower(punctuations.ReplaceAllString(token, "")) 41 | 42 | for _, stopWord := range stopWords { 43 | if stopWord == simpleToken { 44 | return "", false 45 | } 46 | } 47 | 48 | return simpleToken, true 49 | } 50 | -------------------------------------------------------------------------------- /Chapter08/goophr/concierge/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | 8 | "github.com/last-ent/distributed-go/chapter8/goophr/concierge/api" 9 | "github.com/last-ent/distributed-go/chapter8/goophr/concierge/common" 10 | ) 11 | 12 | func main() { 13 | common.Log("Adding API handlers...") 14 | http.HandleFunc("/api/feeder", api.FeedHandler) 15 | http.HandleFunc("/api/query", api.QueryHandler) 16 | 17 | common.Log("Starting feeder...") 18 | api.StartFeederSystem() 19 | 20 | port := fmt.Sprintf(":%s", os.Getenv("API_PORT")) 21 | common.Log(fmt.Sprintf("Starting Goophr Concierge server on port %s...", port)) 22 | http.ListenAndServe(port, nil) 23 | } 24 | -------------------------------------------------------------------------------- /Chapter08/goophr/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | a_m_librarian: 5 | build: librarian/. 6 | image: goophr_librarian 7 | environment: 8 | - API_PORT=${A_M_PORT} 9 | ports: 10 | - ${A_M_PORT}:${A_M_PORT} 11 | n_z_librarian: 12 | image: goophr_librarian 13 | environment: 14 | - API_PORT=${N_Z_PORT} 15 | ports: 16 | - ${N_Z_PORT}:${N_Z_PORT} 17 | others_librarian: 18 | image: goophr_librarian 19 | environment: 20 | - API_PORT=${OTHERS_PORT} 21 | ports: 22 | - ${OTHERS_PORT}:${OTHERS_PORT} 23 | concierge: 24 | build: concierge/. 25 | environment: 26 | - API_PORT=${CONCIERGE_PORT} 27 | - LIB_A_M=http://a_m_librarian:${A_M_PORT}/api 28 | - LIB_N_Z=http://n_z_librarian:${N_Z_PORT}/api 29 | - LIB_OTHERS=http://others_librarian:${OTHERS_PORT}/api 30 | ports: 31 | - ${CONCIERGE_PORT}:${CONCIERGE_PORT} 32 | links: 33 | - a_m_librarian 34 | - n_z_librarian 35 | - others_librarian 36 | - file_server 37 | file_server: 38 | build: simple-server/. 39 | ports: 40 | - ${SERVER_PORT}:${SERVER_PORT} 41 | 42 | -------------------------------------------------------------------------------- /Chapter08/goophr/librarian/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.10 2 | 3 | 4 | ADD . /go/src/github.com/last-ent/distributed-go/chapter8/goophr/librarian 5 | 6 | RUN go install github.com/last-ent/distributed-go/chapter8/goophr/librarian 7 | 8 | ENTRYPOINT /go/bin/librarian 9 | 10 | -------------------------------------------------------------------------------- /Chapter08/goophr/librarian/api/index.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | ) 10 | 11 | type tPayload struct { 12 | Token string `json:"token"` 13 | Title string `json:"title"` 14 | DocID string `json:"doc_id"` 15 | LIndex int `json:"line_index"` 16 | Index int `json:"token_index"` 17 | } 18 | 19 | type tIndex struct { 20 | Index int 21 | LIndex int 22 | } 23 | 24 | func (ti *tIndex) String() string { 25 | return fmt.Sprintf("i: %d, li: %d", ti.Index, ti.LIndex) 26 | } 27 | 28 | type tIndices []tIndex 29 | 30 | // document - key in Indices represent Line Index. 31 | type document struct { 32 | Count int 33 | DocID string 34 | Title string 35 | Indices map[int]tIndices 36 | } 37 | 38 | func (d *document) String() string { 39 | str := fmt.Sprintf("%s (%s): %d\n", d.Title, d.DocID, d.Count) 40 | var buffer bytes.Buffer 41 | 42 | for lin, tis := range d.Indices { 43 | var lBuffer bytes.Buffer 44 | for _, ti := range tis { 45 | lBuffer.WriteString(fmt.Sprintf("%s ", ti.String())) 46 | } 47 | buffer.WriteString(fmt.Sprintf("@%d -> %s\n", lin, lBuffer.String())) 48 | } 49 | return str + buffer.String() 50 | } 51 | 52 | // documentCatalog - key represents DocID. 53 | type documentCatalog map[string]*document 54 | 55 | func (dc *documentCatalog) String() string { 56 | return fmt.Sprintf("%#v", dc) 57 | } 58 | 59 | // tCatalog - key in map represents Token. 60 | type tCatalog map[string]documentCatalog 61 | 62 | func (tc *tCatalog) String() string { 63 | return fmt.Sprintf("%#v", tc) 64 | } 65 | 66 | type tcCallback struct { 67 | Token string 68 | Ch chan tcMsg 69 | } 70 | 71 | type tcMsg struct { 72 | Token string 73 | DC documentCatalog 74 | } 75 | 76 | // pProcessCh is used to process /index's payload and start process to add the token to catalog (tCatalog). 77 | var pProcessCh chan tPayload 78 | 79 | // tcGet is used to retrieve a token's catalog (documentCatalog). 80 | var tcGet chan tcCallback 81 | 82 | func StartIndexSystem() { 83 | pProcessCh = make(chan tPayload, 100) 84 | tcGet = make(chan tcCallback, 20) 85 | go tIndexer(pProcessCh, tcGet) 86 | } 87 | 88 | // tIndexer maintains a catalog of all tokens along with where they occur within documents. 89 | func tIndexer(ch chan tPayload, callback chan tcCallback) { 90 | store := tCatalog{} 91 | for { 92 | select { 93 | case msg := <-callback: 94 | dc := store[msg.Token] 95 | msg.Ch <- tcMsg{ 96 | DC: dc, 97 | Token: msg.Token, 98 | } 99 | 100 | case pd := <-ch: 101 | dc, exists := store[pd.Token] 102 | if !exists { 103 | dc = documentCatalog{} 104 | store[pd.Token] = dc 105 | } 106 | 107 | doc, exists := dc[pd.DocID] 108 | if !exists { 109 | doc = &document{ 110 | DocID: pd.DocID, 111 | Title: pd.Title, 112 | Indices: map[int]tIndices{}, 113 | } 114 | dc[pd.DocID] = doc 115 | } 116 | 117 | tin := tIndex{ 118 | Index: pd.Index, 119 | LIndex: pd.LIndex, 120 | } 121 | doc.Indices[tin.LIndex] = append(doc.Indices[tin.LIndex], tin) 122 | doc.Count++ 123 | } 124 | } 125 | } 126 | 127 | func IndexHandler(w http.ResponseWriter, r *http.Request) { 128 | if r.Method != "POST" { 129 | w.WriteHeader(http.StatusMethodNotAllowed) 130 | w.Write([]byte(`{"code": 405, "msg": "Method Not Allowed."}`)) 131 | return 132 | } 133 | 134 | decoder := json.NewDecoder(r.Body) 135 | defer r.Body.Close() 136 | 137 | var tp tPayload 138 | decoder.Decode(&tp) 139 | log.Println("Token received", fmt.Sprintf("%#v", tp)) 140 | 141 | pProcessCh <- tp 142 | 143 | w.Write([]byte(`{"code": 200, "msg": "Tokens are being added to index."}`)) 144 | } 145 | -------------------------------------------------------------------------------- /Chapter08/goophr/librarian/api/query.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "sort" 7 | 8 | "github.com/last-ent/distributed-go/chapter8/goophr/librarian/common" 9 | ) 10 | 11 | type docResult struct { 12 | DocID string `json:"doc_id"` 13 | Score int `json:"doc_score"` 14 | Indices tIndices `json:"token_indices"` 15 | } 16 | 17 | type result struct { 18 | Count int `json:"count"` 19 | Data []docResult `json:"data"` 20 | } 21 | 22 | // getResults returns unsorted search results & a map of documents containing tokens. 23 | func getResults(out chan tcMsg, count int) tCatalog { 24 | tc := tCatalog{} 25 | for i := 0; i < count; i++ { 26 | dc := <-out 27 | tc[dc.Token] = dc.DC 28 | } 29 | close(out) 30 | 31 | return tc 32 | } 33 | 34 | func getFScores(docIDScore map[string]int) (map[int][]string, []int) { 35 | // fScore maps frequency score to set of documents. 36 | fScore := map[int][]string{} 37 | 38 | fSorted := []int{} 39 | 40 | for dID, score := range docIDScore { 41 | if _, exists := fScore[score]; !exists { 42 | fScore[score] = []string{} 43 | } 44 | fScore[score] = append(fScore[score], dID) 45 | fSorted = append(fSorted, score) 46 | } 47 | 48 | sort.Sort(sort.Reverse(sort.IntSlice(fSorted))) 49 | 50 | return fScore, fSorted 51 | } 52 | 53 | func getDocMaps(tc tCatalog) (map[string]int, map[string]tIndices) { 54 | // docIDScore maps DocIDs to occurences of all tokens. 55 | // key: DocID. 56 | // val: Sum of all occurences of tokens so far. 57 | docIDScore := map[string]int{} 58 | docIndices := map[string]tIndices{} 59 | 60 | // for each token's catalog 61 | for _, dc := range tc { 62 | // for each document registered under the token 63 | for dID, doc := range dc { 64 | if _, exists := docIDScore[dID]; !exists { 65 | docIDScore[dID] = 0 66 | } 67 | // add to docID score 68 | var tokIndices tIndices 69 | for _, tList := range doc.Indices { 70 | tokIndices = append(tokIndices, tList...) 71 | } 72 | docIDScore[dID] += doc.Count 73 | 74 | dti, exists := docIndices[dID] 75 | if !exists { 76 | dti := tIndices{} 77 | docIndices[dID] = dti 78 | } 79 | 80 | docIndices[dID] = append(dti, tokIndices...) 81 | } 82 | } 83 | 84 | return docIDScore, docIndices 85 | } 86 | 87 | func sortResults(tc tCatalog) []docResult { 88 | docIDScore, docIndices := getDocMaps(tc) 89 | fScore, fSorted := getFScores(docIDScore) 90 | 91 | results := []docResult{} 92 | addedDocs := map[string]bool{} 93 | 94 | for _, score := range fSorted { 95 | for _, docID := range fScore[score] { 96 | if _, exists := addedDocs[docID]; exists { 97 | continue 98 | } 99 | results = append(results, docResult{ 100 | DocID: docID, 101 | Score: score, 102 | Indices: docIndices[docID], 103 | }) 104 | addedDocs[docID] = false 105 | } 106 | } 107 | return results 108 | } 109 | 110 | // getSearchResults returns a list of documents. 111 | // They are listed in descending order of occurences. 112 | func getSearchResults(sts []string) []docResult { 113 | 114 | callback := make(chan tcMsg) 115 | 116 | for _, st := range sts { 117 | go func(term string) { 118 | tcGet <- tcCallback{ 119 | Token: term, 120 | Ch: callback, 121 | } 122 | }(st) 123 | } 124 | 125 | cts := getResults(callback, len(sts)) 126 | results := sortResults(cts) 127 | return results 128 | } 129 | 130 | func QueryHandler(w http.ResponseWriter, r *http.Request) { 131 | if r.Method != "POST" { 132 | w.WriteHeader(http.StatusMethodNotAllowed) 133 | w.Write([]byte(`{"code": 405, "msg": "Method Not Allowed."}`)) 134 | return 135 | } 136 | 137 | decoder := json.NewDecoder(r.Body) 138 | defer r.Body.Close() 139 | 140 | var searchTerms []string 141 | decoder.Decode(&searchTerms) 142 | 143 | results := getSearchResults(searchTerms) 144 | 145 | payload := result{ 146 | Count: len(results), 147 | Data: results, 148 | } 149 | 150 | if serializedPayload, err := json.Marshal(payload); err == nil { 151 | w.Header().Add("Content-Type", "application/json") 152 | w.Write(serializedPayload) 153 | } else { 154 | common.Warn("Unable to serialize all docs: " + err.Error()) 155 | w.WriteHeader(http.StatusInternalServerError) 156 | w.Write([]byte(`{"code": 500, "msg": "Error occurred while trying to retrieve documents."}`)) 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /Chapter08/goophr/librarian/common/helpers.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | ) 7 | 8 | func Log(msg string) { 9 | log.Println("INFO - ", msg) 10 | } 11 | 12 | func Warn(msg string) { 13 | log.Println("---------------------------") 14 | log.Println(fmt.Sprintf("WARN: %s", msg)) 15 | log.Println("---------------------------") 16 | } 17 | -------------------------------------------------------------------------------- /Chapter08/goophr/librarian/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | 8 | "github.com/last-ent/distributed-go/chapter8/goophr/librarian/api" 9 | "github.com/last-ent/distributed-go/chapter8/goophr/librarian/common" 10 | ) 11 | 12 | func main() { 13 | common.Log("Adding API handlers...") 14 | http.HandleFunc("/api/index", api.IndexHandler) 15 | http.HandleFunc("/api/query", api.QueryHandler) 16 | 17 | common.Log("Starting index...") 18 | api.StartIndexSystem() 19 | 20 | port := fmt.Sprintf(":%s", os.Getenv("API_PORT")) 21 | common.Log(fmt.Sprintf("Starting Goophr Librarian server on port %s...", port)) 22 | http.ListenAndServe(port, nil) 23 | } 24 | -------------------------------------------------------------------------------- /Chapter08/goophr/simple-server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.10 2 | 3 | 4 | ADD . /go/src/littlefs 5 | 6 | WORKDIR /go/src/littlefs 7 | 8 | RUN go install littlefs 9 | 10 | ENTRYPOINT /go/bin/littlefs 11 | 12 | -------------------------------------------------------------------------------- /Chapter08/goophr/simple-server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | ) 7 | 8 | func reqHandler(w http.ResponseWriter, r *http.Request) { 9 | books := map[string]string{ 10 | "book1": `apple apple cat zebra`, 11 | "book2": `banana cake zebra`, 12 | "book3": `apple cake cake whale`, 13 | } 14 | 15 | bookID := r.URL.Path[1:] 16 | book, _ := books[bookID] 17 | w.Write([]byte(book)) 18 | } 19 | 20 | func main() { 21 | 22 | log.Println("Starting File Server on Port :9876...") 23 | http.HandleFunc("/", reqHandler) 24 | http.ListenAndServe(":9876", nil) 25 | } 26 | -------------------------------------------------------------------------------- /Chapter08/secure/secure.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "strings" 8 | ) 9 | 10 | var authTokens = map[string]string{ 11 | "AUTH-TOKEN-1": "User 1", 12 | "AUTH-TOKEN-2": "User 2", 13 | } 14 | 15 | // getAuthorizedUser tries to retrieve user for the given token. 16 | func getAuthorizedUser(token string) (string, error) { 17 | var err error 18 | 19 | user, valid := authTokens[token] 20 | if !valid { 21 | err = fmt.Errorf("Auth token '%s' does not exist.", token) 22 | } 23 | 24 | return user, err 25 | } 26 | 27 | // isAuthorized checks request to ensure that it has Authorization header 28 | // with defined value: "Bearer AUTH-TOKEN" 29 | func isAuthorized(r *http.Request) bool { 30 | rawToken := r.Header["Authorization"] 31 | if len(rawToken) != 1 { 32 | return false 33 | } 34 | 35 | authToken := strings.Split(rawToken[0], " ") 36 | if !(len(authToken) == 2 && authToken[0] == "Bearer") { 37 | return false 38 | } 39 | 40 | user, err := getAuthorizedUser(authToken[1]) 41 | if err != nil { 42 | log.Printf("Error: %s", err) 43 | return false 44 | } 45 | 46 | log.Printf("Successful request made by '%s'", user) 47 | return true 48 | } 49 | 50 | var success = []byte("Received authorized request.") 51 | var failure = []byte("Received unauthorized request.") 52 | 53 | func requestHandler(w http.ResponseWriter, r *http.Request) { 54 | if isAuthorized(r) { 55 | w.Write(success) 56 | } else { 57 | w.WriteHeader(http.StatusUnauthorized) 58 | w.Write(failure) 59 | } 60 | } 61 | 62 | func main() { 63 | http.HandleFunc("/", requestHandler) 64 | fmt.Println("Starting server @ http://localhost:8080") 65 | http.ListenAndServe(":8080", nil) 66 | } 67 | -------------------------------------------------------------------------------- /Chapter08/secure/secure_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | ) 8 | 9 | func TestIsAuthorizedSuccess(t *testing.T) { 10 | req, err := http.NewRequest("GET", "http://example.com", nil) 11 | if err != nil { 12 | t.Error("Unable to create request") 13 | } 14 | 15 | req.Header["Authorization"] = []string{"Bearer AUTH-TOKEN-1"} 16 | 17 | if isAuthorized(req) { 18 | t.Log("Request with correct Auth token was correctly processed.") 19 | } else { 20 | t.Error("Request with correct Auth token failed.") 21 | } 22 | } 23 | 24 | func TestIsAuthorizedFailTokenType(t *testing.T) { 25 | req, err := http.NewRequest("GET", "http://example.com", nil) 26 | if err != nil { 27 | t.Error("Unable to create request") 28 | } 29 | 30 | req.Header["Authorization"] = []string{"Token AUTH-TOKEN-1"} 31 | 32 | if isAuthorized(req) { 33 | t.Error("Request with incorrect Auth token type was successfully processed.") 34 | } else { 35 | t.Log("Request with incorrect Auth token type failed as expected.") 36 | } 37 | } 38 | 39 | func TestIsAuthorizedFailToken(t *testing.T) { 40 | req, err := http.NewRequest("GET", "http://example.com", nil) 41 | if err != nil { 42 | t.Error("Unable to create request") 43 | } 44 | 45 | req.Header["Authorization"] = []string{"Token WRONG-AUTH-TOKEN"} 46 | 47 | if isAuthorized(req) { 48 | t.Error("Request with incorrect Auth token was successfully processed.") 49 | } else { 50 | t.Log("Request with incorrect Auth token failed as expected.") 51 | } 52 | } 53 | 54 | func TestRequestHandlerFailToken(t *testing.T) { 55 | req, err := http.NewRequest("GET", "http://example.com", nil) 56 | if err != nil { 57 | t.Error("Unable to create request") 58 | } 59 | 60 | req.Header["Authorization"] = []string{"Token WRONG-AUTH-TOKEN"} 61 | 62 | // http.ResponseWriter it is an interface hence we use 63 | // httptest.NewRecorder which implements the interface http.ResponseWriter 64 | rr := httptest.NewRecorder() 65 | requestHandler(rr, req) 66 | 67 | if rr.Code == 401 { 68 | t.Log("Request with incorrect Auth token failed as expected.") 69 | } else { 70 | t.Error("Request with incorrect Auth token was successfully processed.") 71 | } 72 | } 73 | 74 | func TestGetAuthorizedUser(t *testing.T) { 75 | if user, err := getAuthorizedUser("AUTH-TOKEN-2"); err != nil { 76 | t.Errorf("Couldn't find User 2. Error: %s", err) 77 | } else if user != "User 2" { 78 | t.Errorf("Found incorrect user: %s", user) 79 | } else { 80 | t.Log("Found User 2.") 81 | } 82 | } 83 | 84 | func TestGetAuthorizedUserFail(t *testing.T) { 85 | if user, err := getAuthorizedUser("WRONG-AUTH-TOKEN"); err == nil { 86 | t.Errorf("Found user for invalid token!. User: %s", user) 87 | } else if err.Error() != "Auth token 'WRONG-AUTH-TOKEN' does not exist." { 88 | t.Errorf("Error message does not match.") 89 | } else { 90 | t.Log("Got expected error message for invalid auth token") 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # Distributed Computing with Go 5 | This is the code repository for [Distributed Computing with Go](https://www.packtpub.com/application-development/distributed-computing-go?utm_source=github&utm_medium=repository&utm_campaign=9781787125384), published by [Packt](https://www.packtpub.com/?utm_source=github). It contains all the supporting project files necessary to work through the book from start to finish. 6 | ## About the Book 7 | Distributed Computing with Go gives developers with a good idea how basic Go development works the tools to fulfill the true potential of Golang development in a world of concurrent web and cloud applications. Nikhil starts out by setting up a professional Go development environment. Then you’ll learn the basic concepts and practices of Golang concurrent and parallel development. 8 | 9 | You’ll find out in the new few chapters how to balance resources and data with REST and standard web approaches while keeping concurrency in mind. Most Go applications these days will run in a data center or on the cloud, which is a condition upon which the next chapter depends. There, you’ll expand your skills considerably by writing a distributed document indexing system during the next two chapters. This system has to balance a large corpus of documents with considerable analytical demands. 10 | 11 | Another use case is the way in which a web application written in Go can be consciously redesigned to take distributed features into account. The chapter is rather interesting for Go developers who have to migrate existing Go applications to computationally and memory-intensive environments. The final chapter relates to the rather onerous task of testing parallel and distributed applications, something that is not usually taught in standard computer science curricula. 12 | 13 | ## Instructions and Navigation 14 | All of the code is organized into folders. Each folder starts with a number followed by the application name. For example, Chapter02. 15 | 16 | 17 | 18 | The code will look like the following: 19 | ``` 20 | package main 21 | import ( 22 | "fmt" 23 | "os" 24 | ) 25 | func main() { 26 | fmt.Println(os.Getenv("NAME") + " is your uncle.") 27 | } 28 | ``` 29 | 30 | The material in the book is designed to enable a hands-on approach. Throughout the book, a conscious effort has been made to provide all the relevant information to the reader beforehand so that, if the reader chooses, they can try to solve the problem on their own and then refer to the solution provided in the book. The code in the book does not have any Go dependencies beyond the standard library. This is done in order to ensure that the code examples provided in the book never change, and this also allows us to explore the standard library. 31 | The source code in the book should be placed at $GOPATH/src/distributedgo. The source code for examples given will be located inside the $GOPATH/src/distributed-go/chapterX folder, where X stands for the chapter number. 32 | Download and install Go from https://golang.org/ and Docker from https://www.docker.com/community-edition website 33 | 34 | ## Related Products 35 | * [Isomorphic Go](https://www.packtpub.com/web-development/isomorphic-go?utm_source=github&utm_medium=repository&utm_content=9781788394185) 36 | 37 | * [Go Systems Programming](https://www.packtpub.com/networking-and-servers/go-systems-programming?utm_source=github&utm_medium=repository&utm_content=9781787125643) 38 | 39 | * [Security with Go](https://www.packtpub.com/networking-and-servers/security-go?utm_source=github&utm_medium=repository&utm_campaign=9781788627917) 40 | ### Download a free PDF 41 | 42 | If you have already purchased a print or Kindle version of this book, you can get a DRM-free PDF version at no cost.
Simply click on the link to claim your free PDF. 43 |

https://packt.link/free-ebook/9781787125384

--------------------------------------------------------------------------------