├── .gitattributes ├── .gitignore ├── README.md ├── data ├── itcont.txt.7z ├── itcont_sample_40.txt └── itcont_sample_4000.txt ├── go.mod ├── go.sum └── main_test.go /.gitattributes: -------------------------------------------------------------------------------- 1 | data/itcont.txt.7z filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | itcont.txt 2 | __debug_bin 3 | .DS_Store 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Processing Large Files in Go (Golang) 2 | Comparing sequential and concurrent processing of files in Go. 3 | 4 | ## Setup 5 | Before running this program, ensure you unzip the itcont.txt.zip file to the data folder. 6 | ```bash 7 | # install p7zip as needed 8 | brew install p7zip 9 | # unzip to folder using p7zip 10 | 7z e data/itcont.txt.7z -odata/ 11 | ``` 12 | 13 | ## Test 14 | ```bash 15 | go test -timeout 120s -run ^Test$ github.com/snassr/blog-0010-processinglargefilesingo 16 | ``` 17 | 18 | ## Benchmark 19 | ```bash 20 | go test -benchmem -run=^$ -bench ^Benchmark$ github.com/snassr/blog-0010-processinglargefilesingo 21 | ``` 22 | -------------------------------------------------------------------------------- /data/itcont.txt.7z: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:a60b0bffc9258a4419574817b51f1b863289ee94e61532e589f4ff8349387296 3 | size 397632167 4 | -------------------------------------------------------------------------------- /data/itcont_sample_40.txt: -------------------------------------------------------------------------------- 1 | C00580100|A|YE|P2020|201903139145682053|15|IND|BALTHASAR, SUSAN|INDIAN SHORES|FL|33785|RETIRED|RETIRED|01072017|100||SA17A.8965|1319152|||4031420191645043402 2 | C00580100|A|YE|P2020|201903139145683412|15|IND|GAZZIER, JAY|FRIENDSWOOD|TX|77546|MERRILL LYNCH|FINANCIAL ADVISOR|01072017|100||SA17A.13989|1319152|||4031420191645048289 3 | C00580100|A|YE|P2020|201903139145684137|15|IND|KENNEY, PAUL|ACTON|ME|04001|P&E SUPPLY|SELF-EMPLOYED|01072018|100||SA17A.16626|1319152|||4031420191645052637 4 | C00580100|A|YE|P2020|201903139145685972|15|IND|STUBBLEFIELD, LANE|SIGNAL HILL|CA|90755|RETIRED|RETIRED|01082018|200||SA17A.23496|1319152|||4031420191645060122 5 | C00580100|A|YE|P2020|201903139145684455|15|IND|LOPEZ, CAROLYN|SAN ANTONIO|TX|78240|RETIRED|RETIRED|01082018|35||SA17A.17813|1319152|||4031420191645054547 6 | C00580100|A|YE|P2020|201903139145686118|15|IND|TOMAT, PEGGY|RIVERTON|WY|82501|RETIRED|RETIRED|02082018|-35||SA17A.27035|1319152|||4031420191645060559 7 | C00580100|A|YE|P2020|201903139145685701|15|IND|SERAFINE, LUANN|VALLEY SPRINGS|CA|95252|SUTTER HEALTH|RN|02082018|100||SA17A.22473|1319152|||4031420191645059309 8 | C00580100|A|YE|P2020|201903139145686272|15|IND|WALKER, LANA|ATLANTA|GA|30350|GENUINE PARTS|IT BUSINESS ANALYST|11092018|50||SA17A.24591|1319152|||4031420191645061020 9 | C00580100|A|YE|P2020|201903139145684723|15|IND|MELOT, TERRY|PEORIA|AZ|85345|RETIRED|RETIRED|11092018|250||SA17A.18792|1319152|||4031420191645056153 10 | C00580100|A|YE|P2020|201903139145683562|15|IND|GRIEVE, GILBERT|RENO|NV|89502|CONCOURS|CONSULANT|11072018|250||SA17A.14542|1319152|||4031420191645049187 11 | C00580100|A|YE|P2020|201903139145686441|15|IND|WINGFIELD, DEAN|SEATTLE|WA|98103|RETIRED|RETIRED|11092018|100||SA17A.25259|1319152|||4031420191645061528 12 | C00580100|A|YE|P2020|201903139145682814|15|IND|DACORTE, TRINIDAD P|SAN ANTONIO|TX|78230|CAREGIVER|CAREGIVER|11062018|35||SA17A.11897|1319152|||4031420191645046112 13 | C00580100|A|YE|P2020|201903139145682574|15|IND|CHEESEMAN, GERI|ELMER|NJ|08318|C & H DISPOSAL|OWNER|03062018|250||SA17A.10951|1319152|||4031420191645045391 14 | C00580100|A|YE|P2020|201903139145683045|15|IND|DUBRAWSKY, CHAGAI|HOUSTON|TX|77066|RETIRED|RETIRED|03062018|50||SA17A.12683|1319152|||4031420191645046805 15 | C00580100|A|YE|P2020|201903139145682070|15|IND|BARKEN, GARY|LAGUNA BEACH|CA|92651|BHC INDISTRIES OF TEXAS|EXECUTIVE|11072018|100||SA17A.9026|1319152|||4031420191645043500 16 | C00580100|A|YE|P2020|201903139145682546|15|IND|CAULFIELD, JAMES|WHEATON|IL|60189|ALLEN/DAVIS L.L.C.|SALES ENGINEER|11072018|100||SA17A.10836|1319152|||4031420191645045306 17 | C00580100|A|YE|P2020|201903139145684337|15|IND|LARSEN, VICKIE|RAMONA|CA|92065|RETIRED|RETIRED|05082018|25||SA17A.17361|1319152|||4031420191645053839 18 | C00580100|A|YE|P2020|201903139145684338|15|IND|LARSEN, VICKIE|RAMONA|CA|92065|RETIRED|RETIRED|06082018|35||SA17A.17362|1319152|||4031420191645053841 19 | C00580100|A|YE|P2020|201903139145685380|15|IND|REARDON, JOHN|WAREHAM|MA|02571|RETIRED|RETIRED|11092018|50||SA17A.21259|1319152|||4031420191645058344 20 | C00580100|A|YE|P2020|201903139145683376|15|IND|GALLEGOS, LORENA|NICHOLASVILLE|KY|40356|SALAS INTERPRISES|BUSINESS OWNER|11072018|250||SA17A.13873|1319152|||4031420191645048071 21 | C00580100|A|YE|P2020|201903139145683860|15|IND|HORNBECK, LAURA|VAN ALSTYNE|TX|75495|RETIRED|RETIRED|01072018|250||SA17A.15675|1319152|||4031420191645050977 22 | C00580100|A|YE|P2020|201903139145683861|15|IND|HORNBECK, LAURA|VAN ALSTYNE|TX|75495|RETIRED|RETIRED|03072018|-250||SA17A.25811|1319152|X||4031420191645050979 23 | C00580100|A|YE|G2020|201903139145683861|15|IND|HORNBECK, LAURA|VAN ALSTYNE|TX|75495|RETIRED|RETIRED|11072018|250||SA17A.25812|1319152|X||4031420191645050981 24 | C00580100|A|YE|P2020|201903139145681902|15|IND|AKANA, LAURA|CANYON COUNTRY|CA|91387|MEDTRONIC|PRESIDENT|11072018|100||SA17A.8306|1319152|||4031420191645042872 25 | C00580100|A|YE|P2020|201903139145682641|15|IND|CLAUSEN, DARWIN|LA CRESCENT|MN|55947|RETIRED|RETIRED|11072018|50||SA17A.11208|1319152|||4031420191645045592 26 | C00580100|A|YE|P2020|201903139145683260|15|IND|FLEMIN, BARBARA|CARY|NC|27519|MSM EQUIPMENT|OFFICE MANAGER|11072018|250||SA17A.13496|1319152|||4031420191645047449 27 | C00580100|A|YE|P2020|201904139145682990|15|IND|DISMORE, JAMES|BENTONVILLE|AR|72712|RETIRED|RETIRED|01072018|10||SA17A.12501|1319152|||4031420191645046640 28 | C00580100|A|YE|P2020|201904139145682842|15|IND|DAVIS, CONSUELO|CAMARILLO|CA|93010|RETIRED|RETIRED|03062018|50||SA17A.12012|1319152|||4031420191645046195 29 | C00580100|A|YE|P2020|201904139145683315|15|IND|FRANK, MICHAELENE|MILWAUKEE|WI|53204|RETIRED|RETIRED|11092018|35||SA17A.13685|1319152|||4031420191645047707 30 | C00580100|A|YE|P2020|201904139145684104|15|IND|KEINATH, WARREN C|LAKE SAINT LOUIS|MO|63367|RETIRED|RETIRED|11072018|35||SA17A.16513|1319152|||4031420191645052439 31 | C00580100|A|YE|P2020|201904139145683316|15|IND|FRANK, MICHAELENE|MILWAUKEE|WI|53204|RETIRED|RETIRED|03072018|-35||SA17A.-2147478659|1319152|X||4031420191645047709 32 | C00580100|A|YE|G2020|201904139145683316|15|IND|FRANK, MICHAELENE|MILWAUKEE|WI|53204|RETIRED|RETIRED|03072018|35||SA17A.-2147478575|1319152|X||4031420191645047711 33 | C00580100|A|YE|P2020|201904139145686046|15|IND|TAYLOR, RAMSEY|SUPPLY|NC|28462|RETIRED|RETIRED|04072019|50||SA17A.23753|1319152|||4031420191645060343 34 | C00580100|A|YE|P2020|201904139145684378|15|IND|LEININGER, DANIEL|LAKE WALES|FL|33859|RETIRED|RETIRED|04072019|200||SA17A.17517|1319152|||4031420191645054083 35 | C00580100|A|YE|P2020|201904139145685886|15|IND|SPROULL, SHELLEY|BOERNE|TX|78006|DR. SARAH KINARD|DENTIST|05072019|100||SA17A.23187|1319152|||4031420191645059864 36 | C00580100|A|YE|P2020|201904139145682212|15|IND|BLAIR, ALAN|VIENNA|VA|22181|ALTAMIRA|ARCHITECT|05072019|150||SA17A.9572|1319152|||4031420191645044289 37 | C00580100|A|YE|P2020|201905139145684108|15|IND|KELCHNER, KEITH|ACWORTH|GA|30101|2HIREAHANDYMAN|CEO|06072020|100||SA17A.16522|1319152|||4031420191645052463 38 | C00580100|A|YE|P2020|201905139145681871|15|IND|ADAMS, DAVID M|CIRCLEVILLE|OH|43113|NOT EMPLOYED|NOT EMPLOYED|07072020|35||SA17A.8194|1319152|||4031420191645042731 39 | C00580100|A|YE|P2020|201905139145686307|15|IND|WARRINER, ALLEN|SLIDELL|LA|70459|SELF-EMPLOYED|MARINE CONTRACTOR|08072020|250||SA17A.24751|1319152|||4031420191645061125 40 | C00580100|A|YE|P2020|201905139145682301|15|IND|BRADSHAW, JERRY|HERRIMAN|UT|84096|RETIRED|RETIRED|08072020|35||SA17A.9907|1319152|||4031420191645044572 41 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/snassr/blog-0010-processinglargefilesingo 2 | 3 | go 1.19 4 | 5 | require github.com/stretchr/testify v1.8.1 6 | 7 | require ( 8 | github.com/davecgh/go-spew v1.1.1 // indirect 9 | github.com/pmezard/go-difflib v1.0.0 // indirect 10 | gopkg.in/yaml.v3 v3.0.1 // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 5 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 6 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 7 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 8 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 9 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 10 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 11 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 12 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 13 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 14 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 15 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 16 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 17 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 18 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strings" 10 | "sync" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func Test(t *testing.T) { 17 | tableTests := []struct { 18 | file string 19 | expected result 20 | }{ 21 | { 22 | file: "./data/itcont_sample_40.txt", 23 | expected: result{ 24 | numRows: 40, 25 | peopleCount: 35, 26 | commonName: "LAURA", 27 | commonNameCount: 4, 28 | donationMonthFreq: map[string]int{"01": 7, "02": 2, "03": 6, "04": 2, "05": 3, "06": 2, "07": 1, "08": 2, "11": 15}, 29 | }, 30 | }, 31 | { 32 | file: "./data/itcont_sample_4000.txt", 33 | expected: result{ 34 | numRows: 4000, 35 | peopleCount: 35, 36 | commonName: "LAURA", 37 | commonNameCount: 400, 38 | donationMonthFreq: map[string]int{"01": 700, "02": 200, "03": 600, "04": 200, "05": 300, "06": 200, "07": 100, "08": 200, "11": 1500}, 39 | }, 40 | }, 41 | } 42 | 43 | for _, tt := range tableTests { 44 | require.Equal(t, tt.expected, sequential(tt.file)) 45 | require.Equal(t, tt.expected, concurrent(tt.file, 2, 10)) 46 | } 47 | } 48 | 49 | func Benchmark(b *testing.B) { 50 | tableBenchmarks := []struct { 51 | name string 52 | file string 53 | inputs [][]int 54 | benchFn func(file string, numWorkers, batchSize int) result 55 | }{ 56 | { 57 | name: "Sequential", 58 | file: "./data/itcont.txt", 59 | inputs: [][]int{{0, 0}}, 60 | benchFn: func(file string, numWorkers, batchSize int) result { 61 | return sequential(file) 62 | }, 63 | }, 64 | { 65 | name: "Concurrent", 66 | file: "./data/itcont.txt", 67 | inputs: [][]int{{1, 1}, {1, 1000}, {10, 1000}, {10, 10000}, {10, 100000}}, 68 | benchFn: func(file string, numWorkers, batchSize int) result { 69 | return concurrent(file, numWorkers, batchSize) 70 | }, 71 | }, 72 | } 73 | 74 | for _, tb := range tableBenchmarks { 75 | for _, x := range tb.inputs { 76 | numWorkers := x[0] 77 | batchSize := x[1] 78 | 79 | bName := fmt.Sprintf("%s %03d workers %04d batchSize", tb.name, numWorkers, batchSize) 80 | b.Run(bName, func(b *testing.B) { 81 | for i := 0; i < b.N; i++ { 82 | tb.benchFn(tb.file, numWorkers, batchSize) 83 | } 84 | }) 85 | } 86 | } 87 | } 88 | 89 | type result struct { 90 | numRows int 91 | peopleCount int 92 | commonName string 93 | commonNameCount int 94 | donationMonthFreq map[string]int 95 | } 96 | 97 | // processRow takes a pipe-separated line and returns the firstName, fullName, and month. 98 | // this function was created to be somewhat compute intensive and not accurate. 99 | func processRow(text string) (firstName, fullName, month string) { 100 | row := strings.Split(text, "|") 101 | 102 | // extract full name 103 | fullName = strings.Replace(strings.TrimSpace(row[7]), " ", "", -1) 104 | 105 | // extract first name 106 | name := strings.TrimSpace(row[7]) 107 | if name != "" { 108 | startOfName := strings.Index(name, ", ") + 2 109 | if endOfName := strings.Index(name[startOfName:], " "); endOfName < 0 { 110 | firstName = name[startOfName:] 111 | } else { 112 | firstName = name[startOfName : startOfName+endOfName] 113 | } 114 | if strings.HasSuffix(firstName, ",") { 115 | firstName = strings.Replace(firstName, ",", "", -1) 116 | } 117 | } 118 | 119 | // extract month 120 | date := strings.TrimSpace(row[13]) 121 | if len(date) == 8 { 122 | month = date[:2] 123 | } else { 124 | month = "--" 125 | } 126 | 127 | return firstName, fullName, month 128 | } 129 | 130 | // sequential processes a file line by line using processRow. 131 | func sequential(file string) result { 132 | res := result{donationMonthFreq: map[string]int{}} 133 | 134 | f, err := os.Open(file) 135 | if err != nil { 136 | log.Fatal(err) 137 | } 138 | 139 | // track full names 140 | fullNamesRegister := make(map[string]bool) 141 | 142 | // track first name frequency 143 | firstNameMap := make(map[string]int) 144 | 145 | scanner := bufio.NewScanner(f) 146 | for scanner.Scan() { 147 | row := scanner.Text() 148 | firstName, fullName, month := processRow(row) 149 | 150 | // add fullname 151 | fullNamesRegister[fullName] = true 152 | 153 | // update common firstName 154 | firstNameMap[firstName]++ 155 | if firstNameMap[firstName] > res.commonNameCount { 156 | res.commonName = firstName 157 | res.commonNameCount = firstNameMap[firstName] 158 | } 159 | // add month freq 160 | res.donationMonthFreq[month]++ 161 | // update numRows 162 | res.numRows++ 163 | res.peopleCount = len(fullNamesRegister) 164 | } 165 | 166 | return res 167 | } 168 | 169 | // concurrent processes a file by splitting the file 170 | // processing the files concurrently and returning the result. 171 | func concurrent(file string, numWorkers, batchSize int) (res result) { 172 | res = result{donationMonthFreq: map[string]int{}} 173 | 174 | type processed struct { 175 | numRows int 176 | fullNames []string 177 | firstNames []string 178 | months []string 179 | } 180 | 181 | // open file 182 | f, err := os.Open(file) 183 | if err != nil { 184 | log.Fatal(err) 185 | } 186 | 187 | // reader creates and returns a channel that recieves 188 | // batches of rows (of length batchSize) from the file 189 | reader := func(ctx context.Context, rowsBatch *[]string) <-chan []string { 190 | out := make(chan []string) 191 | 192 | scanner := bufio.NewScanner(f) 193 | 194 | go func() { 195 | defer close(out) // close channel when we are done sending all rows 196 | 197 | for { 198 | scanned := scanner.Scan() 199 | 200 | select { 201 | case <-ctx.Done(): 202 | return 203 | default: 204 | row := scanner.Text() 205 | // if batch size is complete or end of file, send batch out 206 | if len(*rowsBatch) == batchSize || !scanned { 207 | out <- *rowsBatch 208 | *rowsBatch = []string{} // clear batch 209 | } 210 | *rowsBatch = append(*rowsBatch, row) // add row to current batch 211 | } 212 | 213 | // if nothing else to scan return 214 | if !scanned { 215 | return 216 | } 217 | } 218 | }() 219 | 220 | return out 221 | } 222 | 223 | // worker takes in a read-only channel to recieve batches of rows. 224 | // After it processes each row-batch it sends out the processed output 225 | // on its channel. 226 | worker := func(ctx context.Context, rowBatch <-chan []string) <-chan processed { 227 | out := make(chan processed) 228 | 229 | go func() { 230 | defer close(out) 231 | 232 | p := processed{} 233 | for rowBatch := range rowBatch { 234 | for _, row := range rowBatch { 235 | firstName, fullName, month := processRow(row) 236 | p.fullNames = append(p.fullNames, fullName) 237 | p.firstNames = append(p.firstNames, firstName) 238 | p.months = append(p.months, month) 239 | p.numRows++ 240 | } 241 | } 242 | out <- p 243 | }() 244 | 245 | return out 246 | } 247 | 248 | // combiner takes in multiple read-only channels that receive processed output 249 | // (from workers) and sends it out on it's own channel via a multiplexer. 250 | combiner := func(ctx context.Context, inputs ...<-chan processed) <-chan processed { 251 | out := make(chan processed) 252 | 253 | var wg sync.WaitGroup 254 | multiplexer := func(p <-chan processed) { 255 | defer wg.Done() 256 | 257 | for in := range p { 258 | select { 259 | case <-ctx.Done(): 260 | case out <- in: 261 | } 262 | } 263 | } 264 | 265 | // add length of input channels to be consumed by mutiplexer 266 | wg.Add(len(inputs)) 267 | for _, in := range inputs { 268 | go multiplexer(in) 269 | } 270 | 271 | // close channel after all inputs channels are closed 272 | go func() { 273 | wg.Wait() 274 | close(out) 275 | }() 276 | 277 | return out 278 | } 279 | 280 | // create a main context, and call cancel at the end, to ensure all our 281 | // goroutines exit without leaving leaks. 282 | // Particularly, if this function becomes part of a program with 283 | // a longer lifetime than this function. 284 | ctx, cancel := context.WithCancel(context.Background()) 285 | defer cancel() 286 | 287 | // STAGE 1: start reader 288 | rowsBatch := []string{} 289 | rowsCh := reader(ctx, &rowsBatch) 290 | 291 | // STAGE 2: create a slice of processed output channels with size of numWorkers 292 | // and assign each slot with the out channel from each worker. 293 | workersCh := make([]<-chan processed, numWorkers) 294 | for i := 0; i < numWorkers; i++ { 295 | workersCh[i] = worker(ctx, rowsCh) 296 | } 297 | 298 | firstNameCount := map[string]int{} 299 | fullNameCount := map[string]bool{} 300 | 301 | // STAGE 3: read from the combined channel and calculate the final result. 302 | // this will end once all channels from workers are closed! 303 | for processed := range combiner(ctx, workersCh...) { 304 | // add number of rows processed by worker 305 | res.numRows += processed.numRows 306 | 307 | // add months processed by worker 308 | for _, month := range processed.months { 309 | res.donationMonthFreq[month]++ 310 | } 311 | 312 | // use full names to count people 313 | for _, fullName := range processed.fullNames { 314 | fullNameCount[fullName] = true 315 | } 316 | res.peopleCount = len(fullNameCount) 317 | 318 | // update most common first name based on processed results 319 | for _, firstName := range processed.firstNames { 320 | firstNameCount[firstName]++ 321 | 322 | if firstNameCount[firstName] > res.commonNameCount { 323 | res.commonName = firstName 324 | res.commonNameCount = firstNameCount[firstName] 325 | } 326 | } 327 | } 328 | 329 | return res 330 | } 331 | --------------------------------------------------------------------------------