|
333 |
334 |
335 | ```go
336 | // MD5All reads all the files in the file tree rooted at root
337 | // and returns a map from file path to the MD5 sum of the file's contents.
338 | // If the directory walk fails or any read operation fails,
339 | // MD5All returns an error.
340 | func MD5All(ctx context.Context, root string) (map[string][md5.Size]byte, error) {
341 | // Make a pool of 20 digesters
342 | in, out := flowmatic.TaskPool(20, digest)
343 |
344 | m := make(map[string][md5.Size]byte)
345 | // Open two goroutines:
346 | // one for reading file names by walking the filesystem
347 | // one for recording results from the digesters in a map
348 | err := flowmatic.All(ctx,
349 | func(ctx context.Context) error {
350 | return walkFilesystem(ctx, root, in)
351 | },
352 | func(ctx context.Context) error {
353 | for r := range out {
354 | if r.Err != nil {
355 | return r.Err
356 | }
357 | m[r.In] = *r.Out
358 | }
359 | return nil
360 | },
361 | )
362 |
363 | return m, err
364 | }
365 |
366 | func walkFilesystem(ctx context.Context, root string, in chan<- string) error {
367 | defer close(in)
368 |
369 | return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
370 | if err != nil {
371 | return err
372 | }
373 | if !info.Mode().IsRegular() {
374 | return nil
375 | }
376 | select {
377 | case in <- path:
378 | case <-ctx.Done():
379 | return ctx.Err()
380 | }
381 |
382 | return nil
383 | })
384 | }
385 |
386 | func digest(path string) (*[md5.Size]byte, error) {
387 | data, err := os.ReadFile(path)
388 | if err != nil {
389 | return nil, err
390 | }
391 | hash := md5.Sum(data)
392 | return &hash, nil
393 | }
394 | ```
395 |
396 | |
397 |
398 |
399 | ```go
400 | type result struct {
401 | path string
402 | sum [md5.Size]byte
403 | }
404 |
405 | // MD5All reads all the files in the file tree rooted at root and returns a map
406 | // from file path to the MD5 sum of the file's contents. If the directory walk
407 | // fails or any read operation fails, MD5All returns an error.
408 | func MD5All(ctx context.Context, root string) (map[string][md5.Size]byte, error) {
409 | // ctx is canceled when g.Wait() returns. When this version of MD5All returns
410 | // - even in case of error! - we know that all of the goroutines have finished
411 | // and the memory they were using can be garbage-collected.
412 | g, ctx := errgroup.WithContext(ctx)
413 | paths := make(chan string)
414 |
415 | g.Go(func() error {
416 | defer close(paths)
417 | return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
418 | if err != nil {
419 | return err
420 | }
421 | if !info.Mode().IsRegular() {
422 | return nil
423 | }
424 | select {
425 | case paths <- path:
426 | case <-ctx.Done():
427 | return ctx.Err()
428 | }
429 | return nil
430 | })
431 | })
432 |
433 | // Start a fixed number of goroutines to read and digest files.
434 | c := make(chan result)
435 | const numDigesters = 20
436 | for i := 0; i < numDigesters; i++ {
437 | g.Go(func() error {
438 | for path := range paths {
439 | data, err := ioutil.ReadFile(path)
440 | if err != nil {
441 | return err
442 | }
443 | select {
444 | case c <- result{path, md5.Sum(data)}:
445 | case <-ctx.Done():
446 | return ctx.Err()
447 | }
448 | }
449 | return nil
450 | })
451 | }
452 | go func() {
453 | g.Wait()
454 | close(c)
455 | }()
456 |
457 | m := make(map[string][md5.Size]byte)
458 | for r := range c {
459 | m[r.path] = r.sum
460 | }
461 | // Check whether any of the goroutines failed. Since g is accumulating the
462 | // errors, we don't need to send them (or check for them) in the individual
463 | // results sent on the channel.
464 | if err := g.Wait(); err != nil {
465 | return nil, err
466 | }
467 | return m, nil
468 | }
469 | ```
470 |
471 | |
472 |
473 |