Further repair parallelism

drive-demuxer.go

@@ -13,14 +13,15 @@ import (
 )
 
 type TopLevel struct {
-	LeftInput      string
-	RightInput     string
-	LeftOutput     string
-	CombinedOutput string
-	RightOutput    string
-	OpenHandles    chan int
-	DryRun         bool
-	Bar            *pb.ProgressBar
+	LeftInput       string
+	RightInput      string
+	LeftOutput      string
+	CombinedOutput  string
+	RightOutput     string
+	OpenHandles     chan int
+	OpenDirectories chan int
+	DryRun          bool
+	Bar             *pb.ProgressBar
 }
 
 type Step struct {
@@ -297,111 +298,77 @@ func (s *Step) Walk() {
 		rightPath := s.InputPath(child, RIGHT)
 		rightState, err := s.CheckState(rightPath)
 		if err != nil {
-			wg.Add(1)
-			go func(rightPath string) {
-				defer func() {
-					wg.Done()
-				}()
-				log.Printf("Error statting path %s: %s\n", rightPath, err)
-				s.Bar.Increment()
-			}(rightPath)
+			log.Printf("Error statting path %s: %s\n", rightPath, err)
+			s.Bar.Increment()
 			continue
 		} else if rightState == UNKNOWN {
-			wg.Add(1)
-			go func(rightPath string) {
-				defer func() {
-					wg.Done()
-				}()
-				log.Printf("Unknown stat value for path %s\n", rightPath)
-				s.Bar.Increment()
-			}(rightPath)
+			log.Printf("Unknown stat value for path %s\n", rightPath)
+			s.Bar.Increment()
 			continue
 		} else if rightState == MISSING {
-			wg.Add(1)
-			go func(child string) {
-				defer func() {
-					wg.Done()
-				}()
-				s.SeparateLeft(child)
-				s.Bar.Increment()
-			}(child)
+			s.SeparateLeft(child)
+			s.Bar.Increment()
 			continue
 		}
 
 		leftPath := s.InputPath(child, LEFT)
 		leftState, err := s.CheckState(leftPath)
 		if err != nil {
-			wg.Add(1)
-			go func(leftPath string) {
-				defer func() {
-					wg.Done()
-				}()
-				log.Printf("Error statting path %s: %s\n", leftPath, err)
-				s.Bar.Increment()
-			}(leftPath)
+			log.Printf("Error statting path %s: %s\n", leftPath, err)
+			s.Bar.Increment()
 			continue
 		} else if leftState == UNKNOWN {
-			wg.Add(1)
-			go func(leftPath string) {
-				defer func() {
-					wg.Done()
-				}()
-				log.Printf("Unknown stat value for path %s\n", leftPath)
-				s.Bar.Increment()
-			}(leftPath)
+			log.Printf("Unknown stat value for path %s\n", leftPath)
+			s.Bar.Increment()
 			continue
 		} else if leftState == MISSING {
-			wg.Add(1)
-			go func(child string) {
-				defer func() {
-					wg.Done()
-				}()
-				s.SeparateRight(child)
-				s.Bar.Increment()
-			}(child)
+			s.SeparateRight(child)
+			s.Bar.Increment()
 			continue
 		}
 
 		switch rightState {
 		case FILE:
 			wg.Add(1)
-			go func(child string) {
-				defer func() {
-					wg.Done()
-				}()
-				if leftState == FILE && s.AreFilesIdentical(child) {
-					s.Combine(child)
-					s.Bar.Increment()
-				} else {
-					s.Separate(child)
-					s.Bar.Increment()
-				}
-			}(child)
+			if leftState == FILE {
+				go func(child string) {
+					defer func() {
+						wg.Done()
+					}()
+					if s.AreFilesIdentical(child) {
+						s.Combine(child)
+						s.Bar.Increment()
+					} else {
+						s.Separate(child)
+						s.Bar.Increment()
+					}
+				}(child)
+			} else {
+				s.Separate(child)
+				s.Bar.Increment()
+			}
 		case DIRECTORY:
 			if leftState == DIRECTORY {
-				substep := Step{
-					TopLevel: s.TopLevel,
-					Subpath:  path.Join(s.Subpath, child),
-				}
+				s.OpenDirectories <- 1
 				wg.Add(1)
-				go func(child string, substep *Step) {
+				go func(child string) {
 					defer func() {
 						wg.Done()
+						<-s.OpenDirectories
 					}()
+					substep := Step{
+						TopLevel: s.TopLevel,
+						Subpath:  path.Join(s.Subpath, child),
+					}
 					s.MakeCombinedDir(child)
 					substep.Walk()
 					s.RemoveInputDirs(child)
 					s.Bar.Increment()
-				}(child, &substep)
+				}(child)
 			} else {
 				wg.Add(1)
-				go func(child string) {
-					defer func() {
-						wg.Done()
-					}()
-					s.Separate(child)
-					s.Bar.Increment()
-				}(child)
+				s.Separate(child)
+				s.Bar.Increment()
 			}
 		default:
 			panic("Unexpected state")
@@ -413,8 +380,9 @@ func (s *Step) Walk() {
 
 func main() {
 	settings := TopLevel{
-		OpenHandles: make(chan int, 450),
-		Bar:         pb.StartNew(1),
+		OpenHandles:     make(chan int, 450),
+		OpenDirectories: make(chan int, 1000),
+		Bar:             pb.StartNew(1),
 	}
 	flag.StringVar(&settings.LeftInput,
 		"left-input", "./input/left", "The name of the left side of the input.")