Increase parallelism by putting more things in goroutines

main
Mari 4 years ago
parent 305cf18f10
commit be0b9b076c
  1. 59
      drive-demuxer.go

@ -190,6 +190,10 @@ func (s *Step) RemoveInputDirs(child string) {
} }
func (s *Step) ListChildren() []string { func (s *Step) ListChildren() []string {
s.OpenHandles <- 1
defer func() {
<-s.OpenHandles
}()
leftInPath := s.InputPath("", LEFT) leftInPath := s.InputPath("", LEFT)
rightInPath := s.InputPath("", RIGHT) rightInPath := s.InputPath("", RIGHT)
results := make(map[string]bool) results := make(map[string]bool)
@ -215,6 +219,10 @@ func (s *Step) ListChildren() []string {
} }
func (s *Step) AreFilesIdentical(child string) bool { func (s *Step) AreFilesIdentical(child string) bool {
s.OpenHandles <- 1
defer func() {
<-s.OpenHandles
}()
leftInPath := s.InputPath(child, LEFT) leftInPath := s.InputPath(child, LEFT)
rightInPath := s.InputPath(child, RIGHT) rightInPath := s.InputPath(child, RIGHT)
leftInfo, err := os.Stat(leftInPath) leftInfo, err := os.Stat(leftInPath)
@ -254,8 +262,9 @@ func (s *Step) AreFilesIdentical(child string) bool {
} }
}() }()
var leftData [4096]byte // 4 MB buffers
var rightData [4096]byte var leftData [4194304]byte
var rightData [4194304]byte
for { for {
leftRead, err := leftFile.Read(leftData[:]) leftRead, err := leftFile.Read(leftData[:])
if err != nil && err != io.EOF { if err != nil && err != io.EOF {
@ -270,7 +279,7 @@ func (s *Step) AreFilesIdentical(child string) bool {
if leftRead != rightRead { if leftRead != rightRead {
return false return false
} }
if !bytes.Equal(leftData[:], rightData[:]) { if !bytes.Equal(leftData[:leftRead], rightData[:rightRead]) {
return false return false
} }
if err == io.EOF { if err == io.EOF {
@ -282,44 +291,84 @@ func (s *Step) AreFilesIdentical(child string) bool {
func (s *Step) Walk() { func (s *Step) Walk() {
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
s.OpenHandles <- 1
children := s.ListChildren() children := s.ListChildren()
s.Bar.AddTotal(int64(len(children))) s.Bar.AddTotal(int64(len(children)))
for _, child := range children { for _, child := range children {
rightPath := s.InputPath(child, RIGHT) rightPath := s.InputPath(child, RIGHT)
rightState, err := s.CheckState(rightPath) rightState, err := s.CheckState(rightPath)
if err != nil { if err != nil {
wg.Add(1)
go func() {
defer func() {
wg.Done()
}()
log.Printf("Error statting path %s: %s\n", rightPath, err) log.Printf("Error statting path %s: %s\n", rightPath, err)
s.Bar.Increment() s.Bar.Increment()
}()
continue continue
} else if rightState == UNKNOWN { } else if rightState == UNKNOWN {
wg.Add(1)
go func() {
defer func() {
wg.Done()
}()
log.Printf("Unknown stat value for path %s\n", rightPath) log.Printf("Unknown stat value for path %s\n", rightPath)
s.Bar.Increment() s.Bar.Increment()
}()
continue continue
} else if rightState == MISSING { } else if rightState == MISSING {
wg.Add(1)
go func() {
defer func() {
wg.Done()
}()
s.SeparateLeft(child) s.SeparateLeft(child)
s.Bar.Increment() s.Bar.Increment()
}()
continue continue
} }
leftPath := s.InputPath(child, LEFT) leftPath := s.InputPath(child, LEFT)
leftState, err := s.CheckState(leftPath) leftState, err := s.CheckState(leftPath)
if err != nil { if err != nil {
wg.Add(1)
go func() {
defer func() {
wg.Done()
}()
log.Printf("Error statting path %s: %s\n", leftPath, err) log.Printf("Error statting path %s: %s\n", leftPath, err)
s.Bar.Increment() s.Bar.Increment()
}()
continue continue
} else if leftState == UNKNOWN { } else if leftState == UNKNOWN {
wg.Add(1)
go func() {
defer func() {
wg.Done()
}()
log.Printf("Unknown stat value for path %s\n", leftPath) log.Printf("Unknown stat value for path %s\n", leftPath)
s.Bar.Increment() s.Bar.Increment()
}()
continue continue
} else if leftState == MISSING { } else if leftState == MISSING {
wg.Add(1)
go func() {
defer func() {
wg.Done()
}()
s.SeparateRight(child) s.SeparateRight(child)
s.Bar.Increment() s.Bar.Increment()
}()
continue continue
} }
switch rightState { switch rightState {
case FILE: case FILE:
wg.Add(1)
go func() {
defer func() {
wg.Done()
}()
if leftState == FILE && s.AreFilesIdentical(child) { if leftState == FILE && s.AreFilesIdentical(child) {
s.Combine(child) s.Combine(child)
s.Bar.Increment() s.Bar.Increment()
@ -327,6 +376,7 @@ func (s *Step) Walk() {
s.Separate(child) s.Separate(child)
s.Bar.Increment() s.Bar.Increment()
} }
}()
case DIRECTORY: case DIRECTORY:
if leftState == DIRECTORY { if leftState == DIRECTORY {
s.MakeCombinedDir(child) s.MakeCombinedDir(child)
@ -352,7 +402,6 @@ func (s *Step) Walk() {
} }
} }
<-s.OpenHandles
wg.Wait() wg.Wait()
} }

Loading…
Cancel
Save