commit 0dd446c29324ac9a9aafd8ada5ac409cf28efae9 Author: Reya C Date: Fri Apr 2 23:34:54 2021 -0400 Drive Demuxer utility diff --git a/drive-demuxer.go b/drive-demuxer.go new file mode 100644 index 0000000..87fe422 --- /dev/null +++ b/drive-demuxer.go @@ -0,0 +1,333 @@ +package main + +import ( + "bytes" + "flag" + "io" + "log" + "os" + "path" + "sync" +) + +type TopLevel struct { + LeftInput string + RightInput string + LeftOutput string + CombinedOutput string + RightOutput string +} + +type Step struct { + *TopLevel + Subpath string +} + +type Direction byte + +const ( + LEFT Direction = iota + RIGHT + COMBINED +) + +type FileState byte + +const ( + MISSING FileState = iota + FILE + DIRECTORY + UNKNOWN +) + +func (s *Step) CheckState(filePath string) (out FileState, err error) { + info, err := os.Stat(filePath) + if err != nil { + if os.IsNotExist(err) { + return MISSING, nil + } else { + return UNKNOWN, err + } + } + if info.IsDir() { + return DIRECTORY, nil + } else if info.Mode().IsRegular() { + return FILE, nil + } else { + return UNKNOWN, nil + } +} + +func (s *Step) InputPath(child string, side Direction) string { + var basePath string + switch side { + case LEFT: + basePath = s.LeftInput + case RIGHT: + basePath = s.RightInput + default: + panic("Unexpected side for input path") + } + return path.Join(basePath, s.Subpath, child) +} + +func (s *Step) OutputPath(child string, side Direction) string { + var basePath string + switch side { + case LEFT: + basePath = s.LeftOutput + case RIGHT: + basePath = s.RightOutput + case COMBINED: + basePath = s.CombinedOutput + default: + panic("Unexpected side for output path") + } + return path.Join(basePath, s.Subpath, child) +} + +func (s *Step) Separate(child string) { + s.SeparateLeft(child) + s.SeparateRight(child) +} + +func (s *Step) SeparateLeft(child string) { + leftInPath := s.InputPath(child, LEFT) + leftOutPath := s.OutputPath(child, LEFT) + + leftBase := path.Dir(leftOutPath) + err := os.MkdirAll(leftBase, 0755) + if err != nil { + log.Printf("Failed creating %s: %s\n", leftBase, err) + } + err = os.Rename(leftInPath, leftOutPath) + if err != nil && !os.IsNotExist(err) { + log.Printf("Failed moving %s to %s: %s\n", leftInPath, leftOutPath, err) + } +} + +func (s *Step) SeparateRight(child string) { + rightInPath := s.InputPath(child, RIGHT) + rightOutPath := s.OutputPath(child, RIGHT) + + rightBase := path.Dir(rightOutPath) + err := os.MkdirAll(rightBase, 0755) + if err != nil { + log.Printf("Failed creating %s: %s\n", rightBase, err) + } + err = os.Rename(rightInPath, rightOutPath) + if err != nil && !os.IsNotExist(err) { + log.Printf("Failed moving %s to %s: %s\n", rightInPath, rightOutPath, err) + } +} + +func (s *Step) Combine(child string) { + leftInPath := s.InputPath(child, LEFT) + rightInPath := s.InputPath(child, RIGHT) + combinedOutPath := s.OutputPath(child, COMBINED) + + combinedBase := path.Dir(combinedOutPath) + err := os.MkdirAll(combinedBase, 0755) + if err != nil { + log.Printf("Failed creating %s: %s\n", combinedBase, err) + } + err = os.Rename(leftInPath, combinedOutPath) + if err != nil && !os.IsNotExist(err) { + log.Printf("Failed moving %s to %s: %s\n", leftInPath, combinedOutPath, err) + } + err = os.Remove(rightInPath) + if err != nil && !os.IsNotExist(err) { + log.Printf("Failed removing %s: %s\n", rightInPath, err) + } +} + +func (s *Step) MakeCombinedDir(child string) { + combinedOutPath := s.OutputPath(child, COMBINED) + err := os.MkdirAll(combinedOutPath, 0755) + if err != nil { + log.Printf("Failed creating %s: %s\n", combinedOutPath, err) + } +} + +func (s *Step) RemoveInputDirs(child string) { + leftInPath := s.InputPath(child, LEFT) + rightInPath := s.InputPath(child, RIGHT) + err := os.Remove(leftInPath) + if err != nil && !os.IsNotExist(err) { + log.Printf("Failed removing %s: %s\n", leftInPath, err) + } + err = os.Remove(rightInPath) + if err != nil && !os.IsNotExist(err) { + log.Printf("Failed removing %s: %s\n", rightInPath, err) + } +} + +func (s *Step) ListChildren() []string { + leftInPath := s.InputPath("", LEFT) + rightInPath := s.InputPath("", RIGHT) + results := make(map[string]bool) + leftFiles, err := os.ReadDir(leftInPath) + if err != nil { + log.Printf("Failed listing %s: %s\n", leftInPath, err) + } + rightFiles, err := os.ReadDir(rightInPath) + if err != nil { + log.Printf("Failed listing %s: %s\n", rightInPath, err) + } + for _, file := range leftFiles { + results[file.Name()] = true + } + for _, file := range rightFiles { + results[file.Name()] = true + } + out := make([]string, 0, len(results)) + for file, _ := range results { + out = append(out, file) + } + return out +} + +func (s *Step) AreFilesIdentical(child string) bool { + leftInPath := s.InputPath(child, LEFT) + rightInPath := s.InputPath(child, RIGHT) + leftInfo, err := os.Stat(leftInPath) + if err != nil { + log.Printf("Error statting path %s: %s\n", leftInPath, err) + return false + } + rightInfo, err := os.Stat(rightInPath) + if err != nil { + log.Printf("Error statting path %s: %s\n", rightInPath, err) + return false + } + if leftInfo.Size() != rightInfo.Size() { + return false + } + + leftFile, err := os.OpenFile(leftInPath, os.O_RDONLY, 0) + if err != nil { + log.Printf("Error opening path %s: %s\n", leftInPath, err) + return false + } + defer func() { + err := leftFile.Close() + if err != nil { + log.Printf("Error closing %s: %s\n", leftInPath, err) + } + }() + rightFile, err := os.OpenFile(rightInPath, os.O_RDONLY, 0) + if err != nil { + log.Printf("Error opening path %s: %s\n", rightInPath, err) + return false + } + defer func() { + err := rightFile.Close() + if err != nil { + log.Printf("Error closing %s: %s\n", rightInPath, err) + } + }() + + var leftData [4096]byte + var rightData [4096]byte + for { + leftRead, err := leftFile.Read(leftData[:]) + if err != nil && err != io.EOF { + log.Printf("Error reading %s: %s\n", leftInPath, err) + return false + } + rightRead, err := rightFile.Read(rightData[:]) + if err != nil && err != io.EOF { + log.Printf("Error reading %s: %s\n", rightInPath, err) + return false + } + if leftRead != rightRead { + return false + } + if !bytes.Equal(leftData[:], rightData[:]) { + return false + } + if err == io.EOF { + return true + } + } +} + +func (s *Step) Walk() { + wg := sync.WaitGroup{} + + for _, child := range s.ListChildren() { + rightPath := s.InputPath(child, RIGHT) + rightState, err := s.CheckState(rightPath) + if err != nil { + log.Printf("Error statting path %s: %s\n", rightPath, err) + continue + } else if rightState == UNKNOWN { + log.Printf("Unknown stat value for path %s\n", rightPath) + continue + } else if rightState == MISSING { + s.SeparateLeft(child) + continue + } + + leftPath := s.InputPath(child, LEFT) + leftState, err := s.CheckState(leftPath) + if err != nil { + log.Printf("Error statting path %s: %s\n", leftPath, err) + continue + } else if leftState == UNKNOWN { + log.Printf("Unknown stat value for path %s\n", leftPath) + continue + } else if leftState == MISSING { + s.SeparateRight(child) + continue + } + + switch rightState { + case FILE: + if leftState == FILE && s.AreFilesIdentical(child) { + s.Combine(child) + } else { + s.Separate(child) + } + case DIRECTORY: + if leftState == DIRECTORY { + s.MakeCombinedDir(child) + substep := Step{ + TopLevel: s.TopLevel, + Subpath: path.Join(s.Subpath, child), + } + wg.Add(1) + go func() { + defer func() { wg.Done() }() + substep.Walk() + s.RemoveInputDirs(child) + }() + } else { + s.Separate(child) + } + default: + panic("Unexpected state") + } + } + + wg.Wait() +} + +func main() { + settings := TopLevel{} + flag.StringVar(&settings.LeftInput, + "left-input", "./input/left", "The name of the left side of the input.") + flag.StringVar(&settings.RightInput, + "right-input", "./input/right", "The name of the right side of the input.") + flag.StringVar(&settings.LeftOutput, + "left-output", "./output/left", "The name of the left side of the output.") + flag.StringVar(&settings.CombinedOutput, + "combined-output", "./output/combined", "The name of the combined side of the output.") + flag.StringVar(&settings.RightOutput, + "right-output", "./output/right", "The name of the right side of the output.") + flag.Parse() + (&Step{ + TopLevel: &settings, + Subpath: "", + }).Walk() +}