123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173 |
- // Copyright 2017 Google Inc. All rights reserved.
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
- // fileslist.py replacement written in GO, which utilizes multi-cores.
- package main
- import (
- "crypto/sha256"
- "encoding/json"
- "flag"
- "fmt"
- "io"
- "os"
- "path/filepath"
- "runtime"
- "sort"
- "strings"
- "sync"
- )
- const (
- MAX_DEFAULT_PARA = 24
- )
- func defaultPara() int {
- ret := runtime.NumCPU()
- if ret > MAX_DEFAULT_PARA {
- return MAX_DEFAULT_PARA
- }
- return ret
- }
- var (
- para = flag.Int("para", defaultPara(), "Number of goroutines")
- )
- // Represents each file.
- type Node struct {
- SHA256 string
- Name string // device side path.
- Size int64
- path string // host side path.
- stat os.FileInfo
- }
- func newNode(hostPath string, devicePath string, stat os.FileInfo) Node {
- return Node{Name: devicePath, path: hostPath, stat: stat}
- }
- // Scan a Node and returns true if it should be added to the result.
- func (n *Node) scan() bool {
- n.Size = n.stat.Size()
- // Calculate SHA256.
- h := sha256.New()
- if n.stat.Mode()&os.ModeSymlink == 0 {
- f, err := os.Open(n.path)
- if err != nil {
- panic(err)
- }
- defer f.Close()
- if _, err := io.Copy(h, f); err != nil {
- panic(err)
- }
- } else {
- // Hash the content of symlink, not the file it points to.
- s, err := os.Readlink(n.path)
- if err != nil {
- panic(err)
- }
- if _, err := io.WriteString(h, s); err != nil {
- panic(err)
- }
- }
- n.SHA256 = fmt.Sprintf("%x", h.Sum(nil))
- return true
- }
- func main() {
- flag.Parse()
- allOutput := make([]Node, 0, 1024) // Store all outputs.
- mutex := &sync.Mutex{} // Guard allOutput
- ch := make(chan Node) // Pass nodes to goroutines.
- var wg sync.WaitGroup // To wait for all goroutines.
- wg.Add(*para)
- // Scan files in multiple goroutines.
- for i := 0; i < *para; i++ {
- go func() {
- defer wg.Done()
- output := make([]Node, 0, 1024) // Local output list.
- for node := range ch {
- if node.scan() {
- output = append(output, node)
- }
- }
- // Add to the global output list.
- mutex.Lock()
- allOutput = append(allOutput, output...)
- mutex.Unlock()
- }()
- }
- // Walk the directories and find files to scan.
- for _, dir := range flag.Args() {
- absDir, err := filepath.Abs(dir)
- if err != nil {
- panic(err)
- }
- deviceRoot := filepath.Clean(absDir + "/..")
- err = filepath.Walk(dir, func(path string, stat os.FileInfo, err error) error {
- if err != nil {
- panic(err)
- }
- if stat.IsDir() {
- return nil
- }
- absPath, err := filepath.Abs(path)
- if err != nil {
- panic(err)
- }
- devicePath, err := filepath.Rel(deviceRoot, absPath)
- if err != nil {
- panic(err)
- }
- devicePath = "/" + devicePath
- ch <- newNode(absPath, devicePath, stat)
- return nil
- })
- if err != nil {
- panic(err)
- }
- }
- // Wait until all the goroutines finish.
- close(ch)
- wg.Wait()
- // Sort the entries and dump as json.
- sort.Slice(allOutput, func(i, j int) bool {
- if allOutput[i].Size > allOutput[j].Size {
- return true
- }
- if allOutput[i].Size == allOutput[j].Size && strings.Compare(allOutput[i].Name, allOutput[j].Name) > 0 {
- return true
- }
- return false
- })
- j, err := json.MarshalIndent(allOutput, "", " ")
- if err != nil {
- panic(nil)
- }
- fmt.Printf("%s\n", j)
- }
|