/* File Name: Warehouse.go Copyright: 2021 Peernet Foundation s.r.o. Author: Peter Kleissner */ package warehouse import ( "encoding/hex" "io/ioutil" "os" "path/filepath" ) // Blake3 hash size = 32 bytes. const ( hashSize = 256 / 8 ) // Warehouse represents a folder on disk. type Warehouse struct { Directory string // The main directory for the files Temp string // Temporary folder } // Init initializes the warehouse func Init(Directory string) (wh *Warehouse, err error) { // The temp folder will always be a sub-folder named "_Temp" wh = &Warehouse{Directory: Directory, Temp: filepath.Join(Directory, "_Temp")} if err = createDirectory(wh.Directory); err != nil { return nil, err } if err = createDirectory(wh.Temp); err != nil { return nil, err } return } // ---- hash functions ---- func ValidateHash(hash []byte) (hashA string, err error) { if len(hash) != hashSize { return "", os.ErrInvalid } return hex.EncodeToString(hash), nil } // ---- path ---- func createDirectory(path string) (err error) { if _, err = os.Stat(path); err != nil && os.IsNotExist(err) { err = os.MkdirAll(path, os.ModePerm) } return err } // buildPath returns the full directory and the filename for the hash func buildPath(storagePath, hash string) (directory string, filename string) { part1 := hash[:4] part2 := hash[4:8] filename = hash[8:] newPath := filepath.Join(storagePath, part1, part2) return newPath, filename } // tempFile creates a temporary file in the Warehouse. Do not forget to delete. func (wh *Warehouse) tempFile() (file *os.File, err error) { file, err = ioutil.TempFile(wh.Temp, "wh") return } // createFilePath creates the file path for the specified hash and returns the full file path func (wh *Warehouse) createFilePath(hash []byte) (pathFull string, err error) { path, filename := buildPath(wh.Directory, hex.EncodeToString(hash)) return filepath.Join(path, filename), createDirectory(path) } // IterateFiles iterates through all the files and calls the callback func (wh *Warehouse) IterateFiles(Callback func(Hash []byte, Size int64) (Continue bool)) (err error) { // list all directories in the local Storage folder. We have to walk 2 levels down to see the actual files. files, err := ioutil.ReadDir(wh.Directory) if err != nil { return err } for _, file := range files { name1 := file.Name() _, err = hex.DecodeString(name1) // we are only looking for directories. Name has to be "XXXX" hex chars only. if !file.IsDir() || len(name1) != 4 || err != nil { continue } // iterate through the next level files1, err := ioutil.ReadDir(filepath.Join(wh.Directory, name1)) if err != nil { return err } for _, file2 := range files1 { name2 := file2.Name() _, err = hex.DecodeString(name2) // we are only looking for directories. Name has to be "XXXX" hex chars only. if !file2.IsDir() || len(name2) != 4 || err != nil { continue } // iterate through the next level files2, err := ioutil.ReadDir(filepath.Join(wh.Directory, name1, name2)) if err != nil { return err } for _, file3 := range files2 { name3 := file3.Name() _, err = hex.DecodeString(name3) // finally we are only looking for files if file3.IsDir() || len(name3) != hashSize*2-8 || err != nil { continue } size := file3.Size() hash, err := hex.DecodeString(name1 + name2 + name3) if err != nil { return err } // found the hash! forward if !Callback(hash, size) { return nil } } } } return nil }