155 lines
3.2 KiB
Go
155 lines
3.2 KiB
Go
package storefile
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
eenc "git.keganmyers.com/terribleplan/file-store/pkg/erasureencode"
|
|
filemeta "git.keganmyers.com/terribleplan/file-store/pkg/file/meta"
|
|
)
|
|
|
|
func StoreFile(inputPath, outputPath string, size int32, shards, parity uint16, name string) error {
|
|
if size <= 0 {
|
|
return fmt.Errorf("size must be greater than 0")
|
|
}
|
|
|
|
// fmt.Printf("starting...\n")
|
|
inputPath, err := filepath.Abs(inputPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
inputFile, err := os.Open(inputPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer inputFile.Close()
|
|
|
|
if err := os.MkdirAll(outputPath, 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
if dirents, err := os.ReadDir(outputPath); err != nil {
|
|
return err
|
|
} else if len(dirents) > 0 {
|
|
return fmt.Errorf("expected output dir to be empty")
|
|
}
|
|
|
|
outputFileCount := shards + parity
|
|
outputFiles := make([]*os.File, outputFileCount)
|
|
for i := uint16(0); i < outputFileCount; i++ {
|
|
outputFile, err := os.OpenFile(filepath.Join(outputPath, fmt.Sprintf("shard.%04d", i+1)), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer outputFile.Close()
|
|
outputFiles[i] = outputFile
|
|
}
|
|
|
|
outputs := make([]io.Writer, outputFileCount)
|
|
for i, f := range outputFiles {
|
|
outputs[i] = f
|
|
}
|
|
|
|
// fmt.Printf("writing shards...\n")
|
|
meta, err := eenc.EncodeFile(inputFile, outputs, size, shards, parity)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
meta.Name = name
|
|
if meta.Name == "" {
|
|
meta.Name = filepath.Base(inputPath)
|
|
}
|
|
|
|
// fmt.Printf("writing meta...\n")
|
|
metaFile, err := os.OpenFile(filepath.Join(outputPath, "meta.json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer metaFile.Close()
|
|
|
|
enc := json.NewEncoder(metaFile)
|
|
if err := enc.Encode(meta); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := metaFile.Sync(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// fmt.Printf("done...\n")
|
|
return nil
|
|
}
|
|
|
|
func ReadFile(inputPath, outputPath string, overwrite bool) error {
|
|
inputAbs, err := filepath.Abs(inputPath)
|
|
if err != nil {
|
|
panic(err)
|
|
return err
|
|
}
|
|
|
|
metaFile, err := os.OpenFile(filepath.Join(inputAbs, "meta.json"), os.O_RDONLY, 0644)
|
|
if err != nil {
|
|
panic(err)
|
|
return err
|
|
}
|
|
defer metaFile.Close()
|
|
|
|
meta := &filemeta.Meta{}
|
|
dec := json.NewDecoder(metaFile)
|
|
if err := dec.Decode(meta); err != nil {
|
|
panic(err)
|
|
return err
|
|
}
|
|
|
|
// fmt.Printf("%#v", *meta)
|
|
|
|
if outputPath == "" {
|
|
outputPath = meta.Name
|
|
}
|
|
|
|
outputFlags := os.O_WRONLY
|
|
if overwrite {
|
|
outputFlags |= os.O_CREATE | os.O_TRUNC
|
|
} else {
|
|
outputFlags |= os.O_CREATE | os.O_EXCL
|
|
}
|
|
outputAbs, err := filepath.Abs(outputPath)
|
|
if err != nil {
|
|
panic(err)
|
|
return err
|
|
}
|
|
outputFile, err := os.OpenFile(outputAbs, outputFlags, 0644)
|
|
if err != nil {
|
|
panic(err)
|
|
return err
|
|
}
|
|
|
|
inputFiles := make([]*os.File, meta.Params.Shards)
|
|
for i := uint16(0); i < meta.Params.Shards; i++ {
|
|
inputFile, err := os.OpenFile(filepath.Join(inputPath, fmt.Sprintf("shard.%04d", i+1)), os.O_RDONLY, 0644)
|
|
if err != nil {
|
|
panic(err)
|
|
return err
|
|
}
|
|
defer inputFile.Close()
|
|
inputFiles[i] = inputFile
|
|
}
|
|
|
|
inputs := make([]io.ReadSeeker, meta.Params.Shards)
|
|
for i, f := range inputFiles {
|
|
inputs[i] = f
|
|
}
|
|
|
|
if err := eenc.Decode(inputs, outputFile, meta); err != nil {
|
|
panic(err)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|