added recursiveness and new progbar
parent
f859235f45
commit
7973a6fc33
@ -0,0 +1,138 @@
|
||||
package chunk
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.cheetah.cat/worksucc/gma-puzzles/common"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
type PoolRecoveryData struct {
|
||||
PoolID string `json:"_key"`
|
||||
Size uint64 `json:"size"`
|
||||
Created time.Time `json:"date"`
|
||||
Hash string `json:"hash"`
|
||||
|
||||
ItemCount int `json:"itemCount"`
|
||||
Items []string `json:"items"`
|
||||
RecoveryData []common.DB_File `json:"recoveryData"`
|
||||
}
|
||||
|
||||
type ChunkReader struct {
|
||||
FileHandle *os.File
|
||||
ExpectedHash *string
|
||||
ExpectedSize *uint64
|
||||
}
|
||||
|
||||
func NewChunkReader(fileName string) (_ ChunkReader, err error) {
|
||||
return ChunkReader{}.NewReader(fileName)
|
||||
}
|
||||
func (r ChunkReader) NewReader(fileName string) (_ ChunkReader, err error) {
|
||||
r.FileHandle, err = os.Open(fileName)
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
func (r ChunkReader) NewReaderFrom(fileHandle *os.File) (_ ChunkReader, err error) {
|
||||
r.FileHandle = fileHandle
|
||||
return r, nil
|
||||
}
|
||||
func (r *ChunkReader) LoadRecoveryFile(fileName string) (err error) {
|
||||
var poolRecoveryData PoolRecoveryData
|
||||
|
||||
readJSONFile, err := os.Open(fileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer readJSONFile.Close()
|
||||
|
||||
readBytes, err := io.ReadAll(readJSONFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = json.Unmarshal(readBytes, &poolRecoveryData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.ExpectedHash = &poolRecoveryData.Hash
|
||||
r.ExpectedSize = &poolRecoveryData.Size
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ChunkReader) CheckIntegrity() (err error) {
|
||||
// re-open and check
|
||||
r.FileHandle.Seek(0, 0)
|
||||
shaHasher := sha256.New()
|
||||
hashedBytes, err := io.Copy(shaHasher, r.FileHandle)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
readHash := fmt.Sprintf("%x", shaHasher.Sum(nil))
|
||||
//fmt.Printf("PackPoolTar hash is %s\n", readHash)
|
||||
if readHash != *r.ExpectedHash {
|
||||
return fmt.Errorf("WORM Hash %s != Hash %s", readHash, *r.ExpectedHash)
|
||||
}
|
||||
packFileStats, err := r.FileHandle.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
readSize := packFileStats.Size()
|
||||
if readSize != int64(*r.ExpectedSize) {
|
||||
return fmt.Errorf("WORM Copy FileSize %d != FileSize %d", readSize, *r.ExpectedSize)
|
||||
}
|
||||
// validate written tar-chunk
|
||||
_, err = r.FileHandle.Seek(0, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
decompressor, err := zstd.NewReader(r.FileHandle, zstd.WithDecoderConcurrency(8))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer decompressor.Close()
|
||||
tarFileCheckReader := tar.NewReader(decompressor)
|
||||
|
||||
//filenamesReadBackList := []string{}
|
||||
for {
|
||||
header, err := tarFileCheckReader.Next()
|
||||
//header.PAXRecords
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hasher := sha256.New()
|
||||
hashedBytes, err := io.Copy(hasher, tarFileCheckReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
readBackChecksum := fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
if hashedBytes != header.Size {
|
||||
return fmt.Errorf("validation on output archive, incorrect size file %s has %d should be %d", header.Name, hashedBytes, header.Size)
|
||||
}
|
||||
if header.Name != readBackChecksum {
|
||||
return fmt.Errorf("validation on output archive, incorrect checksum file %s has %s", header.Name, readBackChecksum)
|
||||
}
|
||||
//filenamesReadBackList = append(filenamesReadBackList, header.Name)
|
||||
}
|
||||
|
||||
if hashedBytes != int64(*r.ExpectedSize) {
|
||||
return fmt.Errorf("WORM Copy HashedBytes %d != FileSize %d", hashedBytes, *r.ExpectedSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ChunkReader) Close() {
|
||||
r.FileHandle.Close()
|
||||
}
|
||||
@ -0,0 +1,228 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.cheetah.cat/worksucc/gma-puzzles/chunk"
|
||||
"git.cheetah.cat/worksucc/gma-puzzles/common"
|
||||
adriver "github.com/arangodb/go-driver"
|
||||
ahttp "github.com/arangodb/go-driver/http"
|
||||
"github.com/schollz/progressbar/v3"
|
||||
)
|
||||
|
||||
var (
|
||||
arangoDB adriver.Database
|
||||
arangoCTX context.Context
|
||||
colChunk adriver.Collection
|
||||
colFile adriver.Collection
|
||||
colFile2Chunk adriver.Collection
|
||||
)
|
||||
|
||||
func ConnectDB(baseURL string, arangoUser string, arangoPWD string, arangoDatabase string) (driver adriver.Database, ctx context.Context, err error) {
|
||||
log.Println("connectDB:", "Starting Connection Process...")
|
||||
|
||||
// Retry Loop for Failed Connections
|
||||
for i := 0; i < 6; i++ {
|
||||
if i == 5 {
|
||||
return driver, ctx, fmt.Errorf("connectdb unable to connect to database %d times", i)
|
||||
} else if i > 0 {
|
||||
time.Sleep(30 * time.Second)
|
||||
}
|
||||
|
||||
// Connect to ArangoDB URL
|
||||
conn, err := ahttp.NewConnection(ahttp.ConnectionConfig{
|
||||
Endpoints: []string{baseURL},
|
||||
TLSConfig: &tls.Config{ /*...*/ },
|
||||
})
|
||||
if err != nil {
|
||||
log.Println("connectDB:", "Cannot Connect to ArangoDB!", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Connect Driver to User
|
||||
client, err := adriver.NewClient(adriver.ClientConfig{
|
||||
Connection: conn,
|
||||
Authentication: adriver.BasicAuthentication(arangoUser, arangoPWD),
|
||||
})
|
||||
if err != nil {
|
||||
log.Println("connectDB:", "Cannot Authenticate ArangoDB User!", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Create Context for Database Access
|
||||
ctx = context.Background()
|
||||
driver, err = client.Database(ctx, arangoDatabase)
|
||||
if err != nil {
|
||||
log.Println("connectDB:", "Cannot Load ArangoDB Database!", err)
|
||||
continue
|
||||
}
|
||||
log.Println("connectDB:", "Connection Sucessful!")
|
||||
return driver, ctx, nil
|
||||
}
|
||||
|
||||
return driver, ctx, fmt.Errorf("connectDB: FUCK HOW DID THIS EXCUTE?")
|
||||
}
|
||||
func InitDatabase() (err error) {
|
||||
arangoDB, arangoCTX, err = ConnectDB("http://192.168.45.8:8529/", "gma-inator", "gma-inator", "gma-inator")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
colChunk, err = arangoDB.Collection(arangoCTX, "chunk")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
colFile, err = arangoDB.Collection(arangoCTX, "file")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
colFile2Chunk, err = arangoDB.Collection(arangoCTX, "file_chunk_map")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func CheckIntegrity(tarPath string) (err error) {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type PoolRecoveryData struct {
|
||||
PoolID string `json:"_key"`
|
||||
Size uint64 `json:"size"`
|
||||
Created time.Time `json:"date"`
|
||||
Hash string `json:"hash"`
|
||||
|
||||
ItemCount int `json:"itemCount"`
|
||||
Items []string `json:"items"`
|
||||
RecoveryData []common.DB_File `json:"recoveryData"`
|
||||
}
|
||||
|
||||
var DoneTaskCount chan int
|
||||
var TotalTaskCount chan int
|
||||
var DoneTaskCountV int
|
||||
var TotalTaskCountV int
|
||||
var ConcurrencyLimit int = 12
|
||||
var WorkerJobPool chan string
|
||||
|
||||
func bla() error {
|
||||
WorkerJobPool = make(chan string)
|
||||
sem := common.NewSemaphore(ConcurrencyLimit)
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
entries, err := os.ReadDir("/mnt/SC9000/storagePools/")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunkNames := []string{}
|
||||
for _, e := range entries {
|
||||
if strings.Contains(e.Name(), ".json") {
|
||||
continue
|
||||
}
|
||||
if !e.IsDir() {
|
||||
chunkNames = append(chunkNames, e.Name())
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
TotalTaskCount = make(chan int)
|
||||
DoneTaskCount = make(chan int)
|
||||
validationBar := progressbar.Default(int64(len(chunkNames)), "Validating Chunks")
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-TotalTaskCount:
|
||||
TotalTaskCountV++
|
||||
case <-DoneTaskCount:
|
||||
DoneTaskCountV++
|
||||
validationBar.Add(1)
|
||||
if TotalTaskCountV == DoneTaskCountV {
|
||||
fmt.Println("APPARENTLY WE are done")
|
||||
close(TotalTaskCount)
|
||||
close(DoneTaskCount)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
for _, chunkName := range chunkNames {
|
||||
wg.Add(1)
|
||||
TotalTaskCount <- 1
|
||||
go func(job string, wg *sync.WaitGroup) (err error) {
|
||||
sem.Acquire() // Wait for worker to have slot open
|
||||
defer sem.Release() // Release the slot
|
||||
defer wg.Done() // Finish job
|
||||
defer func() {
|
||||
DoneTaskCount <- 1
|
||||
}()
|
||||
|
||||
//fmt.Printf("Scanning For Local Pools, found %s:", job)
|
||||
tarFinalPath := filepath.Join("/mnt/SC9000/storagePools/", job)
|
||||
_, err = os.Stat(tarFinalPath)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
}
|
||||
parts := strings.Split(job, ".")
|
||||
jsonPath := filepath.Join("/mnt/SC9000/storagePools/", fmt.Sprintf("%s.json", parts[0]))
|
||||
_, err = os.Stat(jsonPath)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
}
|
||||
var dboChunk common.DB_Chunk
|
||||
_, err = colChunk.ReadDocument(arangoCTX, parts[0], &dboChunk)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
log.Printf("Chunk %s does exist on disk but not in database\n", job)
|
||||
}
|
||||
|
||||
chunkReader, err := chunk.NewChunkReader(tarFinalPath)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
}
|
||||
err = chunkReader.LoadRecoveryFile(jsonPath)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = chunkReader.CheckIntegrity()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}(chunkName, &wg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Wait for all jobs to finish
|
||||
wg.Wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
func main() {
|
||||
err := InitDatabase()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = bla()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue