package main import ( "archive/tar" "context" "crypto/sha256" "crypto/tls" "encoding/json" "errors" "fmt" "io" "log" "os" "path/filepath" "strings" "time" "git.cheetah.cat/worksucc/gma-puzzles/common" adriver "github.com/arangodb/go-driver" ahttp "github.com/arangodb/go-driver/http" "github.com/klauspost/compress/zstd" ) var ( arangoDB adriver.Database arangoCTX context.Context colChunk adriver.Collection colFile adriver.Collection colFile2Chunk adriver.Collection ) func ConnectDB(baseURL string, arangoUser string, arangoPWD string, arangoDatabase string) (driver adriver.Database, ctx context.Context, err error) { log.Println("connectDB:", "Starting Connection Process...") // Retry Loop for Failed Connections for i := 0; i < 6; i++ { if i == 5 { return driver, ctx, fmt.Errorf("connectdb unable to connect to database %d times", i) } else if i > 0 { time.Sleep(30 * time.Second) } // Connect to ArangoDB URL conn, err := ahttp.NewConnection(ahttp.ConnectionConfig{ Endpoints: []string{baseURL}, TLSConfig: &tls.Config{ /*...*/ }, }) if err != nil { log.Println("connectDB:", "Cannot Connect to ArangoDB!", err) continue } // Connect Driver to User client, err := adriver.NewClient(adriver.ClientConfig{ Connection: conn, Authentication: adriver.BasicAuthentication(arangoUser, arangoPWD), }) if err != nil { log.Println("connectDB:", "Cannot Authenticate ArangoDB User!", err) continue } // Create Context for Database Access ctx = context.Background() driver, err = client.Database(ctx, arangoDatabase) if err != nil { log.Println("connectDB:", "Cannot Load ArangoDB Database!", err) continue } log.Println("connectDB:", "Connection Sucessful!") return driver, ctx, nil } return driver, ctx, fmt.Errorf("connectDB: FUCK HOW DID THIS EXCUTE?") } func InitDatabase() (err error) { arangoDB, arangoCTX, err = ConnectDB("http://192.168.45.8:8529/", "gma-inator", "gma-inator", "gma-inator") if err != nil { return err } colChunk, err = arangoDB.Collection(arangoCTX, "chunk") if err != nil { return err } colFile, err = arangoDB.Collection(arangoCTX, "file") if err != nil { return err } colFile2Chunk, err = arangoDB.Collection(arangoCTX, "file_chunk_map") if err != nil { return err } return nil } type PoolRecoveryData struct { PoolID string `json:"_key"` Size uint64 `json:"size"` Created time.Time `json:"date"` Hash string `json:"hash"` ItemCount int `json:"itemCount"` Items []string `json:"items"` RecoveryData []common.DB_File `json:"recoveryData"` } func bla() error { entries, err := os.ReadDir("/mnt/SC9000/storagePools/") if err != nil { return err } for _, e := range entries { if !e.IsDir() { fmt.Printf("Scanning For Local Pools, found %s:", e.Name()) tarFinalPath := filepath.Join("/mnt/SC9000/storagePools/", e.Name()) stats, err := os.Stat(tarFinalPath) if err != nil { return err } parts := strings.Split(e.Name(), ".") var chunk common.DB_Chunk _, err = colChunk.ReadDocument(arangoCTX, parts[0], &chunk) if err != nil { return err } chunk.Finalized = true chunk.NotReady = false chunk.ReadOnly = true chunk.Size = stats.Size() zstFile, err := os.Open(tarFinalPath) if err != nil { return err } shaHasher := sha256.New() _, err = io.Copy(shaHasher, zstFile) if err != nil { return err } jsonPath := filepath.Join("/mnt/SC9000/storagePools/", fmt.Sprintf("%s.json", parts[0])) _, err = os.Stat(jsonPath) if err != nil { if !errors.Is(err, os.ErrNotExist) { // rewrite json from db zstFile.Seek(0, 0) decompressor, err := zstd.NewReader(zstFile) if err != nil { panic(err) } defer decompressor.Close() items := []string{} tarReader := tar.NewReader(decompressor) for { header, err := tarReader.Next() if err == io.EOF { break } if err != nil { return err } items = append(items, header.Name) } poolRecoveryData := PoolRecoveryData{ PoolID: parts[0], Size: uint64(stats.Size()), Created: time.Now(), Hash: fmt.Sprintf("%x", shaHasher.Sum(nil)), ItemCount: 500, Items: items, //RecoveryData, } chunk.Hash = poolRecoveryData.Hash //TODO: fetch RecoveryData from DB poolRecoveryData.RecoveryData = make([]common.DB_File, len(items)) _, _, err = colFile.ReadDocuments(arangoCTX, items, poolRecoveryData.RecoveryData) if err != nil { return fmt.Errorf("error @ReadDocuments %v", err) } json, err := json.MarshalIndent(poolRecoveryData, "", "\t") if err != nil { return fmt.Errorf("error @json.MarshalIndent %v", err) } recoveryFile, err := os.Create(jsonPath) if err != nil { return err } _, err = recoveryFile.Write(json) if err != nil { return fmt.Errorf("error @recoveryFile.Write %v", err) } } } else { var poolRecoveryData PoolRecoveryData readJSONFile, err := os.Open(jsonPath) if err != nil { return err } defer readJSONFile.Close() readBytes, err := io.ReadAll(readJSONFile) if err != nil { return err } err = json.Unmarshal(readBytes, &poolRecoveryData) if err != nil { return err } poolRecoveryData.Size = uint64(stats.Size()) poolRecoveryData.Created = time.Now() poolRecoveryData.Hash = fmt.Sprintf("%x", shaHasher.Sum(nil)) chunk.Hash = poolRecoveryData.Hash json, err := json.MarshalIndent(poolRecoveryData, "", "\t") if err != nil { return fmt.Errorf("error @json.MarshalIndent %v", err) } recoveryFile, err := os.Create(jsonPath) if err != nil { return err } _, err = recoveryFile.Write(json) if err != nil { return fmt.Errorf("error @recoveryFile.Write %v", err) } } _, err = colChunk.UpdateDocument(arangoCTX, parts[0], &chunk) if err != nil { return err } } } return nil } func main() { err := InitDatabase() if err != nil { panic(err) } err = bla() if err != nil { panic(err) } }