You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
257 lines
6.2 KiB
Go
257 lines
6.2 KiB
Go
package main
|
|
|
|
import (
|
|
"archive/tar"
|
|
"context"
|
|
"crypto/sha256"
|
|
"crypto/tls"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.cheetah.cat/worksucc/gma-puzzles/chunk"
|
|
"git.cheetah.cat/worksucc/gma-puzzles/common"
|
|
adriver "github.com/arangodb/go-driver"
|
|
ahttp "github.com/arangodb/go-driver/http"
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
var (
|
|
arangoDB adriver.Database
|
|
arangoCTX context.Context
|
|
colChunk adriver.Collection
|
|
colFile adriver.Collection
|
|
colFile2Chunk adriver.Collection
|
|
)
|
|
|
|
func ConnectDB(baseURL string, arangoUser string, arangoPWD string, arangoDatabase string) (driver adriver.Database, ctx context.Context, err error) {
|
|
log.Println("connectDB:", "Starting Connection Process...")
|
|
|
|
// Retry Loop for Failed Connections
|
|
for i := 0; i < 6; i++ {
|
|
if i == 5 {
|
|
return driver, ctx, fmt.Errorf("connectdb unable to connect to database %d times", i)
|
|
} else if i > 0 {
|
|
time.Sleep(30 * time.Second)
|
|
}
|
|
|
|
// Connect to ArangoDB URL
|
|
conn, err := ahttp.NewConnection(ahttp.ConnectionConfig{
|
|
Endpoints: []string{baseURL},
|
|
TLSConfig: &tls.Config{ /*...*/ },
|
|
})
|
|
if err != nil {
|
|
log.Println("connectDB:", "Cannot Connect to ArangoDB!", err)
|
|
continue
|
|
}
|
|
|
|
// Connect Driver to User
|
|
client, err := adriver.NewClient(adriver.ClientConfig{
|
|
Connection: conn,
|
|
Authentication: adriver.BasicAuthentication(arangoUser, arangoPWD),
|
|
})
|
|
if err != nil {
|
|
log.Println("connectDB:", "Cannot Authenticate ArangoDB User!", err)
|
|
continue
|
|
}
|
|
|
|
// Create Context for Database Access
|
|
ctx = context.Background()
|
|
driver, err = client.Database(ctx, arangoDatabase)
|
|
if err != nil {
|
|
log.Println("connectDB:", "Cannot Load ArangoDB Database!", err)
|
|
continue
|
|
}
|
|
log.Println("connectDB:", "Connection Sucessful!")
|
|
return driver, ctx, nil
|
|
}
|
|
|
|
return driver, ctx, fmt.Errorf("connectDB: FUCK HOW DID THIS EXCUTE?")
|
|
}
|
|
func InitDatabase() (err error) {
|
|
arangoDB, arangoCTX, err = ConnectDB("http://192.168.45.8:8529/", "gma-inator", "gma-inator", "gma-inator")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
colChunk, err = arangoDB.Collection(arangoCTX, "chunk")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
colFile, err = arangoDB.Collection(arangoCTX, "file")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
colFile2Chunk, err = arangoDB.Collection(arangoCTX, "file_chunk_map")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func bla() error {
|
|
entries, err := os.ReadDir("/mnt/SC9000/storagePools/")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, e := range entries {
|
|
if strings.Contains(e.Name(), ".json") {
|
|
continue
|
|
}
|
|
if !strings.Contains(e.Name(), "78d8553f-d716-47e3-ad21-c94f00c5e19c") {
|
|
continue
|
|
}
|
|
|
|
if !e.IsDir() {
|
|
fmt.Printf("Scanning For Local Pools, found %s:", e.Name())
|
|
|
|
tarFinalPath := filepath.Join("/mnt/SC9000/storagePools/", e.Name())
|
|
stats, err := os.Stat(tarFinalPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
parts := strings.Split(e.Name(), ".")
|
|
var dboChunk common.DB_Chunk
|
|
_, err = colChunk.ReadDocument(arangoCTX, parts[0], &dboChunk)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
dboChunk.Finalized = true
|
|
dboChunk.NotReady = false
|
|
dboChunk.ReadOnly = true
|
|
dboChunk.Size = stats.Size()
|
|
|
|
zstFile, err := os.Open(tarFinalPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
shaHasher := sha256.New()
|
|
_, err = io.Copy(shaHasher, zstFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
jsonPath := filepath.Join("/mnt/SC9000/storagePools/", fmt.Sprintf("%s.json", parts[0]))
|
|
fmt.Print(jsonPath)
|
|
_, err = os.Stat(jsonPath)
|
|
if err != nil {
|
|
if errors.Is(err, os.ErrNotExist) {
|
|
fmt.Println("json 404")
|
|
// rewrite json from db
|
|
|
|
zstFile.Seek(0, 0)
|
|
decompressor, err := zstd.NewReader(zstFile)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
defer decompressor.Close()
|
|
|
|
items := []string{}
|
|
tarReader := tar.NewReader(decompressor)
|
|
for {
|
|
header, err := tarReader.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
items = append(items, header.Name)
|
|
}
|
|
|
|
poolRecoveryData := chunk.PoolRecoveryData{
|
|
PoolID: parts[0],
|
|
Size: uint64(stats.Size()),
|
|
Created: time.Now(),
|
|
Hash: fmt.Sprintf("%x", shaHasher.Sum(nil)),
|
|
ItemCount: len(items),
|
|
Items: items,
|
|
//RecoveryData,
|
|
}
|
|
dboChunk.Hash = poolRecoveryData.Hash
|
|
dboChunk.FileCount = len(items)
|
|
|
|
//TODO: fetch RecoveryData from DB
|
|
poolRecoveryData.RecoveryData = make([]common.DB_File, len(items))
|
|
_, _, err = colFile.ReadDocuments(arangoCTX, items, poolRecoveryData.RecoveryData)
|
|
if err != nil {
|
|
return fmt.Errorf("error @ReadDocuments %v", err)
|
|
}
|
|
|
|
json, err := json.MarshalIndent(poolRecoveryData, "", "\t")
|
|
if err != nil {
|
|
return fmt.Errorf("error @json.MarshalIndent %v", err)
|
|
}
|
|
|
|
recoveryFile, err := os.Create(jsonPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = recoveryFile.Write(json)
|
|
if err != nil {
|
|
return fmt.Errorf("error @recoveryFile.Write %v", err)
|
|
}
|
|
}
|
|
} else {
|
|
var poolRecoveryData chunk.PoolRecoveryData
|
|
|
|
readJSONFile, err := os.Open(jsonPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer readJSONFile.Close()
|
|
|
|
readBytes, err := io.ReadAll(readJSONFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = json.Unmarshal(readBytes, &poolRecoveryData)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
poolRecoveryData.Size = uint64(stats.Size())
|
|
poolRecoveryData.Created = time.Now()
|
|
poolRecoveryData.Hash = fmt.Sprintf("%x", shaHasher.Sum(nil))
|
|
dboChunk.Hash = poolRecoveryData.Hash
|
|
json, err := json.MarshalIndent(poolRecoveryData, "", "\t")
|
|
if err != nil {
|
|
return fmt.Errorf("error @json.MarshalIndent %v", err)
|
|
}
|
|
recoveryFile, err := os.Create(jsonPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = recoveryFile.Write(json)
|
|
if err != nil {
|
|
return fmt.Errorf("error @recoveryFile.Write %v", err)
|
|
}
|
|
}
|
|
_, err = colChunk.UpdateDocument(arangoCTX, parts[0], &dboChunk)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf(":%d\n", dboChunk.FileCount)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
func main() {
|
|
err := InitDatabase()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
err = bla()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|