You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
229 lines
5.3 KiB
Go
229 lines
5.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.cheetah.cat/worksucc/gma-puzzles/chunk"
|
|
"git.cheetah.cat/worksucc/gma-puzzles/common"
|
|
adriver "github.com/arangodb/go-driver"
|
|
ahttp "github.com/arangodb/go-driver/http"
|
|
"github.com/schollz/progressbar/v3"
|
|
)
|
|
|
|
var (
|
|
arangoDB adriver.Database
|
|
arangoCTX context.Context
|
|
colChunk adriver.Collection
|
|
colFile adriver.Collection
|
|
colFile2Chunk adriver.Collection
|
|
)
|
|
|
|
func ConnectDB(baseURL string, arangoUser string, arangoPWD string, arangoDatabase string) (driver adriver.Database, ctx context.Context, err error) {
|
|
log.Println("connectDB:", "Starting Connection Process...")
|
|
|
|
// Retry Loop for Failed Connections
|
|
for i := 0; i < 6; i++ {
|
|
if i == 5 {
|
|
return driver, ctx, fmt.Errorf("connectdb unable to connect to database %d times", i)
|
|
} else if i > 0 {
|
|
time.Sleep(30 * time.Second)
|
|
}
|
|
|
|
// Connect to ArangoDB URL
|
|
conn, err := ahttp.NewConnection(ahttp.ConnectionConfig{
|
|
Endpoints: []string{baseURL},
|
|
TLSConfig: &tls.Config{ /*...*/ },
|
|
})
|
|
if err != nil {
|
|
log.Println("connectDB:", "Cannot Connect to ArangoDB!", err)
|
|
continue
|
|
}
|
|
|
|
// Connect Driver to User
|
|
client, err := adriver.NewClient(adriver.ClientConfig{
|
|
Connection: conn,
|
|
Authentication: adriver.BasicAuthentication(arangoUser, arangoPWD),
|
|
})
|
|
if err != nil {
|
|
log.Println("connectDB:", "Cannot Authenticate ArangoDB User!", err)
|
|
continue
|
|
}
|
|
|
|
// Create Context for Database Access
|
|
ctx = context.Background()
|
|
driver, err = client.Database(ctx, arangoDatabase)
|
|
if err != nil {
|
|
log.Println("connectDB:", "Cannot Load ArangoDB Database!", err)
|
|
continue
|
|
}
|
|
log.Println("connectDB:", "Connection Sucessful!")
|
|
return driver, ctx, nil
|
|
}
|
|
|
|
return driver, ctx, fmt.Errorf("connectDB: FUCK HOW DID THIS EXCUTE?")
|
|
}
|
|
func InitDatabase() (err error) {
|
|
arangoDB, arangoCTX, err = ConnectDB(common.ArangoHost, common.ArangoUser, common.ArangoPass, common.ArangoDatabase)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
colChunk, err = arangoDB.Collection(arangoCTX, "chunk")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
colFile, err = arangoDB.Collection(arangoCTX, "file")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
colFile2Chunk, err = arangoDB.Collection(arangoCTX, "file_chunk_map")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func CheckIntegrity(tarPath string) (err error) {
|
|
|
|
return nil
|
|
}
|
|
|
|
type PoolRecoveryData struct {
|
|
PoolID string `json:"_key"`
|
|
Size uint64 `json:"size"`
|
|
Created time.Time `json:"date"`
|
|
Hash string `json:"hash"`
|
|
|
|
ItemCount int `json:"itemCount"`
|
|
Items []string `json:"items"`
|
|
RecoveryData []common.DB_File `json:"recoveryData"`
|
|
}
|
|
|
|
var DoneTaskCount chan int
|
|
var TotalTaskCount chan int
|
|
var DoneTaskCountV int
|
|
var TotalTaskCountV int
|
|
var ConcurrencyLimit int = 12
|
|
var WorkerJobPool chan string
|
|
|
|
func bla() error {
|
|
WorkerJobPool = make(chan string)
|
|
sem := common.NewSemaphore(ConcurrencyLimit)
|
|
wg := sync.WaitGroup{}
|
|
|
|
entries, err := os.ReadDir("/zpool0/cheetah/workshop/garrysmod/gma-inator/chunks/")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
chunkNames := []string{}
|
|
for _, e := range entries {
|
|
if strings.Contains(e.Name(), ".json") {
|
|
continue
|
|
}
|
|
if !e.IsDir() {
|
|
chunkNames = append(chunkNames, e.Name())
|
|
}
|
|
}
|
|
|
|
//
|
|
TotalTaskCount = make(chan int)
|
|
DoneTaskCount = make(chan int)
|
|
validationBar := progressbar.Default(int64(len(chunkNames)), "Validating Chunks")
|
|
go func() {
|
|
for {
|
|
select {
|
|
case <-TotalTaskCount:
|
|
TotalTaskCountV++
|
|
case <-DoneTaskCount:
|
|
DoneTaskCountV++
|
|
validationBar.Add(1)
|
|
if TotalTaskCountV == DoneTaskCountV {
|
|
fmt.Println("APPARENTLY WE are done")
|
|
close(TotalTaskCount)
|
|
close(DoneTaskCount)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
for _, chunkName := range chunkNames {
|
|
wg.Add(1)
|
|
|
|
TotalTaskCount <- 1
|
|
go func(job string, wg *sync.WaitGroup) (err error) {
|
|
sem.Acquire() // Wait for worker to have slot open
|
|
defer sem.Release() // Release the slot
|
|
defer wg.Done() // Finish job
|
|
defer func() {
|
|
DoneTaskCount <- 1
|
|
}()
|
|
//fmt.Printf("Scanning For Local Pools, found %s:", job)
|
|
tarFinalPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/chunks/", job)
|
|
_, err = os.Stat(tarFinalPath)
|
|
if err != nil {
|
|
log.Println(err)
|
|
return err
|
|
}
|
|
parts := strings.Split(job, ".")
|
|
jsonPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/chunks/", fmt.Sprintf("%s.json", parts[0]))
|
|
_, err = os.Stat(jsonPath)
|
|
if err != nil {
|
|
log.Println(err)
|
|
return err
|
|
}
|
|
var dboChunk common.DB_Chunk
|
|
_, err = colChunk.ReadDocument(arangoCTX, parts[0], &dboChunk)
|
|
if err != nil {
|
|
log.Println(err)
|
|
log.Printf("Chunk %s does exist on disk but not in database\n", job)
|
|
}
|
|
|
|
chunkReader, err := chunk.NewChunkReader(tarFinalPath)
|
|
if err != nil {
|
|
log.Println(err)
|
|
return err
|
|
}
|
|
err = chunkReader.LoadRecoveryFile(jsonPath)
|
|
if err != nil {
|
|
log.Println(err)
|
|
return err
|
|
}
|
|
|
|
err = chunkReader.CheckIntegrity()
|
|
if err != nil {
|
|
log.Println(err)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
|
|
}(chunkName, &wg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
// Wait for all jobs to finish
|
|
wg.Wait()
|
|
|
|
return nil
|
|
}
|
|
func main() {
|
|
err := InitDatabase()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
err = bla()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|