package main import ( "context" "crypto/tls" "fmt" "log" "os" "path/filepath" "strings" "sync" "time" "git.cheetah.cat/worksucc/gma-puzzles/chunk" "git.cheetah.cat/worksucc/gma-puzzles/common" adriver "github.com/arangodb/go-driver" ahttp "github.com/arangodb/go-driver/http" "github.com/schollz/progressbar/v3" ) var ( arangoDB adriver.Database arangoCTX context.Context colChunk adriver.Collection colFile adriver.Collection colFile2Chunk adriver.Collection ) func ConnectDB(baseURL string, arangoUser string, arangoPWD string, arangoDatabase string) (driver adriver.Database, ctx context.Context, err error) { log.Println("connectDB:", "Starting Connection Process...") // Retry Loop for Failed Connections for i := 0; i < 6; i++ { if i == 5 { return driver, ctx, fmt.Errorf("connectdb unable to connect to database %d times", i) } else if i > 0 { time.Sleep(30 * time.Second) } // Connect to ArangoDB URL conn, err := ahttp.NewConnection(ahttp.ConnectionConfig{ Endpoints: []string{baseURL}, TLSConfig: &tls.Config{ /*...*/ }, }) if err != nil { log.Println("connectDB:", "Cannot Connect to ArangoDB!", err) continue } // Connect Driver to User client, err := adriver.NewClient(adriver.ClientConfig{ Connection: conn, Authentication: adriver.BasicAuthentication(arangoUser, arangoPWD), }) if err != nil { log.Println("connectDB:", "Cannot Authenticate ArangoDB User!", err) continue } // Create Context for Database Access ctx = context.Background() driver, err = client.Database(ctx, arangoDatabase) if err != nil { log.Println("connectDB:", "Cannot Load ArangoDB Database!", err) continue } log.Println("connectDB:", "Connection Sucessful!") return driver, ctx, nil } return driver, ctx, fmt.Errorf("connectDB: FUCK HOW DID THIS EXCUTE?") } func InitDatabase() (err error) { arangoDB, arangoCTX, err = ConnectDB(common.ArangoHost, common.ArangoUser, common.ArangoPass, common.ArangoDatabase) if err != nil { return err } colChunk, err = arangoDB.Collection(arangoCTX, "chunk") if err != nil { return err } colFile, err = arangoDB.Collection(arangoCTX, "file") if err != nil { return err } colFile2Chunk, err = arangoDB.Collection(arangoCTX, "file_chunk_map") if err != nil { return err } return nil } func CheckIntegrity(tarPath string) (err error) { return nil } type PoolRecoveryData struct { PoolID string `json:"_key"` Size uint64 `json:"size"` Created time.Time `json:"date"` Hash string `json:"hash"` ItemCount int `json:"itemCount"` Items []string `json:"items"` RecoveryData []common.DB_File `json:"recoveryData"` } var DoneTaskCount chan int var TotalTaskCount chan int var DoneTaskCountV int var TotalTaskCountV int var ConcurrencyLimit int = 12 var WorkerJobPool chan string func bla() error { WorkerJobPool = make(chan string) sem := common.NewSemaphore(ConcurrencyLimit) wg := sync.WaitGroup{} entries, err := os.ReadDir("/zpool0/cheetah/workshop/garrysmod/gma-inator/chunks/") if err != nil { return err } chunkNames := []string{} for _, e := range entries { if strings.Contains(e.Name(), ".json") { continue } if !e.IsDir() { chunkNames = append(chunkNames, e.Name()) } } // TotalTaskCount = make(chan int) DoneTaskCount = make(chan int) validationBar := progressbar.Default(int64(len(chunkNames)), "Validating Chunks") go func() { for { select { case <-TotalTaskCount: TotalTaskCountV++ case <-DoneTaskCount: DoneTaskCountV++ validationBar.Add(1) if TotalTaskCountV == DoneTaskCountV { fmt.Println("APPARENTLY WE are done") close(TotalTaskCount) close(DoneTaskCount) return } } } }() for _, chunkName := range chunkNames { wg.Add(1) TotalTaskCount <- 1 go func(job string, wg *sync.WaitGroup) (err error) { sem.Acquire() // Wait for worker to have slot open defer sem.Release() // Release the slot defer wg.Done() // Finish job defer func() { DoneTaskCount <- 1 }() //fmt.Printf("Scanning For Local Pools, found %s:", job) tarFinalPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/chunks/", job) _, err = os.Stat(tarFinalPath) if err != nil { log.Println(err) return err } parts := strings.Split(job, ".") jsonPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/chunks/", fmt.Sprintf("%s.json", parts[0])) _, err = os.Stat(jsonPath) if err != nil { log.Println(err) return err } var dboChunk common.DB_Chunk _, err = colChunk.ReadDocument(arangoCTX, parts[0], &dboChunk) if err != nil { log.Println(err) log.Printf("Chunk %s does exist on disk but not in database\n", job) } chunkReader, err := chunk.NewChunkReader(tarFinalPath) if err != nil { log.Println(err) return err } err = chunkReader.LoadRecoveryFile(jsonPath) if err != nil { log.Println(err) return err } err = chunkReader.CheckIntegrity() if err != nil { log.Println(err) return err } return nil }(chunkName, &wg) if err != nil { return err } } // Wait for all jobs to finish wg.Wait() return nil } func main() { err := InitDatabase() if err != nil { panic(err) } err = bla() if err != nil { panic(err) } }