diff --git a/.gitignore b/.gitignore index b2ddb69..924228d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ gmad_linux storageserver/test/ zstd-tar-test/ gma-puzzles +common/config.go diff --git a/common/common.go b/common/common.go index 059d036..ecc053d 100644 --- a/common/common.go +++ b/common/common.go @@ -30,6 +30,9 @@ type DB_GMA struct { Success bool `json:"success"` RetryCounter int `json:"retries"` } +type DB_GMADeletionData struct { + Deleted bool `json:"deleted"` +} type DB_File struct { ID string `json:"_key"` diff --git a/main.go b/main.go index 9947187..18a1290 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "flag" "fmt" "io" @@ -147,6 +148,9 @@ func main() { case "ingress": workerID = *workerNameP modeIngress(*folderPathP, *skipNameP) + case "delete": + workerID = *workerNameP + modeDelete(*folderPathP) case "rebuild": flag.Parse() err = modeRebuild(*rebuildIDP) @@ -347,6 +351,74 @@ func recursive(jobs []string, folderPath string) (WorkerJobPool []string) { return jobs } +func modeDelete(folderPath string) { + // skipNameEnabled := skipN > 0 + entries, err := os.ReadDir(folderPath) + if err != nil { + panic(err) + } + var WorkerJobPool []string + + for _, e := range entries { + fullPath := filepath.Join(folderPath, e.Name()) + if e.IsDir() { + WorkerJobPool = recursive(WorkerJobPool, fullPath) + } + if !e.IsDir() { + WorkerJobPool = append(WorkerJobPool, filepath.Join(folderPath, e.Name())) + } + } + wg := sync.WaitGroup{} + + pw := progress.NewWriter() + pw.SetAutoStop(true) + pw.SetTrackerLength(40) + pw.SetMessageWidth(40) + //pw.SetNumTrackersExpected(*flagNumTrackers) + pw.SetSortBy(progress.SortByPercentDsc) + pw.SetStyle(progress.StyleDefault) + pw.SetTrackerPosition(progress.PositionRight) + pw.SetUpdateFrequency(time.Millisecond * 100) + pw.Style().Colors = progress.StyleColorsExample + pw.Style().Options.PercentFormat = "%4.1f%%" + pw.Style().Options.ETAPrecision = time.Second + pw.Style().Options.TimeDonePrecision = time.Second + pw.Style().Options.TimeInProgressPrecision = time.Second + pw.Style().Options.TimeOverallPrecision = time.Second + pw.Style().Visibility.ETA = true + pw.Style().Visibility.ETAOverall = true + pw.Style().Visibility.Percentage = false + pw.Style().Visibility.Speed = true + pw.Style().Visibility.SpeedOverall = false + pw.Style().Visibility.Time = true + pw.Style().Visibility.TrackerOverall = false + pw.Style().Visibility.Value = true + pw.Style().Visibility.Pinned = true + + // call Render() in async mode; yes we don't have any trackers at the moment + go pw.Render() + + trackerDoneMarker := sync.Once{} + tracker := progress.Tracker{Message: fmt.Sprintf("Deleting successfull %d GMAs", len(WorkerJobPool)), Total: int64(len(WorkerJobPool)), Units: progress.UnitsDefault} + pw.AppendTracker(&tracker) + defer trackerDoneMarker.Do(tracker.MarkAsDone) + + for _, jobFile := range WorkerJobPool { + wg.Add(1) + go func(jobFile string, wg *sync.WaitGroup) { + defer wg.Done() + defer tracker.Increment(1) + err = DeleteIfSafeGMA(pw, jobFile) + if err != nil { + pw.Log(fmt.Sprintf("\nERROR: %v\n", err)) + } + + }(jobFile, &wg) + } + + // Wait for all jobs to finish + wg.Wait() +} func modeIngress(folderPath string, skipN int) { // skipNameEnabled := skipN > 0 entries, err := os.ReadDir(folderPath) @@ -443,6 +515,99 @@ func undoBatch(undoBatch bool, gmaID string, fileIDs []string, gma2FileIDs []str */ return nil } + +func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) { + var unlockOnce sync.Once + + GlobalWriteLock.Lock() // Wait for worker to have slot open + defer unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway + dboGMA := common.DB_GMA{} + + dboGMA.OriginalPath = filePath + cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR g IN gma FILTER g.originalPath == '%s' RETURN g", dboGMA.OriginalPath), nil) + if err != nil { + return err + } + + defer cursor.Close() + initialRetryCounter := 0 + if cursor.Count() > 0 || cursor.HasMore() { + for { + gma := common.DB_GMA{} + _, err = cursor.ReadDocument(arangoCTX, &gma) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + return err + } + if !gma.Success { + return fmt.Errorf("GMA with ID %s was not successfull ", gma.ID) + } + } + } + + fileStat, err := os.Stat(filePath) + if err != nil { + return err + } + dboGMA.StatModTime = fileStat.ModTime() + dboGMA.GMASize = fileStat.Size() + dboGMA.RetryCounter = initialRetryCounter + 1 + + if dboGMA.GMASize < 200 { + return fmt.Errorf("GMA File too small, skipping") + } + niceName := filepath.Base(filePath) + trackerProcessDoneMarker := sync.Once{} + trackerProcess := progress.Tracker{Message: fmt.Sprintf("Deleting %s", niceName), Total: 0, Units: progress.UnitsDefault} + defer trackerProcessDoneMarker.Do(trackerProcess.MarkAsDone) + pw.AppendTracker(&trackerProcess) + + gmaReader, err := gma.NewReader(filePath) + if err != nil { + return err + } + defer gmaReader.Close() + + unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway + dboGMA.GMAHash, err = gmaReader.GetSHA256() + if err != nil { + return err + } + dboGMA.ID = dboGMA.GMAHash + gmaReader.FileHandle.Seek(0, 0) + + dboIDExists, err := colGMA.DocumentExists(arangoCTX, dboGMA.ID) + if err != nil { + return err + } + if !dboIDExists { + return fmt.Errorf("GMA with ID %s does not exists", dboGMA.ID) + } + { + recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath)) + if checkFileExists(recoveryPath) { + err = os.Remove(dboGMA.OriginalPath) + if err != nil { + return err + } + fmt.Printf("would delete %s\n", dboGMA.OriginalPath) + } + } + _, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, common.DB_GMADeletionData{ + Deleted: true, + }) + if err != nil { + return err + } + return nil +} + +func checkFileExists(filePath string) bool { + _, error := os.Stat(filePath) + //return !os.IsNotExist(err) + return !errors.Is(error, os.ErrNotExist) +} func ProcessGMA(pw progress.Writer, filePath string) (err error) { var unlockOnce sync.Once