diff --git a/common/common.go b/common/common.go index ecc053d..8737eb2 100644 --- a/common/common.go +++ b/common/common.go @@ -30,6 +30,11 @@ type DB_GMA struct { Success bool `json:"success"` RetryCounter int `json:"retries"` } +type DB_GMA_Alias struct { + Path string `json:"path"` + Hash string `json:"hash"` + Deleted bool `json:"deleted"` +} type DB_GMADeletionData struct { Deleted bool `json:"deleted"` } diff --git a/main.go b/main.go index 5184603..5848f5f 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,7 @@ var ( //colFile2Chunk adriver.Collection colGMA adriver.Collection colGMA2File adriver.Collection + colAliases adriver.Collection workerID string ) @@ -112,6 +113,11 @@ func InitDatabase() (err error) { if err != nil { return err } + + colAliases, err = arangoDB.Collection(arangoCTX, "aliases") + if err != nil { + return err + } return nil } @@ -119,6 +125,7 @@ var JobPoolSize int = 5 var ConcurrencyLimit int = 1 var WorkerJobPool chan string var GlobalWriteLock sync.Mutex +var GlobalDeleteLock sync.Mutex func main() { workerModeP := flag.String("mode", "", "mode (ingress, rebuild)") @@ -375,6 +382,7 @@ func modeDelete(folderPath string) { } } wg := sync.WaitGroup{} + deleteSem := common.NewSemaphore(4) pw := progress.NewWriter() pw.SetAutoStop(true) @@ -412,6 +420,8 @@ func modeDelete(folderPath string) { for _, jobFile := range WorkerJobPool { wg.Add(1) go func(jobFile string, wg *sync.WaitGroup) { + deleteSem.Acquire() + defer deleteSem.Release() defer wg.Done() defer tracker.Increment(1) err = DeleteIfSafeGMA(pw, jobFile) @@ -486,8 +496,8 @@ func modeIngress(folderPath string, skipN int) { for _, jobFile := range WorkerJobPool { wg.Add(1) go func(jobFile string, wg *sync.WaitGroup) { - defer wg.Done() defer tracker.Increment(1) + defer wg.Done() err = ProcessGMA(pw, jobFile) if err != nil { pw.Log(fmt.Sprintf("\nERROR: %v\n", err)) @@ -495,6 +505,10 @@ func modeIngress(folderPath string, skipN int) { panic(err) } } + err = DeleteIfSafeGMA(pw, jobFile) + if err != nil { + pw.Log(fmt.Sprintf("\nERROR: %v\n", err)) + } }(jobFile, &wg) } @@ -528,14 +542,14 @@ func undoBatch(undoBatch bool, gmaID string, fileIDs []string, gma2FileIDs []str func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) { var unlockOnce sync.Once - GlobalWriteLock.Lock() // Wait for worker to have slot open - defer unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway + GlobalDeleteLock.Lock() // Wait for worker to have slot open + defer unlockOnce.Do(GlobalDeleteLock.Unlock) // release anyway dboGMA := common.DB_GMA{} dboGMA.OriginalPath = filePath cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR g IN gma FILTER g.originalPath == '%s' RETURN g", dboGMA.OriginalPath), nil) if err != nil { - return err + return fmt.Errorf("545: %v", err) } defer cursor.Close() @@ -547,14 +561,17 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) { if driver.IsNoMoreDocuments(err) { break } else if err != nil { - return err + return fmt.Errorf("557: %v", err) } if !gma.Success { return fmt.Errorf("GMA with ID %s was not successfull ", gma.ID) } } + //} else { + // return fmt.Errorf("no gmas found with path = %s", dboGMA.OriginalPath) } + fmt.Println(dboGMA) fileStat, err := os.Stat(filePath) if err != nil { return err @@ -578,7 +595,7 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) { } defer gmaReader.Close() - unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway + unlockOnce.Do(GlobalDeleteLock.Unlock) // release anyway dboGMA.GMAHash, err = gmaReader.GetSHA256() if err != nil { return err @@ -590,32 +607,83 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) { if err != nil { return err } + + dboGMA_sameHash := common.DB_GMA{} if !dboIDExists { return fmt.Errorf("GMA with ID %s does not exists", dboGMA.ID) + } else { + _, err := colGMA.ReadDocument(arangoCTX, dboGMA.ID, &dboGMA_sameHash) + if err != nil { + return fmt.Errorf("609-: %v", err) + } + if dboGMA_sameHash.OriginalPath != filePath { + pw.Log("originalpath differs for this hash") + // DB_GMA_Alias + } } - { - recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath)) - if checkFileExists(recoveryPath) { - err = os.Remove(dboGMA.OriginalPath) - if err != nil { - return err + if dboGMA.OriginalPath == filePath { + { + recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath)) + if checkFileExists(recoveryPath) { + err = os.Remove(dboGMA.OriginalPath) + if err != nil { + return err + } + pw.Log("deleted %s", dboGMA.OriginalPath) + } else { + pw.Log("%s recoveryPath does not exist %s", filePath, recoveryPath) + } + lzmaFile := fmt.Sprintf("%s.lzma", dboGMA.OriginalPath) + if checkFileExists(lzmaFile) { + err = os.Remove(lzmaFile) + if err != nil { + return err + } + pw.Log("deleted %s", lzmaFile) } - pw.Log("deleted %s", dboGMA.OriginalPath) } - lzmaFile := fmt.Sprintf("%s.lzma", dboGMA.OriginalPath) - if checkFileExists(lzmaFile) { - err = os.Remove(lzmaFile) + { + _, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, common.DB_GMADeletionData{ + Deleted: true, + }) if err != nil { return err } - pw.Log("deleted %s", lzmaFile) } - } - _, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, common.DB_GMADeletionData{ - Deleted: true, - }) - if err != nil { - return err + } else { + originalDeletionData := common.DB_GMADeletionData{} + _, err := colGMA.ReadDocument(arangoCTX, dboGMA.ID, &originalDeletionData) + if err != nil { + return fmt.Errorf("650-: %v", err) + } + if originalDeletionData.Deleted { + aliasData := common.DB_GMA_Alias{ + Path: filePath, + Hash: dboGMA.GMAHash, + Deleted: true, + } + recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath)) + if checkFileExists(recoveryPath) { + err = os.Remove(aliasData.Path) + if err != nil { + return err + } + pw.Log("deleted %s duplicate, original was %s", aliasData.Path, dboGMA.OriginalPath) + } else { + pw.Log("%s recoveryPath does not exist %s", filePath, recoveryPath) + } + lzmaFile := fmt.Sprintf("%s.lzma", aliasData.Path) + if checkFileExists(lzmaFile) { + err = os.Remove(lzmaFile) + if err != nil { + return err + } + pw.Log("deleted %s", lzmaFile) + } + _, _ = colAliases.CreateDocument(arangoCTX, aliasData) + } else { + return fmt.Errorf("originals have not been deleted") + } } return nil }