deletion now uses a semaphore to not fuck around with IO on many files

master
cheetah 1 year ago
parent c11d776f90
commit 3ab4b38bac

@ -30,6 +30,11 @@ type DB_GMA struct {
Success bool `json:"success"` Success bool `json:"success"`
RetryCounter int `json:"retries"` RetryCounter int `json:"retries"`
} }
type DB_GMA_Alias struct {
Path string `json:"path"`
Hash string `json:"hash"`
Deleted bool `json:"deleted"`
}
type DB_GMADeletionData struct { type DB_GMADeletionData struct {
Deleted bool `json:"deleted"` Deleted bool `json:"deleted"`
} }

@ -39,6 +39,7 @@ var (
//colFile2Chunk adriver.Collection //colFile2Chunk adriver.Collection
colGMA adriver.Collection colGMA adriver.Collection
colGMA2File adriver.Collection colGMA2File adriver.Collection
colAliases adriver.Collection
workerID string workerID string
) )
@ -112,6 +113,11 @@ func InitDatabase() (err error) {
if err != nil { if err != nil {
return err return err
} }
colAliases, err = arangoDB.Collection(arangoCTX, "aliases")
if err != nil {
return err
}
return nil return nil
} }
@ -119,6 +125,7 @@ var JobPoolSize int = 5
var ConcurrencyLimit int = 1 var ConcurrencyLimit int = 1
var WorkerJobPool chan string var WorkerJobPool chan string
var GlobalWriteLock sync.Mutex var GlobalWriteLock sync.Mutex
var GlobalDeleteLock sync.Mutex
func main() { func main() {
workerModeP := flag.String("mode", "", "mode (ingress, rebuild)") workerModeP := flag.String("mode", "", "mode (ingress, rebuild)")
@ -375,6 +382,7 @@ func modeDelete(folderPath string) {
} }
} }
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
deleteSem := common.NewSemaphore(4)
pw := progress.NewWriter() pw := progress.NewWriter()
pw.SetAutoStop(true) pw.SetAutoStop(true)
@ -412,6 +420,8 @@ func modeDelete(folderPath string) {
for _, jobFile := range WorkerJobPool { for _, jobFile := range WorkerJobPool {
wg.Add(1) wg.Add(1)
go func(jobFile string, wg *sync.WaitGroup) { go func(jobFile string, wg *sync.WaitGroup) {
deleteSem.Acquire()
defer deleteSem.Release()
defer wg.Done() defer wg.Done()
defer tracker.Increment(1) defer tracker.Increment(1)
err = DeleteIfSafeGMA(pw, jobFile) err = DeleteIfSafeGMA(pw, jobFile)
@ -486,8 +496,8 @@ func modeIngress(folderPath string, skipN int) {
for _, jobFile := range WorkerJobPool { for _, jobFile := range WorkerJobPool {
wg.Add(1) wg.Add(1)
go func(jobFile string, wg *sync.WaitGroup) { go func(jobFile string, wg *sync.WaitGroup) {
defer wg.Done()
defer tracker.Increment(1) defer tracker.Increment(1)
defer wg.Done()
err = ProcessGMA(pw, jobFile) err = ProcessGMA(pw, jobFile)
if err != nil { if err != nil {
pw.Log(fmt.Sprintf("\nERROR: %v\n", err)) pw.Log(fmt.Sprintf("\nERROR: %v\n", err))
@ -495,6 +505,10 @@ func modeIngress(folderPath string, skipN int) {
panic(err) panic(err)
} }
} }
err = DeleteIfSafeGMA(pw, jobFile)
if err != nil {
pw.Log(fmt.Sprintf("\nERROR: %v\n", err))
}
}(jobFile, &wg) }(jobFile, &wg)
} }
@ -528,14 +542,14 @@ func undoBatch(undoBatch bool, gmaID string, fileIDs []string, gma2FileIDs []str
func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) { func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
var unlockOnce sync.Once var unlockOnce sync.Once
GlobalWriteLock.Lock() // Wait for worker to have slot open GlobalDeleteLock.Lock() // Wait for worker to have slot open
defer unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway defer unlockOnce.Do(GlobalDeleteLock.Unlock) // release anyway
dboGMA := common.DB_GMA{} dboGMA := common.DB_GMA{}
dboGMA.OriginalPath = filePath dboGMA.OriginalPath = filePath
cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR g IN gma FILTER g.originalPath == '%s' RETURN g", dboGMA.OriginalPath), nil) cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR g IN gma FILTER g.originalPath == '%s' RETURN g", dboGMA.OriginalPath), nil)
if err != nil { if err != nil {
return err return fmt.Errorf("545: %v", err)
} }
defer cursor.Close() defer cursor.Close()
@ -547,14 +561,17 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
if driver.IsNoMoreDocuments(err) { if driver.IsNoMoreDocuments(err) {
break break
} else if err != nil { } else if err != nil {
return err return fmt.Errorf("557: %v", err)
} }
if !gma.Success { if !gma.Success {
return fmt.Errorf("GMA with ID %s was not successfull ", gma.ID) return fmt.Errorf("GMA with ID %s was not successfull ", gma.ID)
} }
} }
//} else {
// return fmt.Errorf("no gmas found with path = %s", dboGMA.OriginalPath)
} }
fmt.Println(dboGMA)
fileStat, err := os.Stat(filePath) fileStat, err := os.Stat(filePath)
if err != nil { if err != nil {
return err return err
@ -578,7 +595,7 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
} }
defer gmaReader.Close() defer gmaReader.Close()
unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway unlockOnce.Do(GlobalDeleteLock.Unlock) // release anyway
dboGMA.GMAHash, err = gmaReader.GetSHA256() dboGMA.GMAHash, err = gmaReader.GetSHA256()
if err != nil { if err != nil {
return err return err
@ -590,9 +607,21 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
if err != nil { if err != nil {
return err return err
} }
dboGMA_sameHash := common.DB_GMA{}
if !dboIDExists { if !dboIDExists {
return fmt.Errorf("GMA with ID %s does not exists", dboGMA.ID) return fmt.Errorf("GMA with ID %s does not exists", dboGMA.ID)
} else {
_, err := colGMA.ReadDocument(arangoCTX, dboGMA.ID, &dboGMA_sameHash)
if err != nil {
return fmt.Errorf("609-: %v", err)
}
if dboGMA_sameHash.OriginalPath != filePath {
pw.Log("originalpath differs for this hash")
// DB_GMA_Alias
}
} }
if dboGMA.OriginalPath == filePath {
{ {
recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath)) recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath))
if checkFileExists(recoveryPath) { if checkFileExists(recoveryPath) {
@ -601,6 +630,8 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
return err return err
} }
pw.Log("deleted %s", dboGMA.OriginalPath) pw.Log("deleted %s", dboGMA.OriginalPath)
} else {
pw.Log("%s recoveryPath does not exist %s", filePath, recoveryPath)
} }
lzmaFile := fmt.Sprintf("%s.lzma", dboGMA.OriginalPath) lzmaFile := fmt.Sprintf("%s.lzma", dboGMA.OriginalPath)
if checkFileExists(lzmaFile) { if checkFileExists(lzmaFile) {
@ -611,12 +642,49 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
pw.Log("deleted %s", lzmaFile) pw.Log("deleted %s", lzmaFile)
} }
} }
{
_, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, common.DB_GMADeletionData{ _, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, common.DB_GMADeletionData{
Deleted: true, Deleted: true,
}) })
if err != nil { if err != nil {
return err return err
} }
}
} else {
originalDeletionData := common.DB_GMADeletionData{}
_, err := colGMA.ReadDocument(arangoCTX, dboGMA.ID, &originalDeletionData)
if err != nil {
return fmt.Errorf("650-: %v", err)
}
if originalDeletionData.Deleted {
aliasData := common.DB_GMA_Alias{
Path: filePath,
Hash: dboGMA.GMAHash,
Deleted: true,
}
recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath))
if checkFileExists(recoveryPath) {
err = os.Remove(aliasData.Path)
if err != nil {
return err
}
pw.Log("deleted %s duplicate, original was %s", aliasData.Path, dboGMA.OriginalPath)
} else {
pw.Log("%s recoveryPath does not exist %s", filePath, recoveryPath)
}
lzmaFile := fmt.Sprintf("%s.lzma", aliasData.Path)
if checkFileExists(lzmaFile) {
err = os.Remove(lzmaFile)
if err != nil {
return err
}
pw.Log("deleted %s", lzmaFile)
}
_, _ = colAliases.CreateDocument(arangoCTX, aliasData)
} else {
return fmt.Errorf("originals have not been deleted")
}
}
return nil return nil
} }

Loading…
Cancel
Save