worksucc/gma-puzzles
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

deletion now uses a semaphore to not fuck around with IO on many files

Browse source
This commit is contained in:
cheetah 2023-10-19 23:08:39 +02:00
parent c11d776f90
commit 3ab4b38bac
2 changed files with 96 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
common/common.go
View file

@ -30,6 +30,11 @@ type DB_GMA struct {
Success bool `json:"success"`
RetryCounter int `json:"retries"`
}
type DB_GMA_Alias struct {
Path string `json:"path"`
Hash string `json:"hash"`
Deleted bool `json:"deleted"`
}
type DB_GMADeletionData struct {
Deleted bool `json:"deleted"`
}

114
main.go
View file

@ -39,6 +39,7 @@ var (
//colFile2Chunk adriver.Collection
colGMA adriver.Collection
colGMA2File adriver.Collection
colAliases adriver.Collection
workerID string
)
@ -112,6 +113,11 @@ func InitDatabase() (err error) {
if err != nil {
return err
}
colAliases, err = arangoDB.Collection(arangoCTX, "aliases")
if err != nil {
return err
}
return nil
}
@ -119,6 +125,7 @@ var JobPoolSize int = 5
var ConcurrencyLimit int = 1
var WorkerJobPool chan string
var GlobalWriteLock sync.Mutex
var GlobalDeleteLock sync.Mutex
func main() {
workerModeP := flag.String("mode", "", "mode (ingress, rebuild)")
@ -375,6 +382,7 @@ func modeDelete(folderPath string) {
}
}
wg := sync.WaitGroup{}
deleteSem := common.NewSemaphore(4)
pw := progress.NewWriter()
pw.SetAutoStop(true)
@ -412,6 +420,8 @@ func modeDelete(folderPath string) {
for _, jobFile := range WorkerJobPool {
wg.Add(1)
go func(jobFile string, wg *sync.WaitGroup) {
deleteSem.Acquire()
defer deleteSem.Release()
defer wg.Done()
defer tracker.Increment(1)
err = DeleteIfSafeGMA(pw, jobFile)
@ -486,8 +496,8 @@ func modeIngress(folderPath string, skipN int) {
for _, jobFile := range WorkerJobPool {
wg.Add(1)
go func(jobFile string, wg *sync.WaitGroup) {
defer wg.Done()
defer tracker.Increment(1)
defer wg.Done()
err = ProcessGMA(pw, jobFile)
if err != nil {
pw.Log(fmt.Sprintf("\nERROR: %v\n", err))
@ -495,6 +505,10 @@ func modeIngress(folderPath string, skipN int) {
panic(err)
}
}
err = DeleteIfSafeGMA(pw, jobFile)
if err != nil {
pw.Log(fmt.Sprintf("\nERROR: %v\n", err))
}
}(jobFile, &wg)
}
@ -528,14 +542,14 @@ func undoBatch(undoBatch bool, gmaID string, fileIDs []string, gma2FileIDs []str
func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
var unlockOnce sync.Once
GlobalWriteLock.Lock() // Wait for worker to have slot open
defer unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway
GlobalDeleteLock.Lock() // Wait for worker to have slot open
defer unlockOnce.Do(GlobalDeleteLock.Unlock) // release anyway
dboGMA := common.DB_GMA{}
dboGMA.OriginalPath = filePath
cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR g IN gma FILTER g.originalPath == '%s' RETURN g", dboGMA.OriginalPath), nil)
if err != nil {
return err
return fmt.Errorf("545: %v", err)
}
defer cursor.Close()
@ -547,14 +561,17 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
if driver.IsNoMoreDocuments(err) {
break
} else if err != nil {
return err
return fmt.Errorf("557: %v", err)
}
if !gma.Success {
return fmt.Errorf("GMA with ID %s was not successfull ", gma.ID)
}
}
//} else {
// return fmt.Errorf("no gmas found with path = %s", dboGMA.OriginalPath)
}
fmt.Println(dboGMA)
fileStat, err := os.Stat(filePath)
if err != nil {
return err
@ -578,7 +595,7 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
}
defer gmaReader.Close()
unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway
unlockOnce.Do(GlobalDeleteLock.Unlock) // release anyway
dboGMA.GMAHash, err = gmaReader.GetSHA256()
if err != nil {
return err
@ -590,33 +607,84 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
if err != nil {
return err
}
dboGMA_sameHash := common.DB_GMA{}
if !dboIDExists {
return fmt.Errorf("GMA with ID %s does not exists", dboGMA.ID)
} else {
_, err := colGMA.ReadDocument(arangoCTX, dboGMA.ID, &dboGMA_sameHash)
if err != nil {
return fmt.Errorf("609-: %v", err)
}
if dboGMA_sameHash.OriginalPath != filePath {
pw.Log("originalpath differs for this hash")
// DB_GMA_Alias
}
}
{
recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath))
if checkFileExists(recoveryPath) {
err = os.Remove(dboGMA.OriginalPath)
if dboGMA.OriginalPath == filePath {
{
recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath))
if checkFileExists(recoveryPath) {
err = os.Remove(dboGMA.OriginalPath)
if err != nil {
return err
}
pw.Log("deleted %s", dboGMA.OriginalPath)
} else {
pw.Log("%s recoveryPath does not exist %s", filePath, recoveryPath)
}
lzmaFile := fmt.Sprintf("%s.lzma", dboGMA.OriginalPath)
if checkFileExists(lzmaFile) {
err = os.Remove(lzmaFile)
if err != nil {
return err
}
pw.Log("deleted %s", lzmaFile)
}
}
{
_, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, common.DB_GMADeletionData{
Deleted: true,
})
if err != nil {
return err
}
pw.Log("deleted %s", dboGMA.OriginalPath)
}
lzmaFile := fmt.Sprintf("%s.lzma", dboGMA.OriginalPath)
if checkFileExists(lzmaFile) {
err = os.Remove(lzmaFile)
if err != nil {
return err
} else {
originalDeletionData := common.DB_GMADeletionData{}
_, err := colGMA.ReadDocument(arangoCTX, dboGMA.ID, &originalDeletionData)
if err != nil {
return fmt.Errorf("650-: %v", err)
}
if originalDeletionData.Deleted {
aliasData := common.DB_GMA_Alias{
Path: filePath,
Hash: dboGMA.GMAHash,
Deleted: true,
}
pw.Log("deleted %s", lzmaFile)
recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath))
if checkFileExists(recoveryPath) {
err = os.Remove(aliasData.Path)
if err != nil {
return err
}
pw.Log("deleted %s duplicate, original was %s", aliasData.Path, dboGMA.OriginalPath)
} else {
pw.Log("%s recoveryPath does not exist %s", filePath, recoveryPath)
}
lzmaFile := fmt.Sprintf("%s.lzma", aliasData.Path)
if checkFileExists(lzmaFile) {
err = os.Remove(lzmaFile)
if err != nil {
return err
}
pw.Log("deleted %s", lzmaFile)
}
_, _ = colAliases.CreateDocument(arangoCTX, aliasData)
} else {
return fmt.Errorf("originals have not been deleted")
}
}
_, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, common.DB_GMADeletionData{
Deleted: true,
})
if err != nil {
return err
}
return nil
}