deletion now uses a semaphore to not fuck around with IO on many files

common/common.go

@@ -30,6 +30,11 @@ type DB_GMA struct {
 	Success        bool          `json:"success"`
 	RetryCounter   int           `json:"retries"`
 }
+type DB_GMA_Alias struct {
+	Path    string `json:"path"`
+	Hash    string `json:"hash"`
+	Deleted bool   `json:"deleted"`
+}
 type DB_GMADeletionData struct {
 	Deleted bool `json:"deleted"`
 }

main.go

@@ -39,6 +39,7 @@ var (
 	//colFile2Chunk adriver.Collection
 	colGMA      adriver.Collection
 	colGMA2File adriver.Collection
+	colAliases  adriver.Collection
 
 	workerID string
 )
@@ -112,6 +113,11 @@ func InitDatabase() (err error) {
 	if err != nil {
 		return err
 	}
+
+	colAliases, err = arangoDB.Collection(arangoCTX, "aliases")
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -119,6 +125,7 @@ var JobPoolSize int = 5
 var ConcurrencyLimit int = 1
 var WorkerJobPool chan string
 var GlobalWriteLock sync.Mutex
+var GlobalDeleteLock sync.Mutex
 
 func main() {
 	workerModeP := flag.String("mode", "", "mode (ingress, rebuild)")
@@ -375,6 +382,7 @@ func modeDelete(folderPath string) {
 		}
 	}
 	wg := sync.WaitGroup{}
+	deleteSem := common.NewSemaphore(4)
 
 	pw := progress.NewWriter()
 	pw.SetAutoStop(true)
@@ -412,6 +420,8 @@ func modeDelete(folderPath string) {
 	for _, jobFile := range WorkerJobPool {
 		wg.Add(1)
 		go func(jobFile string, wg *sync.WaitGroup) {
+			deleteSem.Acquire()
+			defer deleteSem.Release()
 			defer wg.Done()
 			defer tracker.Increment(1)
 			err = DeleteIfSafeGMA(pw, jobFile)
@@ -486,8 +496,8 @@ func modeIngress(folderPath string, skipN int) {
 	for _, jobFile := range WorkerJobPool {
 		wg.Add(1)
 		go func(jobFile string, wg *sync.WaitGroup) {
-			defer wg.Done()
 			defer tracker.Increment(1)
+			defer wg.Done()
 			err = ProcessGMA(pw, jobFile)
 			if err != nil {
 				pw.Log(fmt.Sprintf("\nERROR: %v\n", err))
@@ -495,6 +505,10 @@ func modeIngress(folderPath string, skipN int) {
 					panic(err)
 				}
 			}
+			err = DeleteIfSafeGMA(pw, jobFile)
+			if err != nil {
+				pw.Log(fmt.Sprintf("\nERROR: %v\n", err))
+			}
 
 		}(jobFile, &wg)
 	}
@@ -528,14 +542,14 @@ func undoBatch(undoBatch bool, gmaID string, fileIDs []string, gma2FileIDs []str
 func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
 	var unlockOnce sync.Once
 
-	GlobalWriteLock.Lock()                      // Wait for worker to have slot open
+	GlobalDeleteLock.Lock()                      // Wait for worker to have slot open
-	defer unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway
+	defer unlockOnce.Do(GlobalDeleteLock.Unlock) // release anyway
 	dboGMA := common.DB_GMA{}
 
 	dboGMA.OriginalPath = filePath
 	cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR g IN gma FILTER g.originalPath == '%s' RETURN g", dboGMA.OriginalPath), nil)
 	if err != nil {
-		return err
+		return fmt.Errorf("545: %v", err)
 	}
 
 	defer cursor.Close()
@@ -547,14 +561,17 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
 			if driver.IsNoMoreDocuments(err) {
 				break
 			} else if err != nil {
-				return err
+				return fmt.Errorf("557: %v", err)
 			}
 			if !gma.Success {
 				return fmt.Errorf("GMA with ID %s was not successfull ", gma.ID)
 			}
 		}
+		//} else {
+		//	return fmt.Errorf("no gmas found with path = %s", dboGMA.OriginalPath)
 	}
 
+	fmt.Println(dboGMA)
 	fileStat, err := os.Stat(filePath)
 	if err != nil {
 		return err
@@ -578,7 +595,7 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
 	}
 	defer gmaReader.Close()
 
-	unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway
+	unlockOnce.Do(GlobalDeleteLock.Unlock) // release anyway
 	dboGMA.GMAHash, err = gmaReader.GetSHA256()
 	if err != nil {
 		return err
@@ -590,32 +607,83 @@ func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
 	if err != nil {
 		return err
 	}
+
+	dboGMA_sameHash := common.DB_GMA{}
 	if !dboIDExists {
 		return fmt.Errorf("GMA with ID %s does not exists", dboGMA.ID)
+	} else {
+		_, err := colGMA.ReadDocument(arangoCTX, dboGMA.ID, &dboGMA_sameHash)
+		if err != nil {
+			return fmt.Errorf("609-: %v", err)
+		}
+		if dboGMA_sameHash.OriginalPath != filePath {
+			pw.Log("originalpath differs for this hash")
+			// DB_GMA_Alias
+		}
 	}
-	{
+	if dboGMA.OriginalPath == filePath {
-		recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath))
+		{
-		if checkFileExists(recoveryPath) {
+			recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath))
-			err = os.Remove(dboGMA.OriginalPath)
+			if checkFileExists(recoveryPath) {
-			if err != nil {
+				err = os.Remove(dboGMA.OriginalPath)
-				return err
+				if err != nil {
+					return err
+				}
+				pw.Log("deleted %s", dboGMA.OriginalPath)
+			} else {
+				pw.Log("%s recoveryPath does not exist %s", filePath, recoveryPath)
+			}
+			lzmaFile := fmt.Sprintf("%s.lzma", dboGMA.OriginalPath)
+			if checkFileExists(lzmaFile) {
+				err = os.Remove(lzmaFile)
+				if err != nil {
+					return err
+				}
+				pw.Log("deleted %s", lzmaFile)
 			}
-			pw.Log("deleted %s", dboGMA.OriginalPath)
 		}
-		lzmaFile := fmt.Sprintf("%s.lzma", dboGMA.OriginalPath)
+		{
-		if checkFileExists(lzmaFile) {
+			_, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, common.DB_GMADeletionData{
-			err = os.Remove(lzmaFile)
+				Deleted: true,
+			})
 			if err != nil {
 				return err
 			}
-			pw.Log("deleted %s", lzmaFile)
 		}
-	}
+	} else {
-	_, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, common.DB_GMADeletionData{
+		originalDeletionData := common.DB_GMADeletionData{}
-		Deleted: true,
+		_, err := colGMA.ReadDocument(arangoCTX, dboGMA.ID, &originalDeletionData)
-	})
+		if err != nil {
-	if err != nil {
+			return fmt.Errorf("650-: %v", err)
-		return err
+		}
+		if originalDeletionData.Deleted {
+			aliasData := common.DB_GMA_Alias{
+				Path:    filePath,
+				Hash:    dboGMA.GMAHash,
+				Deleted: true,
+			}
+			recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath))
+			if checkFileExists(recoveryPath) {
+				err = os.Remove(aliasData.Path)
+				if err != nil {
+					return err
+				}
+				pw.Log("deleted %s duplicate, original was %s", aliasData.Path, dboGMA.OriginalPath)
+			} else {
+				pw.Log("%s recoveryPath does not exist %s", filePath, recoveryPath)
+			}
+			lzmaFile := fmt.Sprintf("%s.lzma", aliasData.Path)
+			if checkFileExists(lzmaFile) {
+				err = os.Remove(lzmaFile)
+				if err != nil {
+					return err
+				}
+				pw.Log("deleted %s", lzmaFile)
+			}
+			_, _ = colAliases.CreateDocument(arangoCTX, aliasData)
+		} else {
+			return fmt.Errorf("originals have not been deleted")
+		}
 	}
 	return nil
 }