added rebuild feature

main.go

@@ -118,46 +118,163 @@ var WorkerJobPool chan string
 var GlobalWriteLock sync.Mutex
 
 func main() {
+	workerModeP := flag.String("mode", "", "mode (ingress, rebuild)")
+	debugEnabled := flag.Bool("debug", false, "enables debug")
 	folderPathP := flag.String("path", "/mnt/SC9000/TemporaryTestingShit2", "a string")
 	skipNameP := flag.String("skip", "", "skip until this name")
 	workerNameP := flag.String("worker", "def", "worker name")
+	rebuildIDP := flag.String("id", "", "id to rebuild")
 
-	debug.SetMemoryLimit(6e9)
 	flag.Parse()
-	folderPath := *folderPathP
+
-	skipName := *skipNameP
+	workerMode := *workerModeP
-	skipNameEnabled := len(skipName) > 0
+
-	workerID = *workerNameP
+	if *debugEnabled {
-	go func() {
+		debug.SetMemoryLimit(6e9)
-		log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+		go func() {
-	}()
+			log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+		}()
+	}
 
 	err := InitDatabase()
 	if err != nil {
 		panic(err)
 	}
 
-	/*
+	switch workerMode {
-		err = colGMA.Truncate(arangoCTX)
+	case "ingress":
+		/*
+			folderPathP := flag.String("path", "/mnt/SC9000/TemporaryTestingShit2", "a string")
+			skipNameP := flag.String("skip", "", "skip until this name")
+			workerNameP := flag.String("worker", "def", "worker name")
+			flag.Parse()
+		*/
+		workerID = *workerNameP
+		modeIngress(*folderPathP, *skipNameP)
+	case "rebuild":
+		//		rebuildIDP := flag.String("id", "", "id to rebuild")
+		flag.Parse()
+		modeRebuild(*rebuildIDP)
+	}
+}
+func modeRebuild(id string) (err error) {
+	var (
+		dboGMA       common.DB_GMA
+		dboGMA2Files []common.DB_GMA2File
+	)
+
+	_, err = colGMA.ReadDocument(arangoCTX, id, &dboGMA)
+	if err != nil {
+		return err
+	}
+
+	cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR gf IN gma_file_map FILTER gf._from == 'gma/%s' RETURN gf", dboGMA.ID), nil)
+	if err != nil {
+		return err
+	}
+
+	defer cursor.Close()
+	if cursor.Count() > 0 || cursor.HasMore() {
+		for {
+			gma2File := common.DB_GMA2File{}
+			_, err = cursor.ReadDocument(arangoCTX, &gma2File)
+			if driver.IsNoMoreDocuments(err) {
+				break
+			} else if err != nil {
+				return err
+			}
+			gma2File.UploadID = gma2File.File[5:]
+			dboGMA2Files = append(dboGMA2Files, gma2File)
+		}
+	} else {
+		return fmt.Errorf("no files for gma available")
+	}
+
+	{
+		log.Println("rewriting gma")
+		rewriteBar := progressbar.Default(int64(len(dboGMA2Files)), "Rewriting GMA")
+		destPath := fmt.Sprintf("./%s.gma", id) //filepath.Join(gmaTempPath, "rewrite.gma")
+		dir := filepath.Dir(destPath)
+
+		err := os.MkdirAll(dir, os.ModePerm)
 		if err != nil {
-			panic(err)
+			return err
 		}
-		err = colFile.Truncate(arangoCTX)
+
+		gmaWriter, err := gma.NewWriter(destPath)
 		if err != nil {
-			panic(err)
+			return err
+		}
+		defer gmaWriter.Close()
+
+		//fmt.Printf("Writing Header with FormatVersion: %d\n", dboGMA.Header.FormatVersion)
+		err = gmaWriter.WriteHeader(dboGMA.Header)
+		if err != nil {
+			return err
+		}
+		err = gmaWriter.WriteFirstType(dboGMA.FirstType)
+		if err != nil {
+			return err
+		}
+
+		sort.SliceStable(dboGMA2Files, func(i, j int) bool { return dboGMA2Files[i].FileNumber < dboGMA2Files[j].FileNumber })
+		for _, dboGMA2File := range dboGMA2Files {
+			//fmt.Printf("WriteFileIndex for %s number %d\n", dboGMA2File.FileName, dboGMA2File.FileNumber)
+			err = gmaWriter.WriteFileIndex(dboGMA2File.FileName, dboGMA2File.FileSize, dboGMA2File.CRC, dboGMA2File.NextType)
+			if err != nil {
+				return err
+			}
+		}
+		var httpClient *http.Client = &http.Client{
+			Timeout: 15 * time.Minute,
 		}
-		err = colGMA2File.Truncate(arangoCTX)
+
+		for _, dboGMA2File := range dboGMA2Files {
+			//fmt.Printf("WriteFile for %s number %d = %s\n", dboGMA2File.FileName, dboGMA2File.FileNumber, dboGMA2File.UploadID)
+			resp, err := httpClient.Get(fmt.Sprintf("http://127.0.0.1:13371/fetch/%s", dboGMA2File.UploadID))
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+			err = gmaWriter.WriteFile(resp.Body)
+			if err != nil {
+				return err
+			}
+			rewriteBar.Add(1)
+		}
+		gmaWriter.FileHandle.Seek(0, 2)
+		log.Printf("Writing Footer CRC %d\n\n", dboGMA.FooterAddonCRC)
+		gmaWriter.WriteFooterCRC(dboGMA.FooterAddonCRC)
+
+		// TODO: maybe use io.MultiWriter ??
+		gmaWriter.FileHandle.Seek(0, 0)
+		writeHash, err := gmaWriter.GetSHA256()
 		if err != nil {
-			panic(err)
+			return err
-		}*/
+		}
 
-	// /mnt/worksucc/san2/gma/2/1/2/3/2123406190.1591573904.gma
+		log.Printf("Rewrite Hash is %s %s\n", writeHash, destPath)
-	//fileHandle, err := os.Open("2143898000.1593250551.bin.gma") //2143898000.1593250551.bin")
+		log.Printf("Original Hash is %s %s\n", dboGMA.GMAHash, dboGMA.OriginalPath)
-	//gma, err := gma.NewReader("2143898000.1593250551.bin.gma")
+		log.Println()
+		writeStat, err := os.Stat(destPath)
+		if err != nil {
+			return err
+		}
+		writeSize := writeStat.Size()
+		if writeSize != dboGMA.GMASize {
+			//fail
+			return fmt.Errorf("RewriteCheck failed, original=%s (%d bytes), rewrite=%s (%d bytes)", dboGMA.GMAHash, dboGMA.GMASize, writeHash, writeSize)
+		}
+		if writeHash != dboGMA.GMAHash {
+			//fail
+			return fmt.Errorf("RewriteCheck failed, original=%s (%d bytes), rewrite=%s (%d bytes)", dboGMA.GMAHash, dboGMA.GMASize, writeHash, writeSize)
+		}
+	}
 
-	//folderPath := "/mnt/SC9000/TemporaryTestingShit2/"  //"/mnt/worksucc/san1/gma/2/5/4/8/"
+	return nil
-	//folderPathTarget := "/mnt/SC9000/ProcessedGMATest/" //"/mnt/worksucc/san1/gma/2/5/4/8/"
+}
-	//
+func modeIngress(folderPath string, skipName string) {
+	skipNameEnabled := len(skipName) > 0
 	entries, err := os.ReadDir(folderPath)
 	if err != nil {
 		panic(err)
@@ -192,9 +309,7 @@ func main() {
 
 	// Wait for all jobs to finish
 	wg.Wait()
-
 }
-
 func undoBatch(undoBatch bool, gmaID string, fileIDs []string, gma2FileIDs []string) (err error) {
 	log.Printf("undoBatch(%v, %s)\n", undoBatch, gmaID)
 	/*