|
|
|
|
@ -5,6 +5,7 @@ import (
|
|
|
|
|
"crypto/tls"
|
|
|
|
|
"flag"
|
|
|
|
|
"fmt"
|
|
|
|
|
"io"
|
|
|
|
|
"log"
|
|
|
|
|
"net/http"
|
|
|
|
|
"os"
|
|
|
|
|
@ -18,6 +19,7 @@ import (
|
|
|
|
|
"git.cheetah.cat/worksucc/gma-puzzles/gma"
|
|
|
|
|
adriver "github.com/arangodb/go-driver"
|
|
|
|
|
ahttp "github.com/arangodb/go-driver/http"
|
|
|
|
|
"github.com/schollz/progressbar/v3"
|
|
|
|
|
"github.com/twinj/uuid"
|
|
|
|
|
|
|
|
|
|
_ "net/http/pprof"
|
|
|
|
|
@ -143,7 +145,7 @@ func main() {
|
|
|
|
|
//gma, err := gma.NewReader("2143898000.1593250551.bin.gma")
|
|
|
|
|
|
|
|
|
|
//folderPath := "/mnt/SC9000/TemporaryTestingShit2/" //"/mnt/worksucc/san1/gma/2/5/4/8/"
|
|
|
|
|
folderPathTarget := "/mnt/SC9000/ProcessedGMATest/" //"/mnt/worksucc/san1/gma/2/5/4/8/"
|
|
|
|
|
//folderPathTarget := "/mnt/SC9000/ProcessedGMATest/" //"/mnt/worksucc/san1/gma/2/5/4/8/"
|
|
|
|
|
//
|
|
|
|
|
entries, err := os.ReadDir(folderPath)
|
|
|
|
|
if err != nil {
|
|
|
|
|
@ -181,7 +183,7 @@ func main() {
|
|
|
|
|
//panic(err)
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
os.Rename(jobFile, filepath.Join(folderPathTarget, filepath.Base(jobFile)))
|
|
|
|
|
//os.Rename(jobFile, filepath.Join(folderPathTarget, filepath.Base(jobFile)))
|
|
|
|
|
|
|
|
|
|
// sem.Release() // Release the slot
|
|
|
|
|
// wg.Done() // Finish job
|
|
|
|
|
@ -247,7 +249,7 @@ func ProcessGMA(filePath string) (err error) {
|
|
|
|
|
dboGMA.ID = dboGMA.GMAHash
|
|
|
|
|
gmaReader.FileHandle.Seek(0, 0)
|
|
|
|
|
|
|
|
|
|
gmaTempPath := filepath.Join("/home/cheetah/dev/gma-puzzles/temp", dboGMA.ID)
|
|
|
|
|
gmaTempPath := filepath.Join("/mnt/ramfs/gma-extr-temp", dboGMA.ID)
|
|
|
|
|
defer os.RemoveAll(gmaTempPath) // clean up under any circumstances
|
|
|
|
|
|
|
|
|
|
dboIDExists, err := colGMA.DocumentExists(arangoCTX, dboGMA.ID)
|
|
|
|
|
@ -278,13 +280,17 @@ func ProcessGMA(filePath string) (err error) {
|
|
|
|
|
dboGMA2Files []common.DB_GMA2File
|
|
|
|
|
dboFiles []common.DB_File
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
for _, file := range files {
|
|
|
|
|
if file.FileSize < 0 { // Something is fucked
|
|
|
|
|
return fmt.Errorf("GMA Header corrupted, NextType %d, FileNumber %d", file.NextType, file.FileNumber)
|
|
|
|
|
}
|
|
|
|
|
//fmt.Printf("%s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", file.FileName, file.CRC, file.Offset, file.FileSize, file.NextType, file.FileNumber)
|
|
|
|
|
if file.NextType > uint32(file.FileNumber+10) { // Something is fucked
|
|
|
|
|
log.Printf("Current Cursor %d", gmaReader.GetOffset())
|
|
|
|
|
/*log.Printf("Current Cursor %d", gmaReader.GetOffset())
|
|
|
|
|
for _, otherFile := range files[file.FileNumber:] {
|
|
|
|
|
log.Printf("OTHERFILE %s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", otherFile.FileName, otherFile.CRC, otherFile.Offset, otherFile.FileSize, otherFile.NextType, otherFile.FileNumber)
|
|
|
|
|
}
|
|
|
|
|
}*/
|
|
|
|
|
return fmt.Errorf("GMA Header corrupted, NextType %d, FileNumber %d", file.NextType, file.FileNumber)
|
|
|
|
|
}
|
|
|
|
|
destPath := filepath.Join(gmaTempPath, "contents", file.FileName)
|
|
|
|
|
@ -357,83 +363,130 @@ func ProcessGMA(filePath string) (err error) {
|
|
|
|
|
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
/*
|
|
|
|
|
importStartTime := time.Now()
|
|
|
|
|
|
|
|
|
|
importStartTime := time.Now()
|
|
|
|
|
|
|
|
|
|
var newUnknownFiles []string
|
|
|
|
|
chunkSize := 5
|
|
|
|
|
for {
|
|
|
|
|
if len(dboFiles) == 0 {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
var newUnknownFiles []string
|
|
|
|
|
chunkSize := 5
|
|
|
|
|
for {
|
|
|
|
|
if len(dboFiles) == 0 {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// necessary check to avoid slicing beyond
|
|
|
|
|
// slice capacity
|
|
|
|
|
if len(dboFiles) < chunkSize {
|
|
|
|
|
chunkSize = len(dboFiles)
|
|
|
|
|
}
|
|
|
|
|
// necessary check to avoid slicing beyond
|
|
|
|
|
// slice capacity
|
|
|
|
|
if len(dboFiles) < chunkSize {
|
|
|
|
|
chunkSize = len(dboFiles)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// process and work withj
|
|
|
|
|
metaSlice, errorSlice, _ := colFile.CreateDocuments(arangoCTX, dboFiles[0:chunkSize])
|
|
|
|
|
// process and work withj
|
|
|
|
|
metaSlice, errorSlice, _ := colFile.CreateDocuments(arangoCTX, dboFiles[0:chunkSize])
|
|
|
|
|
|
|
|
|
|
for _, meta := range metaSlice {
|
|
|
|
|
if !meta.ID.IsEmpty() {
|
|
|
|
|
newUnknownFiles = append(newUnknownFiles, meta.Key)
|
|
|
|
|
fileIDs = append(fileIDs, meta.Key)
|
|
|
|
|
for _, meta := range metaSlice {
|
|
|
|
|
if !meta.ID.IsEmpty() {
|
|
|
|
|
newUnknownFiles = append(newUnknownFiles, meta.Key)
|
|
|
|
|
fileIDs = append(fileIDs, meta.Key)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
//fmt.Println("ErrorSlice")
|
|
|
|
|
//fmt.Println(errorSlice)
|
|
|
|
|
for _, createError := range errorSlice {
|
|
|
|
|
if createError != nil && strings.Contains(createError.Error(), "unique constraint violated - in index primary of type primary over '_key'") {
|
|
|
|
|
} else if createError != nil {
|
|
|
|
|
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
|
|
|
|
return createError
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
dboFiles = dboFiles[chunkSize:]
|
|
|
|
|
}
|
|
|
|
|
//fmt.Println("ErrorSlice")
|
|
|
|
|
//fmt.Println(errorSlice)
|
|
|
|
|
for _, createError := range errorSlice {
|
|
|
|
|
if createError != nil && strings.Contains(createError.Error(), "unique constraint violated - in index primary of type primary over '_key'") {
|
|
|
|
|
} else if createError != nil {
|
|
|
|
|
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
|
|
|
|
return createError
|
|
|
|
|
log.Println()
|
|
|
|
|
log.Printf("Imported dboFiles into Arango and now we have %d new files from %d addon files\n", len(newUnknownFiles), len(files))
|
|
|
|
|
deltaFileSize := int64(0)
|
|
|
|
|
for _, unknownFile := range newUnknownFiles {
|
|
|
|
|
unknownFileID := fmt.Sprintf("file/%s", unknownFile)
|
|
|
|
|
for _, dboGMA2File := range dboGMA2Files {
|
|
|
|
|
if unknownFileID == dboGMA2File.File {
|
|
|
|
|
deltaFileSize += dboGMA2File.FileSize
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
dboGMA.OptimizedSize = deltaFileSize
|
|
|
|
|
log.Printf("Delta Storage %d bytes\n", deltaFileSize)
|
|
|
|
|
*/
|
|
|
|
|
/*
|
|
|
|
|
_, err = colGMA2File.ImportDocuments(arangoCTX, dboGMA2Files, &adriver.ImportDocumentOptions{
|
|
|
|
|
OnDuplicate: adriver.ImportOnDuplicateIgnore,
|
|
|
|
|
//FromPrefix: "gma/",
|
|
|
|
|
//ToPrefix: "file/",
|
|
|
|
|
Complete: true, // will make it fail if any error occurs (and hopefully reverse the trans-action)
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
|
|
|
|
return fmt.Errorf("ImportDocuments File fail: %v", err)
|
|
|
|
|
}
|
|
|
|
|
//fmt.Printf("Code: %d, Created: %d, Ignored: %d, Errors: %d", statsImportGMA2File.Code, statsImportGMA2File.Created, statsImportGMA2File.Ignored, statsImportGMA2File.Errors)
|
|
|
|
|
|
|
|
|
|
dboFiles = dboFiles[chunkSize:]
|
|
|
|
|
}
|
|
|
|
|
log.Println()
|
|
|
|
|
log.Printf("Import Duration %dms\n", time.Since(importStartTime).Milliseconds())
|
|
|
|
|
log.Println()
|
|
|
|
|
*/
|
|
|
|
|
// TODO: Check all dboFiles and dboGMA2Files if they exist, if something is odd, queue reupload
|
|
|
|
|
|
|
|
|
|
log.Printf("Imported dboFiles into Arango and now we have %d new files from %d addon files\n", len(newUnknownFiles), len(files))
|
|
|
|
|
deltaFileSize := int64(0)
|
|
|
|
|
for _, unknownFile := range newUnknownFiles {
|
|
|
|
|
unknownFileID := fmt.Sprintf("file/%s", unknownFile)
|
|
|
|
|
for _, dboGMA2File := range dboGMA2Files {
|
|
|
|
|
if unknownFileID == dboGMA2File.File {
|
|
|
|
|
deltaFileSize += dboGMA2File.FileSize
|
|
|
|
|
}
|
|
|
|
|
dboExistFile := map[string]bool{}
|
|
|
|
|
dboExistFile2GMA := map[string]bool{}
|
|
|
|
|
for _, dboFile := range dboFiles {
|
|
|
|
|
exists, err := colFile.DocumentExists(arangoCTX, dboFile.ID)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
dboExistFile[dboFile.ID] = exists
|
|
|
|
|
}
|
|
|
|
|
dboGMA.OptimizedSize = deltaFileSize
|
|
|
|
|
log.Printf("Delta Storage %d bytes\n", deltaFileSize)
|
|
|
|
|
|
|
|
|
|
_, err = colGMA2File.ImportDocuments(arangoCTX, dboGMA2Files, &adriver.ImportDocumentOptions{
|
|
|
|
|
OnDuplicate: adriver.ImportOnDuplicateIgnore,
|
|
|
|
|
//FromPrefix: "gma/",
|
|
|
|
|
//ToPrefix: "file/",
|
|
|
|
|
Complete: true, // will make it fail if any error occurs (and hopefully reverse the trans-action)
|
|
|
|
|
})
|
|
|
|
|
if err != nil {
|
|
|
|
|
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
|
|
|
|
return fmt.Errorf("ImportDocuments File fail: %v", err)
|
|
|
|
|
for _, dboGMA2File := range dboGMA2Files {
|
|
|
|
|
exists, err := colGMA2File.DocumentExists(arangoCTX, dboGMA2File.ID)
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
dboExistFile2GMA[dboGMA2File.ID] = exists
|
|
|
|
|
}
|
|
|
|
|
//fmt.Printf("Code: %d, Created: %d, Ignored: %d, Errors: %d", statsImportGMA2File.Code, statsImportGMA2File.Created, statsImportGMA2File.Ignored, statsImportGMA2File.Errors)
|
|
|
|
|
|
|
|
|
|
log.Printf("Import Duration %dms\n", time.Since(importStartTime).Milliseconds())
|
|
|
|
|
log.Println()
|
|
|
|
|
// TODO: upload all unknownNewFiles to StorageServer
|
|
|
|
|
http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 200
|
|
|
|
|
var httpClient *http.Client = http.DefaultClient
|
|
|
|
|
uploadBar := progressbar.Default(int64(len(dboFiles)), "Uploading to StorageServer")
|
|
|
|
|
for _, dboFile := range dboFiles {
|
|
|
|
|
dboFileID := fmt.Sprintf("file/%s", dboFile.ID)
|
|
|
|
|
//dboFile2ChunkID := fmt.Sprintf("file_chunk_map/%s", dboFile.ID)
|
|
|
|
|
|
|
|
|
|
// TODO: Check against Storage backend
|
|
|
|
|
res, err := http.Get(fmt.Sprintf("http://127.0.0.1:13371/check/%s", dboFile.ID))
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
defer res.Body.Close()
|
|
|
|
|
if _, err = io.Copy(io.Discard, res.Body); err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
//body, _ := ioutil.ReadAll(res.Body)
|
|
|
|
|
if res.StatusCode == http.StatusAlreadyReported {
|
|
|
|
|
uploadBar.Add(1)
|
|
|
|
|
uploadBar.Describe("Skipping")
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for _, unknownFile := range newUnknownFiles {
|
|
|
|
|
unknownFileID := fmt.Sprintf("file/%s", unknownFile)
|
|
|
|
|
for _, dboGMA2File := range dboGMA2Files {
|
|
|
|
|
if unknownFileID == dboGMA2File.File {
|
|
|
|
|
if dboFileID == dboGMA2File.File { // find corresponding dboGMA2File
|
|
|
|
|
uploadSuccess := true
|
|
|
|
|
for {
|
|
|
|
|
log.Printf("Uploading %s to Storage\n", dboGMA2File.UploadID)
|
|
|
|
|
//log.Printf("Uploading %s to Storage\n", dboGMA2File.UploadID)
|
|
|
|
|
if !dboExistFile[dboFile.ID] {
|
|
|
|
|
_, err := colFile.CreateDocument(arangoCTX, dboFile)
|
|
|
|
|
if err != nil {
|
|
|
|
|
// TODO: error handling
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
uploadBar.Describe("Uploading")
|
|
|
|
|
err = common.MultipartUpload(httpClient, fmt.Sprintf("http://127.0.0.1:13371/stash/%s/%d", dboGMA2File.UploadID, dboGMA2File.FileSize), dboGMA2File.LocalFileName)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Println(err)
|
|
|
|
|
@ -441,11 +494,41 @@ func ProcessGMA(filePath string) (err error) {
|
|
|
|
|
uploadSuccess = false
|
|
|
|
|
} else {
|
|
|
|
|
log.Println("oopsie")
|
|
|
|
|
// remove fileObject
|
|
|
|
|
if !dboExistFile[dboFile.ID] {
|
|
|
|
|
_, _ = colFile.RemoveDocument(arangoCTX, dboFile.ID)
|
|
|
|
|
}
|
|
|
|
|
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if uploadSuccess {
|
|
|
|
|
// Create File and dboGMA2File Object
|
|
|
|
|
if !dboExistFile2GMA[dboGMA2File.ID] {
|
|
|
|
|
exists, err := colGMA2File.DocumentExists(arangoCTX, dboGMA2File.ID)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Println("oopsie")
|
|
|
|
|
// remove fileObject
|
|
|
|
|
if !dboExistFile[dboFile.ID] { // if the file did not exist prior to this
|
|
|
|
|
_, _ = colFile.RemoveDocument(arangoCTX, dboFile.ID)
|
|
|
|
|
}
|
|
|
|
|
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
if !exists {
|
|
|
|
|
_, err = colGMA2File.CreateDocument(arangoCTX, dboGMA2File)
|
|
|
|
|
if err != nil {
|
|
|
|
|
log.Println("oopsie")
|
|
|
|
|
// remove fileObject
|
|
|
|
|
if !dboExistFile[dboFile.ID] { // if the file did not exist prior to this
|
|
|
|
|
_, _ = colFile.RemoveDocument(arangoCTX, dboFile.ID)
|
|
|
|
|
}
|
|
|
|
|
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
uploadBar.Add(1)
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
time.Sleep(10 * time.Second)
|
|
|
|
|
@ -458,6 +541,7 @@ func ProcessGMA(filePath string) (err error) {
|
|
|
|
|
// TODO : compare hashes
|
|
|
|
|
{
|
|
|
|
|
log.Println("rewriting gma")
|
|
|
|
|
rewriteBar := progressbar.Default(int64(len(dboGMA2Files)), "Rewriting GMA")
|
|
|
|
|
destPath := filepath.Join(gmaTempPath, "rewrite.gma")
|
|
|
|
|
dir := filepath.Dir(destPath)
|
|
|
|
|
|
|
|
|
|
@ -512,6 +596,7 @@ func ProcessGMA(filePath string) (err error) {
|
|
|
|
|
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
rewriteBar.Add(1)
|
|
|
|
|
}
|
|
|
|
|
gmaWriter.FileHandle.Seek(0, 2)
|
|
|
|
|
log.Printf("Writing Footer CRC %d\n\n", dboGMA.FooterAddonCRC)
|
|
|
|
|
|
