worksucc/gma-puzzles
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fmt. to log.

Browse source
This commit is contained in:
cheetah 2023-06-25 03:36:21 -05:00
parent 0f12007588
commit 9976d6d3c0
1 changed files with 34 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
main.go
View file

@ -177,7 +177,7 @@ func main() {
err = ProcessGMA(jobFile) err = ProcessGMA(jobFile)
if err != nil { if err != nil {
fmt.Printf("\nERROR: %v\n", err) log.Printf("\nERROR: %v\n", err)
//panic(err) //panic(err)
continue continue
} }
@ -194,15 +194,19 @@ func main() {
} }
func undoBatch(undoBatch bool, gmaID string, fileIDs []string, gma2FileIDs []string) (err error) { func undoBatch(undoBatch bool, gmaID string, fileIDs []string, gma2FileIDs []string) (err error) {
fmt.Printf("undoBatch(%x, %s)\n", undoBatch, gmaID) log.Printf("undoBatch(%x, %s)\n", undoBatch, gmaID)
_, err = colGMA.RemoveDocument(arangoCTX, gmaID) /*
if err != nil { _, err = colGMA.RemoveDocument(arangoCTX, gmaID)
return err if err != nil {
} return err
_, _, err = colFile.RemoveDocuments(arangoCTX, fileIDs) }
if err != nil { */
return err /*
} _, _, err = colFile.RemoveDocuments(arangoCTX, fileIDs)
if err != nil {
return err
}
*/
_, _, err = colGMA2File.RemoveDocuments(arangoCTX, gma2FileIDs) _, _, err = colGMA2File.RemoveDocuments(arangoCTX, gma2FileIDs)
if err != nil { if err != nil {
return err return err
@ -229,7 +233,7 @@ func ProcessGMA(filePath string) (err error) {
if dboGMA.GMASize < 200 { if dboGMA.GMASize < 200 {
return fmt.Errorf("GMA File too small, skipping") return fmt.Errorf("GMA File too small, skipping")
} }
fmt.Printf("Opening %s\n", filePath) log.Printf("Opening %s\n", filePath)
gmaReader, err := gma.NewReader(filePath) gmaReader, err := gma.NewReader(filePath)
if err != nil { if err != nil {
return err return err
@ -259,9 +263,9 @@ func ProcessGMA(filePath string) (err error) {
return err return err
} }
dboGMA.Header = header dboGMA.Header = header
fmt.Printf("AddonVersion=%d\n", header.AddonVersion) log.Printf("AddonVersion=%d\n", header.AddonVersion)
fmt.Printf("FormatVersion=%d\n", header.FormatVersion) log.Printf("FormatVersion=%d\n", header.FormatVersion)
fmt.Printf("FormatVersionDiscardByte=%d\n", header.FormatVersionDiscardByte) log.Printf("FormatVersionDiscardByte=%d\n", header.FormatVersionDiscardByte)
//fmt.Printf("r.cursorOffset = %d\n", gmaReader.GetOffset()) //fmt.Printf("r.cursorOffset = %d\n", gmaReader.GetOffset())
firstType, files, err := gmaReader.ReadFiles() firstType, files, err := gmaReader.ReadFiles()
if err != nil { if err != nil {
@ -277,9 +281,9 @@ func ProcessGMA(filePath string) (err error) {
for _, file := range files { for _, file := range files {
//fmt.Printf("%s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", file.FileName, file.CRC, file.Offset, file.FileSize, file.NextType, file.FileNumber) //fmt.Printf("%s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", file.FileName, file.CRC, file.Offset, file.FileSize, file.NextType, file.FileNumber)
if file.NextType > uint32(file.FileNumber+10) { // Something is fucked if file.NextType > uint32(file.FileNumber+10) { // Something is fucked
fmt.Printf("Current Cursor %d", gmaReader.GetOffset()) log.Printf("Current Cursor %d", gmaReader.GetOffset())
for _, otherFile := range files[file.FileNumber:] { for _, otherFile := range files[file.FileNumber:] {
fmt.Printf("OTHERFILE %s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", otherFile.FileName, otherFile.CRC, otherFile.Offset, otherFile.FileSize, otherFile.NextType, otherFile.FileNumber) log.Printf("OTHERFILE %s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", otherFile.FileName, otherFile.CRC, otherFile.Offset, otherFile.FileSize, otherFile.NextType, otherFile.FileNumber)
} }
return fmt.Errorf("GMA Header corrupted, NextType %d, FileNumber %d", file.NextType, file.FileNumber) return fmt.Errorf("GMA Header corrupted, NextType %d, FileNumber %d", file.NextType, file.FileNumber)
} }
@ -302,7 +306,7 @@ func ProcessGMA(filePath string) (err error) {
} }
if extractMeta.ExtractedCRC != extractMeta.OriginalMeta.CRC { if extractMeta.ExtractedCRC != extractMeta.OriginalMeta.CRC {
fmt.Printf("gma(%s) checksum in meta (%d) differs from read (%d) [%s]\n", filePath, extractMeta.OriginalMeta.CRC, extractMeta.ExtractedCRC, extractMeta.OriginalMeta.FileName) log.Printf("gma(%s) checksum in meta (%d) differs from read (%d) [%s]\n", filePath, extractMeta.OriginalMeta.CRC, extractMeta.ExtractedCRC, extractMeta.OriginalMeta.FileName)
} }
//fmt.Printf("ExtractedMeta %s CRC: %d SHA256: %s\n", file.FileName, extractMeta.ExtractedCRC, extractMeta.ExtractedSHA256) //fmt.Printf("ExtractedMeta %s CRC: %d SHA256: %s\n", file.FileName, extractMeta.ExtractedCRC, extractMeta.ExtractedSHA256)
dboFile := common.DB_File{ dboFile := common.DB_File{
@ -372,8 +376,6 @@ func ProcessGMA(filePath string) (err error) {
// process and work withj // process and work withj
metaSlice, errorSlice, _ := colFile.CreateDocuments(arangoCTX, dboFiles[0:chunkSize]) metaSlice, errorSlice, _ := colFile.CreateDocuments(arangoCTX, dboFiles[0:chunkSize])
fmt.Println("Metaslice")
fmt.Println(metaSlice)
for _, meta := range metaSlice { for _, meta := range metaSlice {
if !meta.ID.IsEmpty() { if !meta.ID.IsEmpty() {
newUnknownFiles = append(newUnknownFiles, meta.Key) newUnknownFiles = append(newUnknownFiles, meta.Key)
@ -392,9 +394,9 @@ func ProcessGMA(filePath string) (err error) {
dboFiles = dboFiles[chunkSize:] dboFiles = dboFiles[chunkSize:]
} }
fmt.Println() log.Println()
fmt.Printf("Imported dboFiles into Arango and now we have %d new files from %d addon files\n", len(newUnknownFiles), len(files)) log.Printf("Imported dboFiles into Arango and now we have %d new files from %d addon files\n", len(newUnknownFiles), len(files))
deltaFileSize := int64(0) deltaFileSize := int64(0)
for _, unknownFile := range newUnknownFiles { for _, unknownFile := range newUnknownFiles {
unknownFileID := fmt.Sprintf("file/%s", unknownFile) unknownFileID := fmt.Sprintf("file/%s", unknownFile)
@ -405,7 +407,7 @@ func ProcessGMA(filePath string) (err error) {
} }
} }
dboGMA.OptimizedSize = deltaFileSize dboGMA.OptimizedSize = deltaFileSize
fmt.Printf("Delta Storage %d bytes\n", deltaFileSize) log.Printf("Delta Storage %d bytes\n", deltaFileSize)
_, err = colGMA2File.ImportDocuments(arangoCTX, dboGMA2Files, &adriver.ImportDocumentOptions{ _, err = colGMA2File.ImportDocuments(arangoCTX, dboGMA2Files, &adriver.ImportDocumentOptions{
OnDuplicate: adriver.ImportOnDuplicateIgnore, OnDuplicate: adriver.ImportOnDuplicateIgnore,
@ -419,8 +421,8 @@ func ProcessGMA(filePath string) (err error) {
} }
//fmt.Printf("Code: %d, Created: %d, Ignored: %d, Errors: %d", statsImportGMA2File.Code, statsImportGMA2File.Created, statsImportGMA2File.Ignored, statsImportGMA2File.Errors) //fmt.Printf("Code: %d, Created: %d, Ignored: %d, Errors: %d", statsImportGMA2File.Code, statsImportGMA2File.Created, statsImportGMA2File.Ignored, statsImportGMA2File.Errors)
fmt.Printf("Import Duration %dms\n", time.Since(importStartTime).Milliseconds()) log.Printf("Import Duration %dms\n", time.Since(importStartTime).Milliseconds())
fmt.Println() log.Println()
// TODO: upload all unknownNewFiles to StorageServer // TODO: upload all unknownNewFiles to StorageServer
http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 200 http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 200
var httpClient *http.Client = http.DefaultClient var httpClient *http.Client = http.DefaultClient
@ -431,13 +433,14 @@ func ProcessGMA(filePath string) (err error) {
if unknownFileID == dboGMA2File.File { if unknownFileID == dboGMA2File.File {
uploadSuccess := true uploadSuccess := true
for { for {
fmt.Printf("Uploading %s (local %s) to Storage\n", dboGMA2File.UploadID, dboGMA2File.LocalFileName) log.Printf("Uploading %s to Storage\n", dboGMA2File.UploadID)
err = common.MultipartUpload(httpClient, fmt.Sprintf("http://127.0.0.1:13371/stash/%s/%d", dboGMA2File.UploadID, dboGMA2File.FileSize), dboGMA2File.LocalFileName) err = common.MultipartUpload(httpClient, fmt.Sprintf("http://127.0.0.1:13371/stash/%s/%d", dboGMA2File.UploadID, dboGMA2File.FileSize), dboGMA2File.LocalFileName)
if err != nil { if err != nil {
log.Println(err)
if strings.Contains(err.Error(), "cannot assign requested address") { if strings.Contains(err.Error(), "cannot assign requested address") {
uploadSuccess = false uploadSuccess = false
} else { } else {
fmt.Println("oopsie") log.Println("oopsie")
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs) undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
return err return err
} }
@ -454,7 +457,7 @@ func ProcessGMA(filePath string) (err error) {
// TODO : write new gma from arangoinfo // TODO : write new gma from arangoinfo
// TODO : compare hashes // TODO : compare hashes
{ {
fmt.Println("rewriting gma") log.Println("rewriting gma")
destPath := filepath.Join(gmaTempPath, "rewrite.gma") destPath := filepath.Join(gmaTempPath, "rewrite.gma")
dir := filepath.Dir(destPath) dir := filepath.Dir(destPath)
@ -511,7 +514,7 @@ func ProcessGMA(filePath string) (err error) {
} }
} }
gmaWriter.FileHandle.Seek(0, 2) gmaWriter.FileHandle.Seek(0, 2)
fmt.Printf("Writing Footer CRC %d\n\n", dboGMA.FooterAddonCRC) log.Printf("Writing Footer CRC %d\n\n", dboGMA.FooterAddonCRC)
gmaWriter.WriteFooterCRC(dboGMA.FooterAddonCRC) gmaWriter.WriteFooterCRC(dboGMA.FooterAddonCRC)
gmaWriter.FileHandle.Seek(0, 0) gmaWriter.FileHandle.Seek(0, 0)
@ -521,9 +524,9 @@ func ProcessGMA(filePath string) (err error) {
return err return err
} }
fmt.Printf("Rewrite Hash is %s %s\n", writeHash, destPath) log.Printf("Rewrite Hash is %s %s\n", writeHash, destPath)
fmt.Printf("Original Hash is %s %s\n", dboGMA.GMAHash, dboGMA.OriginalPath) log.Printf("Original Hash is %s %s\n", dboGMA.GMAHash, dboGMA.OriginalPath)
fmt.Println() log.Println()
writeStat, err := os.Stat(destPath) writeStat, err := os.Stat(destPath)
if err != nil { if err != nil {
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs) undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)