fmt. to log.
This commit is contained in:
parent
0f12007588
commit
9976d6d3c0
1 changed files with 34 additions and 31 deletions
65
main.go
65
main.go
|
|
@ -177,7 +177,7 @@ func main() {
|
|||
|
||||
err = ProcessGMA(jobFile)
|
||||
if err != nil {
|
||||
fmt.Printf("\nERROR: %v\n", err)
|
||||
log.Printf("\nERROR: %v\n", err)
|
||||
//panic(err)
|
||||
continue
|
||||
}
|
||||
|
|
@ -194,15 +194,19 @@ func main() {
|
|||
}
|
||||
|
||||
func undoBatch(undoBatch bool, gmaID string, fileIDs []string, gma2FileIDs []string) (err error) {
|
||||
fmt.Printf("undoBatch(%x, %s)\n", undoBatch, gmaID)
|
||||
_, err = colGMA.RemoveDocument(arangoCTX, gmaID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, _, err = colFile.RemoveDocuments(arangoCTX, fileIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("undoBatch(%x, %s)\n", undoBatch, gmaID)
|
||||
/*
|
||||
_, err = colGMA.RemoveDocument(arangoCTX, gmaID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*/
|
||||
/*
|
||||
_, _, err = colFile.RemoveDocuments(arangoCTX, fileIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*/
|
||||
_, _, err = colGMA2File.RemoveDocuments(arangoCTX, gma2FileIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -229,7 +233,7 @@ func ProcessGMA(filePath string) (err error) {
|
|||
if dboGMA.GMASize < 200 {
|
||||
return fmt.Errorf("GMA File too small, skipping")
|
||||
}
|
||||
fmt.Printf("Opening %s\n", filePath)
|
||||
log.Printf("Opening %s\n", filePath)
|
||||
gmaReader, err := gma.NewReader(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -259,9 +263,9 @@ func ProcessGMA(filePath string) (err error) {
|
|||
return err
|
||||
}
|
||||
dboGMA.Header = header
|
||||
fmt.Printf("AddonVersion=%d\n", header.AddonVersion)
|
||||
fmt.Printf("FormatVersion=%d\n", header.FormatVersion)
|
||||
fmt.Printf("FormatVersionDiscardByte=%d\n", header.FormatVersionDiscardByte)
|
||||
log.Printf("AddonVersion=%d\n", header.AddonVersion)
|
||||
log.Printf("FormatVersion=%d\n", header.FormatVersion)
|
||||
log.Printf("FormatVersionDiscardByte=%d\n", header.FormatVersionDiscardByte)
|
||||
//fmt.Printf("r.cursorOffset = %d\n", gmaReader.GetOffset())
|
||||
firstType, files, err := gmaReader.ReadFiles()
|
||||
if err != nil {
|
||||
|
|
@ -277,9 +281,9 @@ func ProcessGMA(filePath string) (err error) {
|
|||
for _, file := range files {
|
||||
//fmt.Printf("%s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", file.FileName, file.CRC, file.Offset, file.FileSize, file.NextType, file.FileNumber)
|
||||
if file.NextType > uint32(file.FileNumber+10) { // Something is fucked
|
||||
fmt.Printf("Current Cursor %d", gmaReader.GetOffset())
|
||||
log.Printf("Current Cursor %d", gmaReader.GetOffset())
|
||||
for _, otherFile := range files[file.FileNumber:] {
|
||||
fmt.Printf("OTHERFILE %s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", otherFile.FileName, otherFile.CRC, otherFile.Offset, otherFile.FileSize, otherFile.NextType, otherFile.FileNumber)
|
||||
log.Printf("OTHERFILE %s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", otherFile.FileName, otherFile.CRC, otherFile.Offset, otherFile.FileSize, otherFile.NextType, otherFile.FileNumber)
|
||||
}
|
||||
return fmt.Errorf("GMA Header corrupted, NextType %d, FileNumber %d", file.NextType, file.FileNumber)
|
||||
}
|
||||
|
|
@ -302,7 +306,7 @@ func ProcessGMA(filePath string) (err error) {
|
|||
}
|
||||
|
||||
if extractMeta.ExtractedCRC != extractMeta.OriginalMeta.CRC {
|
||||
fmt.Printf("gma(%s) checksum in meta (%d) differs from read (%d) [%s]\n", filePath, extractMeta.OriginalMeta.CRC, extractMeta.ExtractedCRC, extractMeta.OriginalMeta.FileName)
|
||||
log.Printf("gma(%s) checksum in meta (%d) differs from read (%d) [%s]\n", filePath, extractMeta.OriginalMeta.CRC, extractMeta.ExtractedCRC, extractMeta.OriginalMeta.FileName)
|
||||
}
|
||||
//fmt.Printf("ExtractedMeta %s CRC: %d SHA256: %s\n", file.FileName, extractMeta.ExtractedCRC, extractMeta.ExtractedSHA256)
|
||||
dboFile := common.DB_File{
|
||||
|
|
@ -372,8 +376,6 @@ func ProcessGMA(filePath string) (err error) {
|
|||
// process and work withj
|
||||
metaSlice, errorSlice, _ := colFile.CreateDocuments(arangoCTX, dboFiles[0:chunkSize])
|
||||
|
||||
fmt.Println("Metaslice")
|
||||
fmt.Println(metaSlice)
|
||||
for _, meta := range metaSlice {
|
||||
if !meta.ID.IsEmpty() {
|
||||
newUnknownFiles = append(newUnknownFiles, meta.Key)
|
||||
|
|
@ -392,9 +394,9 @@ func ProcessGMA(filePath string) (err error) {
|
|||
|
||||
dboFiles = dboFiles[chunkSize:]
|
||||
}
|
||||
fmt.Println()
|
||||
log.Println()
|
||||
|
||||
fmt.Printf("Imported dboFiles into Arango and now we have %d new files from %d addon files\n", len(newUnknownFiles), len(files))
|
||||
log.Printf("Imported dboFiles into Arango and now we have %d new files from %d addon files\n", len(newUnknownFiles), len(files))
|
||||
deltaFileSize := int64(0)
|
||||
for _, unknownFile := range newUnknownFiles {
|
||||
unknownFileID := fmt.Sprintf("file/%s", unknownFile)
|
||||
|
|
@ -405,7 +407,7 @@ func ProcessGMA(filePath string) (err error) {
|
|||
}
|
||||
}
|
||||
dboGMA.OptimizedSize = deltaFileSize
|
||||
fmt.Printf("Delta Storage %d bytes\n", deltaFileSize)
|
||||
log.Printf("Delta Storage %d bytes\n", deltaFileSize)
|
||||
|
||||
_, err = colGMA2File.ImportDocuments(arangoCTX, dboGMA2Files, &adriver.ImportDocumentOptions{
|
||||
OnDuplicate: adriver.ImportOnDuplicateIgnore,
|
||||
|
|
@ -419,8 +421,8 @@ func ProcessGMA(filePath string) (err error) {
|
|||
}
|
||||
//fmt.Printf("Code: %d, Created: %d, Ignored: %d, Errors: %d", statsImportGMA2File.Code, statsImportGMA2File.Created, statsImportGMA2File.Ignored, statsImportGMA2File.Errors)
|
||||
|
||||
fmt.Printf("Import Duration %dms\n", time.Since(importStartTime).Milliseconds())
|
||||
fmt.Println()
|
||||
log.Printf("Import Duration %dms\n", time.Since(importStartTime).Milliseconds())
|
||||
log.Println()
|
||||
// TODO: upload all unknownNewFiles to StorageServer
|
||||
http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 200
|
||||
var httpClient *http.Client = http.DefaultClient
|
||||
|
|
@ -431,13 +433,14 @@ func ProcessGMA(filePath string) (err error) {
|
|||
if unknownFileID == dboGMA2File.File {
|
||||
uploadSuccess := true
|
||||
for {
|
||||
fmt.Printf("Uploading %s (local %s) to Storage\n", dboGMA2File.UploadID, dboGMA2File.LocalFileName)
|
||||
log.Printf("Uploading %s to Storage\n", dboGMA2File.UploadID)
|
||||
err = common.MultipartUpload(httpClient, fmt.Sprintf("http://127.0.0.1:13371/stash/%s/%d", dboGMA2File.UploadID, dboGMA2File.FileSize), dboGMA2File.LocalFileName)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
if strings.Contains(err.Error(), "cannot assign requested address") {
|
||||
uploadSuccess = false
|
||||
} else {
|
||||
fmt.Println("oopsie")
|
||||
log.Println("oopsie")
|
||||
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
||||
return err
|
||||
}
|
||||
|
|
@ -454,7 +457,7 @@ func ProcessGMA(filePath string) (err error) {
|
|||
// TODO : write new gma from arangoinfo
|
||||
// TODO : compare hashes
|
||||
{
|
||||
fmt.Println("rewriting gma")
|
||||
log.Println("rewriting gma")
|
||||
destPath := filepath.Join(gmaTempPath, "rewrite.gma")
|
||||
dir := filepath.Dir(destPath)
|
||||
|
||||
|
|
@ -511,7 +514,7 @@ func ProcessGMA(filePath string) (err error) {
|
|||
}
|
||||
}
|
||||
gmaWriter.FileHandle.Seek(0, 2)
|
||||
fmt.Printf("Writing Footer CRC %d\n\n", dboGMA.FooterAddonCRC)
|
||||
log.Printf("Writing Footer CRC %d\n\n", dboGMA.FooterAddonCRC)
|
||||
gmaWriter.WriteFooterCRC(dboGMA.FooterAddonCRC)
|
||||
|
||||
gmaWriter.FileHandle.Seek(0, 0)
|
||||
|
|
@ -521,9 +524,9 @@ func ProcessGMA(filePath string) (err error) {
|
|||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("Rewrite Hash is %s %s\n", writeHash, destPath)
|
||||
fmt.Printf("Original Hash is %s %s\n", dboGMA.GMAHash, dboGMA.OriginalPath)
|
||||
fmt.Println()
|
||||
log.Printf("Rewrite Hash is %s %s\n", writeHash, destPath)
|
||||
log.Printf("Original Hash is %s %s\n", dboGMA.GMAHash, dboGMA.OriginalPath)
|
||||
log.Println()
|
||||
writeStat, err := os.Stat(destPath)
|
||||
if err != nil {
|
||||
undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue