package main

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime/debug"
	"sort"
	"strings"
	"sync"
	"time"

	"git.cheetah.cat/worksucc/gma-puzzles/common"
	"git.cheetah.cat/worksucc/gma-puzzles/gma"
	"github.com/arangodb/go-driver"
	adriver "github.com/arangodb/go-driver"
	ahttp "github.com/arangodb/go-driver/http"
	"github.com/schollz/progressbar/v3"
	"github.com/twinj/uuid"

	"github.com/jedib0t/go-pretty/v6/progress"

	_ "net/http/pprof"
)

var (
	arangoDB  adriver.Database
	arangoCTX context.Context
	//colChunk      adriver.Collection
	colFile adriver.Collection
	//colFile2Chunk adriver.Collection
	colGMA      adriver.Collection
	colGMA2File adriver.Collection
	colAliases  adriver.Collection

	workerID string
)

func ConnectDB(baseURL string, arangoUser string, arangoPWD string, arangoDatabase string) (driver adriver.Database, ctx context.Context, err error) {
	log.Println("connectDB:", "Starting Connection Process...")

	// Retry Loop for Failed Connections
	for i := 0; i < 6; i++ {
		if i == 5 {
			return driver, ctx, fmt.Errorf("connectdb unable to connect to database %d times", i)
		} else if i > 0 {
			time.Sleep(30 * time.Second)
		}

		// Connect to ArangoDB URL
		conn, err := ahttp.NewConnection(ahttp.ConnectionConfig{
			Endpoints: []string{baseURL},
			TLSConfig: &tls.Config{ /*...*/ },
		})
		if err != nil {
			log.Println("connectDB:", "Cannot Connect to ArangoDB!", err)
			continue
		}

		// Connect Driver to User
		client, err := adriver.NewClient(adriver.ClientConfig{
			Connection:     conn,
			Authentication: adriver.BasicAuthentication(arangoUser, arangoPWD),
		})
		if err != nil {
			log.Println("connectDB:", "Cannot Authenticate ArangoDB User!", err)
			continue
		}

		// Create Context for Database Access
		ctx = context.Background()
		driver, err = client.Database(ctx, arangoDatabase)
		if err != nil {
			log.Println("connectDB:", "Cannot Load ArangoDB Database!", err)
			continue
		}
		log.Println("connectDB:", "Connection Sucessful!")
		return driver, ctx, nil
	}

	return driver, ctx, fmt.Errorf("connectDB: FUCK HOW DID THIS EXCUTE?")
}
func InitDatabase() (err error) {
	arangoDB, arangoCTX, err = ConnectDB(common.ArangoHost, common.ArangoUser, common.ArangoPass, common.ArangoDatabase)
	if err != nil {
		return err
	}
	/*colChunk, err = arangoDB.Collection(arangoCTX, "chunk")
	if err != nil {
		return err
	}*/
	colFile, err = arangoDB.Collection(arangoCTX, "file")
	if err != nil {
		return err
	}
	colGMA, err = arangoDB.Collection(arangoCTX, "gma")
	if err != nil {
		return err
	}
	/*colFile2Chunk, err = arangoDB.Collection(arangoCTX, "file_chunk_map")
	if err != nil {
		return err
	}*/
	colGMA2File, err = arangoDB.Collection(arangoCTX, "gma_file_map")
	if err != nil {
		return err
	}

	colAliases, err = arangoDB.Collection(arangoCTX, "aliases")
	if err != nil {
		return err
	}
	return nil
}

var JobPoolSize int = 5
var ConcurrencyLimit int = 1
var WorkerJobPool chan string
var GlobalWriteLock sync.Mutex
var GlobalDeleteLock sync.Mutex

func main() {
	workerModeP := flag.String("mode", "", "mode (ingress, rebuild)")
	debugEnabled := flag.Bool("debug", false, "enables debug")
	folderPathP := flag.String("path", "/mnt/SC9000/TemporaryTestingShit2", "a string")
	skipNameP := flag.Int("skip", -1, "skip n addons")
	workerNameP := flag.String("worker", "def", "worker name")
	rebuildIDP := flag.String("id", "", "id to rebuild")

	flag.Parse()

	workerMode := *workerModeP

	if *debugEnabled {
		debug.SetMemoryLimit(6e9)
		go func() {
			log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
		}()
	}

	err := InitDatabase()
	if err != nil {
		panic(err)
	}

	switch workerMode {
	case "ingress":
		workerID = *workerNameP
		modeIngress(*folderPathP, *skipNameP)
	case "delete":
		workerID = *workerNameP
		modeDelete(*folderPathP)
	case "rebuild":
		flag.Parse()
		err = modeRebuild(*rebuildIDP)
		if err != nil {
			panic(err)
		}
	case "test":
		modeTest()
	}
}
func modeTest() (err error) {
	filePath := "/mnt/worksucc/san1/gma/2/5/0/0/2500735732.1623884796.gma"

	gmaReader, err := gma.NewReader(filePath)
	if err != nil {
		return err
	}
	defer gmaReader.Close()

	hash, err := gmaReader.GetSHA256()
	if err != nil {
		return err
	}
	fmt.Printf("GMA Hash: %s\n", hash)
	gmaReader.FileHandle.Seek(0, 0)

	header, err := gmaReader.ReadHeader()
	if err != nil {
		return err
	}
	log.Printf("Name=%s\n", header.Title)
	log.Printf("Desc=%s\n", header.Description)
	log.Printf("AddonVersion=%d\n", header.AddonVersion)
	log.Printf("FormatVersion=%d\n", header.FormatVersion)
	log.Printf("FormatVersionDiscardByte=%d\n", header.FormatVersionDiscardByte)
	firstType, files, err := gmaReader.ReadFiles()
	if err != nil {
		return err
	}
	fmt.Printf("firstType = %d\n", firstType)

	for _, file := range files {
		if file.FileSize < 0 { // Something is fucked
			return fmt.Errorf("GMA Header corrupted, NextType %d, FileNumber %d", file.NextType, file.FileNumber)
		}
		//fmt.Printf("%s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", file.FileName, file.CRC, file.Offset, file.FileSize, file.NextType, file.FileNumber)
		if file.NextType > uint32(file.FileNumber+10) { // Something is fucked
			/*log.Printf("Current Cursor %d", gmaReader.GetOffset())
			for _, otherFile := range files[file.FileNumber:] {
				log.Printf("OTHERFILE %s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", otherFile.FileName, otherFile.CRC, otherFile.Offset, otherFile.FileSize, otherFile.NextType, otherFile.FileNumber)
			}*/
			return fmt.Errorf("GMA Header corrupted, NextType %d, FileNumber %d", file.NextType, file.FileNumber)
		}
		extractMeta, err := gmaReader.ExtractFileTo(file, io.Discard)
		if err != nil {
			return err
		}

		if extractMeta.ExtractedCRC != extractMeta.OriginalMeta.CRC {
			fmt.Printf("gma(%s) checksum in meta (%d) differs from read (%d) [%s]\n", filePath, extractMeta.OriginalMeta.CRC, extractMeta.ExtractedCRC, extractMeta.OriginalMeta.FileName)
		}
		fmt.Println("#%d = %s -%d bytes [%s]\n", extractMeta.OriginalMeta.FileNumber, extractMeta.OriginalMeta.FileName, extractMeta.OriginalMeta.FileSize, extractMeta.ExtractedSHA256)
		//fmt.Printf("Extra
	}
	return nil
}

func modeRebuild(id string) (err error) {
	var (
		dboGMA       common.DB_GMA
		dboGMA2Files []common.DB_GMA2File
	)

	_, err = colGMA.ReadDocument(arangoCTX, id, &dboGMA)
	if err != nil {
		return err
	}

	cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR gf IN gma_file_map FILTER gf._from == 'gma/%s' RETURN gf", dboGMA.ID), nil)
	if err != nil {
		return err
	}

	defer cursor.Close()
	if cursor.Count() > 0 || cursor.HasMore() {
		for {
			gma2File := common.DB_GMA2File{}
			_, err = cursor.ReadDocument(arangoCTX, &gma2File)
			if driver.IsNoMoreDocuments(err) {
				break
			} else if err != nil {
				return err
			}
			gma2File.UploadID = gma2File.File[5:]
			dboGMA2Files = append(dboGMA2Files, gma2File)
		}
	} else {
		return fmt.Errorf("no files for gma available")
	}

	{
		log.Println("rewriting gma")
		rewriteBar := progressbar.Default(int64(len(dboGMA2Files)), "Rewriting GMA")
		destPath := fmt.Sprintf("./%s.gma", id) //filepath.Join(gmaTempPath, "rewrite.gma")
		dir := filepath.Dir(destPath)

		err := os.MkdirAll(dir, os.ModePerm)
		if err != nil {
			return err
		}

		gmaWriter, err := gma.NewWriter(destPath)
		if err != nil {
			return err
		}
		defer gmaWriter.Close()

		//fmt.Printf("Writing Header with FormatVersion: %d\n", dboGMA.Header.FormatVersion)
		err = gmaWriter.WriteHeader(dboGMA.Header)
		if err != nil {
			return err
		}
		err = gmaWriter.WriteFirstType(dboGMA.FirstType)
		if err != nil {
			return err
		}

		sort.SliceStable(dboGMA2Files, func(i, j int) bool { return dboGMA2Files[i].FileNumber < dboGMA2Files[j].FileNumber })
		for _, dboGMA2File := range dboGMA2Files {
			//fmt.Printf("WriteFileIndex for %s number %d\n", dboGMA2File.FileName, dboGMA2File.FileNumber)
			err = gmaWriter.WriteFileIndex(dboGMA2File.FileName, dboGMA2File.FileSize, dboGMA2File.CRC, dboGMA2File.NextType)
			if err != nil {
				return err
			}
		}
		var httpClient *http.Client = &http.Client{
			Timeout: 15 * time.Minute,
		}

		for _, dboGMA2File := range dboGMA2Files {
			//fmt.Printf("WriteFile for %s number %d = %s\n", dboGMA2File.FileName, dboGMA2File.FileNumber, dboGMA2File.UploadID)
			resp, err := httpClient.Get(fmt.Sprintf("http://192.168.133.118:13371/fetch/%s", dboGMA2File.UploadID))
			if err != nil {
				return err
			}
			defer resp.Body.Close()
			err = gmaWriter.WriteFile(resp.Body)
			if err != nil {
				return err
			}
			rewriteBar.Add(1)
		}
		gmaWriter.FileHandle.Seek(0, 2)
		log.Printf("Writing Footer CRC %d\n\n", dboGMA.FooterAddonCRC)
		gmaWriter.WriteFooterCRC(dboGMA.FooterAddonCRC)

		// TODO: maybe use io.MultiWriter ??
		gmaWriter.FileHandle.Seek(0, 0)
		writeHash, err := gmaWriter.GetSHA256()
		if err != nil {
			return err
		}

		log.Printf("Rewrite Hash is %s %s\n", writeHash, destPath)
		log.Printf("Original Hash is %s %s\n", dboGMA.GMAHash, dboGMA.OriginalPath)
		log.Println()
		writeStat, err := os.Stat(destPath)
		if err != nil {
			return err
		}
		writeSize := writeStat.Size()
		if writeSize != dboGMA.GMASize {
			//fail
			return fmt.Errorf("RewriteCheck failed, original=%s (%d bytes), rewrite=%s (%d bytes)", dboGMA.GMAHash, dboGMA.GMASize, writeHash, writeSize)
		}
		if writeHash != dboGMA.GMAHash {
			//fail
			return fmt.Errorf("RewriteCheck failed, original=%s (%d bytes), rewrite=%s (%d bytes)", dboGMA.GMAHash, dboGMA.GMASize, writeHash, writeSize)
		}
	}

	return nil
}
func recursive(jobs []string, folderPath string) (WorkerJobPool []string) {
	entries, err := os.ReadDir(folderPath)
	if err != nil {
		panic(err)
	}
	for _, e := range entries {
		fullPath := filepath.Join(folderPath, e.Name())
		if e.IsDir() {
			jobs = recursive(jobs, fullPath)
		}
		fmt.Println(e.Name())
		if strings.HasSuffix(e.Name(), ".gma") && !strings.Contains(e.Name(), ".lzma") {
			if !e.IsDir() {
				jobs = append(jobs, filepath.Join(folderPath, e.Name()))
			}
		}
	}
	return jobs
}

func modeDelete(folderPath string) {
	// skipNameEnabled := skipN > 0
	entries, err := os.ReadDir(folderPath)
	if err != nil {
		panic(err)
	}
	var WorkerJobPool []string

	for _, e := range entries {
		fullPath := filepath.Join(folderPath, e.Name())
		if e.IsDir() {
			WorkerJobPool = recursive(WorkerJobPool, fullPath)
		}
		fmt.Println(e.Name())
		if strings.HasSuffix(e.Name(), ".gma") && !strings.Contains(e.Name(), ".lzma") {
			if !e.IsDir() {
				WorkerJobPool = append(WorkerJobPool, filepath.Join(folderPath, e.Name()))
			}
		}
	}
	wg := sync.WaitGroup{}
	deleteSem := common.NewSemaphore(4)

	pw := progress.NewWriter()
	pw.SetAutoStop(true)
	pw.SetTrackerLength(40)
	pw.SetMessageWidth(40)
	//pw.SetNumTrackersExpected(*flagNumTrackers)
	pw.SetSortBy(progress.SortByPercentDsc)
	pw.SetStyle(progress.StyleDefault)
	pw.SetTrackerPosition(progress.PositionRight)
	pw.SetUpdateFrequency(time.Millisecond * 100)
	pw.Style().Colors = progress.StyleColorsExample
	pw.Style().Options.PercentFormat = "%4.1f%%"
	pw.Style().Options.ETAPrecision = time.Second
	pw.Style().Options.TimeDonePrecision = time.Second
	pw.Style().Options.TimeInProgressPrecision = time.Second
	pw.Style().Options.TimeOverallPrecision = time.Second
	pw.Style().Visibility.ETA = true
	pw.Style().Visibility.ETAOverall = true
	pw.Style().Visibility.Percentage = false
	pw.Style().Visibility.Speed = true
	pw.Style().Visibility.SpeedOverall = false
	pw.Style().Visibility.Time = true
	pw.Style().Visibility.TrackerOverall = false
	pw.Style().Visibility.Value = true
	pw.Style().Visibility.Pinned = true

	// call Render() in async mode; yes we don't have any trackers at the moment
	go pw.Render()

	trackerDoneMarker := sync.Once{}
	tracker := progress.Tracker{Message: fmt.Sprintf("Deleting successfull %d GMAs", len(WorkerJobPool)), Total: int64(len(WorkerJobPool)), Units: progress.UnitsDefault}
	pw.AppendTracker(&tracker)
	defer trackerDoneMarker.Do(tracker.MarkAsDone)

	for _, jobFile := range WorkerJobPool {
		wg.Add(1)
		go func(jobFile string, wg *sync.WaitGroup) {
			deleteSem.Acquire()
			defer deleteSem.Release()
			defer wg.Done()
			defer tracker.Increment(1)
			err = DeleteIfSafeGMA(pw, jobFile)
			if err != nil {
				pw.Log(fmt.Sprintf("\nERROR: %v\n", err))
			}

		}(jobFile, &wg)
	}

	// Wait for all jobs to finish
	wg.Wait()
}
func modeIngress(folderPath string, skipN int) {
	// skipNameEnabled := skipN > 0
	entries, err := os.ReadDir(folderPath)
	if err != nil {
		panic(err)
	}
	var WorkerJobPool []string

	for _, e := range entries {
		fullPath := filepath.Join(folderPath, e.Name())
		if e.IsDir() {
			WorkerJobPool = recursive(WorkerJobPool, fullPath)
		}
		fmt.Println(e.Name())
		if strings.HasSuffix(e.Name(), ".gma") && !strings.Contains(e.Name(), ".lzma") {
			if !e.IsDir() {
				WorkerJobPool = append(WorkerJobPool, filepath.Join(folderPath, e.Name()))
			}
		}
	}
	if skipN > 0 {
		WorkerJobPool = WorkerJobPool[skipN:]
	}
	wg := sync.WaitGroup{}

	pw := progress.NewWriter()
	pw.SetAutoStop(true)
	pw.SetTrackerLength(40)
	pw.SetMessageWidth(40)
	//pw.SetNumTrackersExpected(*flagNumTrackers)
	pw.SetSortBy(progress.SortByPercentDsc)
	pw.SetStyle(progress.StyleDefault)
	pw.SetTrackerPosition(progress.PositionRight)
	pw.SetUpdateFrequency(time.Millisecond * 100)
	pw.Style().Colors = progress.StyleColorsExample
	pw.Style().Options.PercentFormat = "%4.1f%%"
	pw.Style().Options.ETAPrecision = time.Second
	pw.Style().Options.TimeDonePrecision = time.Second
	pw.Style().Options.TimeInProgressPrecision = time.Second
	pw.Style().Options.TimeOverallPrecision = time.Second
	pw.Style().Visibility.ETA = true
	pw.Style().Visibility.ETAOverall = true
	pw.Style().Visibility.Percentage = false
	pw.Style().Visibility.Speed = true
	pw.Style().Visibility.SpeedOverall = false
	pw.Style().Visibility.Time = true
	pw.Style().Visibility.TrackerOverall = false
	pw.Style().Visibility.Value = true
	pw.Style().Visibility.Pinned = true

	// call Render() in async mode; yes we don't have any trackers at the moment
	go pw.Render()

	trackerDoneMarker := sync.Once{}
	tracker := progress.Tracker{Message: fmt.Sprintf("Working %d GMAs", len(WorkerJobPool)), Total: int64(len(WorkerJobPool)), Units: progress.UnitsDefault}
	pw.AppendTracker(&tracker)
	defer trackerDoneMarker.Do(tracker.MarkAsDone)

	for _, jobFile := range WorkerJobPool {
		wg.Add(1)
		go func(jobFile string, wg *sync.WaitGroup) {
			defer tracker.Increment(1)
			defer wg.Done()
			err = ProcessGMA(pw, jobFile)
			if err != nil {
				pw.Log(fmt.Sprintf("\nERROR: %v\n", err))
				if strings.Contains(err.Error(), "refused") {
					panic(err)
				}
			}
			err = DeleteIfSafeGMA(pw, jobFile)
			if err != nil {
				pw.Log(fmt.Sprintf("\nERROR: %v\n", err))
			}

		}(jobFile, &wg)
	}

	// Wait for all jobs to finish
	wg.Wait()
}
func undoBatch(undoBatch bool, gmaID string, fileIDs []string, gma2FileIDs []string) (err error) {
	//log.Printf("undoBatch(%v, %s)\n", undoBatch, gmaID)
	/*
		_, err = colGMA.RemoveDocument(arangoCTX, gmaID)
		if err != nil {
			return err
		}
	*/
	/*
		_, _, err = colFile.RemoveDocuments(arangoCTX, fileIDs)
		if err != nil {
			return err
		}
	*/
	/*
		_, _, err = colGMA2File.RemoveDocuments(arangoCTX, gma2FileIDs)
		if err != nil {
			return err
		}
	*/
	return nil
}

func DeleteIfSafeGMA(pw progress.Writer, filePath string) (err error) {
	var unlockOnce sync.Once

	GlobalDeleteLock.Lock()                      // Wait for worker to have slot open
	defer unlockOnce.Do(GlobalDeleteLock.Unlock) // release anyway
	dboGMA := common.DB_GMA{}

	dboGMA.OriginalPath = filePath
	cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR g IN gma FILTER g.originalPath == '%s' RETURN g", dboGMA.OriginalPath), nil)
	if err != nil {
		return fmt.Errorf("545: %v", err)
	}

	defer cursor.Close()
	initialRetryCounter := 0
	if cursor.Count() > 0 || cursor.HasMore() {
		for {
			gma := common.DB_GMA{}
			_, err = cursor.ReadDocument(arangoCTX, &gma)
			if driver.IsNoMoreDocuments(err) {
				break
			} else if err != nil {
				return fmt.Errorf("557: %v", err)
			}
			if !gma.Success {
				return fmt.Errorf("GMA with ID %s was not successfull ", gma.ID)
			}
		}
		//} else {
		//	return fmt.Errorf("no gmas found with path = %s", dboGMA.OriginalPath)
	}

	fmt.Println(dboGMA)
	fileStat, err := os.Stat(filePath)
	if err != nil {
		return err
	}
	dboGMA.StatModTime = fileStat.ModTime()
	dboGMA.GMASize = fileStat.Size()
	dboGMA.RetryCounter = initialRetryCounter + 1

	if dboGMA.GMASize < 200 {
		return fmt.Errorf("GMA File too small, skipping")
	}
	//niceName := filepath.Base(filePath)
	//trackerProcessDoneMarker := sync.Once{}
	//trackerProcess := progress.Tracker{Message: fmt.Sprintf("Deleting %s", niceName), Total: 0, Units: progress.UnitsDefault}
	//defer trackerProcessDoneMarker.Do(trackerProcess.MarkAsDone)
	//pw.AppendTracker(&trackerProcess)

	gmaReader, err := gma.NewReader(filePath)
	if err != nil {
		return err
	}
	defer gmaReader.Close()

	unlockOnce.Do(GlobalDeleteLock.Unlock) // release anyway
	dboGMA.GMAHash, err = gmaReader.GetSHA256()
	if err != nil {
		return err
	}
	dboGMA.ID = dboGMA.GMAHash
	gmaReader.FileHandle.Seek(0, 0)

	dboIDExists, err := colGMA.DocumentExists(arangoCTX, dboGMA.ID)
	if err != nil {
		return err
	}

	dboGMA_sameHash := common.DB_GMA{}
	if !dboIDExists {
		return fmt.Errorf("GMA with ID %s does not exists", dboGMA.ID)
	} else {
		_, err := colGMA.ReadDocument(arangoCTX, dboGMA.ID, &dboGMA_sameHash)
		if err != nil {
			return fmt.Errorf("609-: %v", err)
		}
		if dboGMA_sameHash.OriginalPath != filePath {
			pw.Log("originalpath differs for this hash")
			// DB_GMA_Alias
			dboGMA.OriginalPath = dboGMA_sameHash.OriginalPath
		}
	}
	if dboGMA.OriginalPath == filePath {
		{
			recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath))
			if checkFileExists(recoveryPath) {
				err = os.Remove(dboGMA.OriginalPath)
				if err != nil {
					return err
				}
				pw.Log("deleted %s", dboGMA.OriginalPath)
			} else {
				pw.Log("%s recoveryPath does not exist %s", filePath, recoveryPath)
			}
			lzmaFile := fmt.Sprintf("%s.lzma", dboGMA.OriginalPath)
			if checkFileExists(lzmaFile) {
				err = os.Remove(lzmaFile)
				if err != nil {
					return err
				}
				pw.Log("deleted %s", lzmaFile)
			}
		}
		{
			_, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, common.DB_GMADeletionData{
				Deleted: true,
			})
			if err != nil {
				return err
			}
		}
	} else {
		originalDeletionData := common.DB_GMADeletionData{}
		_, err := colGMA.ReadDocument(arangoCTX, dboGMA.ID, &originalDeletionData)
		if err != nil {
			return fmt.Errorf("650-: %v", err)
		}
		if originalDeletionData.Deleted {
			pw.Log(filepath.Base(filePath))
			pw.Log(filepath.Base(dboGMA_sameHash.OriginalPath))
			pw.Log("")
			aliasData := common.DB_GMA_Alias{
				Path:        filePath,
				Hash:        dboGMA.GMAHash,
				Deleted:     true,
				MigrationID: common.CreateMigrationID(filePath),
			}
			err = os.Remove(aliasData.Path)
			if err != nil {
				return err
			}
			pw.Log("deleted %s", aliasData.Path)
			lzmaFile := fmt.Sprintf("%s.lzma", aliasData.Path)
			if checkFileExists(lzmaFile) {
				err = os.Remove(lzmaFile)
				if err != nil {
					return err
				}
				pw.Log("deleted %s", lzmaFile)
			}
			_, err = colAliases.CreateDocument(arangoCTX, aliasData)
			if err != nil {
				return err
			}
		} else {
			return fmt.Errorf("originals have not been deleted")
		}
	}
	return nil
}

func checkFileExists(filePath string) bool {
	_, error := os.Stat(filePath)
	//return !os.IsNotExist(err)
	return !errors.Is(error, os.ErrNotExist)
}
func ProcessGMA(pw progress.Writer, filePath string) (err error) {
	var unlockOnce sync.Once

	//fmt.Println("trying to acquire global write lock")
	GlobalWriteLock.Lock() // Wait for worker to have slot open
	//fmt.Println("aquired global write lock")
	defer unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway
	//defer fmt.Println("unlocking GlobalWriteLock")
	//time.Sleep(5 * time.Second)

	var (
		fileIDs     []string
		gma2FileIDs []string
	)
	dboGMA := common.DB_GMA{}
	dboGMA.BatchID = uuid.NewV4().String() // use this for rapid unscheduled dissassembly

	dboGMA.OriginalPath = filePath
	dboGMA.ProcessingStart = time.Now()

	cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR g IN gma FILTER g.originalPath == '%s' RETURN g", dboGMA.OriginalPath), nil)
	if err != nil {
		return err
	}

	defer cursor.Close()
	initialRetryCounter := 0
	skipHashCheck := false
	if cursor.Count() > 0 || cursor.HasMore() {
		for {
			gma := common.DB_GMA{}
			_, err = cursor.ReadDocument(arangoCTX, &gma)
			if driver.IsNoMoreDocuments(err) {
				break
			} else if err != nil {
				return err
			}
			if gma.Success {
				return fmt.Errorf("GMA with ID %s was successfull at %v", gma.ID, gma.ProcessingEnd)
			} else {
				if gma.RetryCounter > 3 {
					return fmt.Errorf("GMA with ID %s was too many times unsuccessfull", gma.ID)
				}
				skipHashCheck = true
				initialRetryCounter = gma.RetryCounter + 1
				_, err = colGMA.UpdateDocument(arangoCTX, gma.ID, &gma)
				if err != nil {
					return err
				}
			}
		}
	}

	fileStat, err := os.Stat(filePath)
	if err != nil {
		return err
	}
	dboGMA.StatModTime = fileStat.ModTime()
	dboGMA.GMASize = fileStat.Size()
	dboGMA.RetryCounter = initialRetryCounter + 1

	if dboGMA.GMASize < 200 {
		return fmt.Errorf("GMA File too small, skipping")
	}
	niceName := filepath.Base(filePath)
	trackerProcessDoneMarker := sync.Once{}
	trackerProcess := progress.Tracker{Message: fmt.Sprintf("Extracting %s", niceName), Total: 0, Units: progress.UnitsDefault}
	defer trackerProcessDoneMarker.Do(trackerProcess.MarkAsDone)
	pw.AppendTracker(&trackerProcess)

	//log.Printf("Opening %s\n", filePath)
	gmaReader, err := gma.NewReader(filePath)
	if err != nil {
		return err
	}
	defer gmaReader.Close()

	dboGMA.GMAHash, err = gmaReader.GetSHA256()
	if err != nil {
		return err
	}
	dboGMA.ID = dboGMA.GMAHash
	gmaReader.FileHandle.Seek(0, 0)

	gmaTempPath := filepath.Join("/mnt/ramfs/gma-extr-temp", dboGMA.ID)
	defer os.RemoveAll(gmaTempPath) // clean up under any circumstances

	dboIDExists, err := colGMA.DocumentExists(arangoCTX, dboGMA.ID)
	if err != nil {
		return err
	}
	if dboIDExists && !skipHashCheck {
		return fmt.Errorf("GMA with ID %s exists", dboGMA.ID)
	}
	if dboIDExists && skipHashCheck {
		_, err = colGMA.RemoveDocument(arangoCTX, dboGMA.ID)
		if err != nil {
			return err
		}
	}

	header, err := gmaReader.ReadHeader()
	if err != nil {
		return err
	}
	dboGMA.Header = header
	//log.Printf("Name=%s\n", header.Title)
	//log.Printf("Desc=%s\n", header.Description)
	// log.Printf("AddonVersion=%d\n", header.AddonVersion)
	// log.Printf("FormatVersion=%d\n", header.FormatVersion)
	// log.Printf("FormatVersionDiscardByte=%d\n", header.FormatVersionDiscardByte)
	// //fmt.Printf("r.cursorOffset = %d\n", gmaReader.GetOffset())
	firstType, files, err := gmaReader.ReadFiles()
	if err != nil {
		return err
	}
	trackerProcess.UpdateTotal(int64(len(files)))
	dboGMA.FirstType = firstType

	//fmt.Printf("r.cursorOffset = %d\n", gmaReader.GetOffset())
	var (
		dboGMA2Files []common.DB_GMA2File
		dboFiles     []common.DB_File
	)
	// Convert GMA Files into DB Metadata
	for _, file := range files {
		if file.FileSize < 0 { // Something is fucked
			return fmt.Errorf("GMA Header corrupted, NextType %d, FileNumber %d", file.NextType, file.FileNumber)
		}
		//fmt.Printf("%s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", file.FileName, file.CRC, file.Offset, file.FileSize, file.NextType, file.FileNumber)
		if file.NextType > uint32(file.FileNumber+10) { // Something is fucked
			/*log.Printf("Current Cursor %d", gmaReader.GetOffset())
			for _, otherFile := range files[file.FileNumber:] {
				log.Printf("OTHERFILE %s CRC: %d Offset: %d Size: %d NextType: %d FileNumber: %d\n", otherFile.FileName, otherFile.CRC, otherFile.Offset, otherFile.FileSize, otherFile.NextType, otherFile.FileNumber)
			}*/
			return fmt.Errorf("GMA Header corrupted, NextType %d, FileNumber %d", file.NextType, file.FileNumber)
		}
		destPath := filepath.Join(gmaTempPath, "contents", file.FileName)
		dir := filepath.Dir(destPath)

		err := os.MkdirAll(dir, os.ModePerm)
		if err != nil {
			return err
		}

		destFile, err := os.Create(destPath)
		if err != nil {
			return err
		}
		defer destFile.Close()
		extractMeta, err := gmaReader.ExtractFileTo(file, destFile)
		if err != nil {
			return err
		}

		if extractMeta.ExtractedCRC != extractMeta.OriginalMeta.CRC {
			pw.Log(fmt.Sprintf("gma(%s) checksum in meta (%d) differs from read (%d) [%s]\n", filePath, extractMeta.OriginalMeta.CRC, extractMeta.ExtractedCRC, extractMeta.OriginalMeta.FileName))
		}
		//fmt.Printf("ExtractedMeta %s CRC: %d SHA256: %s\n", file.FileName, extractMeta.ExtractedCRC, extractMeta.ExtractedSHA256)
		dboFile := common.DB_File{
			ID:          extractMeta.ExtractedSHA256, // ID is the SHA256, i guess that is good enough?
			BatchID:     dboGMA.BatchID,
			InitialPath: file.FileName,
			CRC:         file.CRC,
			Size:        file.FileSize,
			Hash:        extractMeta.ExtractedSHA256,
			Extension:   filepath.Ext(file.FileName),
			G2FRef:      fmt.Sprintf("%s_%s_%d", dboGMA.ID, extractMeta.ExtractedSHA256, extractMeta.OriginalMeta.FileNumber), // reference for the GMA2File Thing so that we can find it agian in the list
		}
		dboGMA2File := common.DB_GMA2File{
			ID:         dboFile.G2FRef,
			BatchID:    dboGMA.BatchID,
			File:       fmt.Sprintf("file/%s", extractMeta.ExtractedSHA256),
			GMA:        fmt.Sprintf("gma/%s", dboGMA.ID),
			FileNumber: extractMeta.OriginalMeta.FileNumber,
			FileName:   extractMeta.OriginalMeta.FileName,
			Offset:     extractMeta.OriginalMeta.Offset,
			FileSize:   extractMeta.OriginalMeta.FileSize,
			CRCMatch:   extractMeta.ExtractedCRC == extractMeta.OriginalMeta.CRC,
			CRC:        extractMeta.OriginalMeta.CRC,
			NextType:   extractMeta.OriginalMeta.NextType,

			LocalFileName: destPath,
			UploadID:      extractMeta.ExtractedSHA256,
		}
		//fmt.Println(dboGMA2File)
		// Add fileIDs from new unknowns
		dboFiles = append(dboFiles, dboFile)
		//fmt.Println(dboGMA2File)
		gma2FileIDs = append(gma2FileIDs, dboGMA2File.ID)
		dboGMA2Files = append(dboGMA2Files, dboGMA2File)
		trackerProcess.Increment(1)
	}
	if len(files) < 1 {
		return fmt.Errorf("GMA %s empty, len=%d", dboGMA.OriginalPath, len(files))
	}

	lastFile := files[len(files)-1]
	dboGMA.FooterAddonCRC, err = gmaReader.ReadAddonCRC(lastFile.Offset + lastFile.FileSize)
	if err != nil {
		return err
	}
	dboGMA.ProcessingEnd = time.Now()
	dboGMA.ProcessingDuration = dboGMA.ProcessingEnd.Sub(dboGMA.ProcessingStart).Milliseconds()

	// TODO: Calculate dboGMA.OptimizedSize
	dboGMA.OptimizedSize = 0
	_, err = colGMA.CreateDocument(arangoCTX, dboGMA)
	if err != nil {
		undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
		return err
	}
	// TODO: Check all dboFiles and dboGMA2Files if they exist, if something is odd, queue reupload

	dboExistFile := map[string]bool{}
	dboExistFile2GMA := map[string]bool{}
	for _, dboFile := range dboFiles {
		exists, err := colFile.DocumentExists(arangoCTX, dboFile.ID)
		if err != nil {
			return err
		}
		dboExistFile[dboFile.ID] = exists
	}
	for _, dboGMA2File := range dboGMA2Files {
		exists, err := colGMA2File.DocumentExists(arangoCTX, dboGMA2File.ID)
		if err != nil {
			return err
		}
		dboExistFile2GMA[dboGMA2File.ID] = exists
	}
	trackerProcessDoneMarker.Do(trackerProcess.MarkAsDone)

	// TODO: upload all unknownNewFiles to StorageServer
	http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 200
	var httpClient *http.Client = http.DefaultClient

	trackerUploadDoneMarker := sync.Once{}
	trackerUpload := progress.Tracker{Message: fmt.Sprintf("Uploading %s", niceName), Total: int64(len(dboFiles)), Units: progress.UnitsDefault}
	pw.AppendTracker(&trackerUpload)
	defer trackerUploadDoneMarker.Do(trackerUpload.MarkAsDone)

	for _, dboFile := range dboFiles {
		dboFileID := fmt.Sprintf("file/%s", dboFile.ID)
		//fmt.Printf("Line 460: %s checking if we need to store this on the server", dboFileID)
		//dboFile2ChunkID := fmt.Sprintf("file_chunk_map/%s", dboFile.ID)

		// TODO: Check against Storage backend
		res, err := http.Get(fmt.Sprintf("http://192.168.133.118:13371/check/%s", dboFile.ID))
		if err != nil {
			return err
		}
		defer res.Body.Close()
		if _, err = io.Copy(io.Discard, res.Body); err != nil {
			return err
		}
		//body, _ := ioutil.ReadAll(res.Body)
		//fmt.Printf("res.StatusCode = %d\n", res.StatusCode)
		if res.StatusCode == http.StatusAlreadyReported {
			trackerUpload.UpdateMessage(fmt.Sprintf("Skipping %s", niceName))

			for _, dboGMA2File := range dboGMA2Files {
				if dboFile.G2FRef == dboGMA2File.ID {
					// Create File and dboGMA2File Object
					exists, err := colGMA2File.DocumentExists(arangoCTX, dboGMA2File.ID)
					if err != nil {
						log.Println("err @colGMA2File.DocumentExists")
						log.Println("oopsie")
						undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
						trackerUpload.MarkAsErrored()
						return err
					}
					if !exists {
						_, err = colGMA2File.CreateDocument(arangoCTX, dboGMA2File)
						if err != nil {
							log.Println("err @colGMA2File.CreateDocument")
							log.Println("oopsie")
							undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
							trackerUpload.MarkAsErrored()
							return err
						}
						//} else {
						//log.Println("already exists... weird")
					}
					break
				}
			}

			trackerUpload.Increment(1)
			continue
		}

		for _, dboGMA2File := range dboGMA2Files {
			if dboFileID == dboGMA2File.File { // find corresponding dboGMA2File
				//log.Println("Found dboFileID == dboGMA2File.Ref ID")
				uploadSuccess := true
				for {
					fileInfoJSON, err := json.Marshal(dboFile)
					if err != nil {
						log.Println("err @json.Marshal dboFile")
						return err
					}

					//uploadBar.Describe("Uploading")
					trackerUpload.UpdateMessage(fmt.Sprintf("Uploading %s", niceName))
					err = common.MultipartUpload(httpClient, fmt.Sprintf("http://192.168.133.118:13371/stash/%s/%d", dboGMA2File.UploadID, dboGMA2File.FileSize), dboGMA2File.LocalFileName, fileInfoJSON, workerID)
					if err != nil {
						log.Println("err @common.MultipartUpload")
						log.Println(err)
						if strings.Contains(err.Error(), "cannot assign requested address") {
							uploadSuccess = false
						} else if strings.Contains(err.Error(), "refused") {
							panic(err)
						} else {
							log.Println("oopsie")
							undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
							trackerUpload.MarkAsErrored()
							return err
						}
					}
					if uploadSuccess {
						// Create File and dboGMA2File Object
						exists, err := colGMA2File.DocumentExists(arangoCTX, dboGMA2File.ID)
						if err != nil {
							log.Println("err @colGMA2File.DocumentExists")
							log.Println("oopsie")
							undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
							trackerUpload.MarkAsErrored()
							return err
						}
						if !exists {
							_, err = colGMA2File.CreateDocument(arangoCTX, dboGMA2File)
							if err != nil {
								log.Println("err @colGMA2File.CreateDocument")
								log.Println("oopsie")
								undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
								trackerUpload.MarkAsErrored()
								return err
							}
							//} else {
							//log.Println("already exists... weird")
						}
						trackerUpload.Increment(1)
						break
					}
					time.Sleep(10 * time.Second)
				}
				if uploadSuccess {
					break
				}
			}
		}
	}
	trackerUploadDoneMarker.Do(trackerUpload.MarkAsDone)
	// at this point we can release the write semaphore
	unlockOnce.Do(GlobalWriteLock.Unlock) // release anyway
	//fmt.Println("unlocking GlobalWriteLock")

	// TODO : fetch all files from storageServer
	// TODO : write new gma from arangoinfo
	// TODO : compare hashes
	{
		var (
			rw_dboGMA2Files []common.DB_GMA2File
		)

		cursor, err := arangoDB.Query(arangoCTX, fmt.Sprintf("FOR gf IN gma_file_map FILTER gf._from == 'gma/%s' RETURN gf", dboGMA.ID), nil)
		if err != nil {
			return err
		}

		defer cursor.Close()
		if cursor.Count() > 0 || cursor.HasMore() {
			for {
				gma2File := common.DB_GMA2File{}
				_, err = cursor.ReadDocument(arangoCTX, &gma2File)
				if driver.IsNoMoreDocuments(err) {
					break
				} else if err != nil {
					return err
				}
				gma2File.UploadID = gma2File.File[5:]
				rw_dboGMA2Files = append(rw_dboGMA2Files, gma2File)
			}
		} else {
			return fmt.Errorf("no files for gma available")
		}

		trackerRewriteDoneMarker := sync.Once{}
		trackerRewrite := progress.Tracker{Message: fmt.Sprintf("Rewriting %s", niceName), Total: int64(len(dboFiles)), Units: progress.UnitsDefault}
		pw.AppendTracker(&trackerRewrite)
		defer trackerRewriteDoneMarker.Do(trackerRewrite.MarkAsDone)
		destPath := filepath.Join(gmaTempPath, "rewrite.gma")
		dir := filepath.Dir(destPath)

		err = os.MkdirAll(dir, os.ModePerm)
		if err != nil {
			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			return err
		}

		gmaWriter, err := gma.NewWriter(destPath)
		if err != nil {
			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			return err
		}
		defer gmaWriter.Close()

		//fmt.Printf("Writing Header with FormatVersion: %d\n", dboGMA.Header.FormatVersion)
		err = gmaWriter.WriteHeader(dboGMA.Header)
		if err != nil {
			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			return err
		}
		err = gmaWriter.WriteFirstType(dboGMA.FirstType)
		if err != nil {
			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			return err
		}

		sort.SliceStable(rw_dboGMA2Files, func(i, j int) bool { return rw_dboGMA2Files[i].FileNumber < rw_dboGMA2Files[j].FileNumber })
		for _, dboGMA2File := range rw_dboGMA2Files {
			//fmt.Printf("WriteFileIndex for %s number %d\n", dboGMA2File.FileName, dboGMA2File.FileNumber)
			err = gmaWriter.WriteFileIndex(dboGMA2File.FileName, dboGMA2File.FileSize, dboGMA2File.CRC, dboGMA2File.NextType)
			if err != nil {
				undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
				return err
			}
		}
		var httpClient *http.Client = &http.Client{
			Timeout: 15 * time.Minute,
		}

		for _, dboGMA2File := range rw_dboGMA2Files {
			//fmt.Printf("WriteFile for %s number %d = %s\n", dboGMA2File.FileName, dboGMA2File.FileNumber, dboGMA2File.UploadID)
			resp, err := httpClient.Get(fmt.Sprintf("http://192.168.133.118:13371/fetch/%s", dboGMA2File.UploadID))
			if err != nil {
				undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
				return err
			}
			defer resp.Body.Close()
			err = gmaWriter.WriteFile(resp.Body)
			if err != nil {
				undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
				return err
			}
			trackerRewrite.Increment(1)
		}
		gmaWriter.FileHandle.Seek(0, 2)
		//log.Printf("Writing Footer CRC %d\n\n", dboGMA.FooterAddonCRC)
		gmaWriter.WriteFooterCRC(dboGMA.FooterAddonCRC)

		// TODO: maybe use io.MultiWriter ??
		gmaWriter.FileHandle.Seek(0, 0)
		writeHash, err := gmaWriter.GetSHA256()
		if err != nil {
			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			return err
		}
		//log.Printf("Rewrite Hash is %s %s\n", writeHash, destPath)
		//log.Printf("Original Hash is %s %s\n", dboGMA.GMAHash, dboGMA.OriginalPath)
		//log.Println()
		writeStat, err := os.Stat(destPath)
		if err != nil {
			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			return err
		}
		writeSize := writeStat.Size()
		if writeSize != dboGMA.GMASize {
			//fail
			//createDebugInformation

			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			trackerRewrite.MarkAsErrored()
			return fmt.Errorf("RewriteCheck failed, original=%s (%d bytes), rewrite=%s (%d bytes)", dboGMA.GMAHash, dboGMA.GMASize, writeHash, writeSize)
		}
		if writeHash != dboGMA.GMAHash {
			//fail
			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			trackerRewrite.MarkAsErrored()
			return fmt.Errorf("RewriteCheck failed, original=%s (%d bytes), rewrite=%s (%d bytes)", dboGMA.GMAHash, dboGMA.GMASize, writeHash, writeSize)
		}
		trackerRewriteDoneMarker.Do(trackerRewrite.MarkAsDone)

		recoveryData := common.JSON_GMARecovery{
			GMA:  dboGMA,
			Refs: rw_dboGMA2Files,
		}
		recoveryBytes, err := json.MarshalIndent(recoveryData, "", "\t")
		if err != nil {
			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			return err
		}

		recoveryPath := filepath.Join("/zpool0/cheetah/workshop/garrysmod/gma-inator/recovery-tree", fmt.Sprintf("%s.json", dboGMA.OriginalPath))
		err = os.MkdirAll(filepath.Dir(recoveryPath), os.ModePerm)
		if err != nil {
			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			return err
		}
		recoveryFile, err := os.Create(recoveryPath)
		if err != nil {
			undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
			return err
		}
		_, err = recoveryFile.Write(recoveryBytes)
		if err != nil {
			return fmt.Errorf("error @recoveryFile.Write %v", err)
		}

	}
	// TODO: 4... profit?

	dboGMA.ProcessingEnd = time.Now()
	dboGMA.ProcessingDuration = dboGMA.ProcessingEnd.Sub(dboGMA.ProcessingStart).Milliseconds()
	dboGMA.Success = true
	dboGMA.MigrationID = common.CreateMigrationID(dboGMA.OriginalPath)

	_, err = colGMA.UpdateDocument(arangoCTX, dboGMA.ID, dboGMA)
	if err != nil {
		undoBatch(true, dboGMA.ID, fileIDs, gma2FileIDs)
		return err
	}
	return nil
}