From c5b6c3c12ef3da415ac55d0a39ea5a1263a58515 Mon Sep 17 00:00:00 2001 From: cheetah Date: Fri, 7 Jul 2023 00:23:07 -0500 Subject: [PATCH] moved some crap to .ignore folder --- fix/fix.go | 256 --------------------------------------- zstd-tar-test/tartest.go | 76 ------------ 2 files changed, 332 deletions(-) delete mode 100644 fix/fix.go delete mode 100644 zstd-tar-test/tartest.go diff --git a/fix/fix.go b/fix/fix.go deleted file mode 100644 index bc02ac6..0000000 --- a/fix/fix.go +++ /dev/null @@ -1,256 +0,0 @@ -package main - -import ( - "archive/tar" - "context" - "crypto/sha256" - "crypto/tls" - "encoding/json" - "errors" - "fmt" - "io" - "log" - "os" - "path/filepath" - "strings" - "time" - - "git.cheetah.cat/worksucc/gma-puzzles/chunk" - "git.cheetah.cat/worksucc/gma-puzzles/common" - adriver "github.com/arangodb/go-driver" - ahttp "github.com/arangodb/go-driver/http" - "github.com/klauspost/compress/zstd" -) - -var ( - arangoDB adriver.Database - arangoCTX context.Context - colChunk adriver.Collection - colFile adriver.Collection - colFile2Chunk adriver.Collection -) - -func ConnectDB(baseURL string, arangoUser string, arangoPWD string, arangoDatabase string) (driver adriver.Database, ctx context.Context, err error) { - log.Println("connectDB:", "Starting Connection Process...") - - // Retry Loop for Failed Connections - for i := 0; i < 6; i++ { - if i == 5 { - return driver, ctx, fmt.Errorf("connectdb unable to connect to database %d times", i) - } else if i > 0 { - time.Sleep(30 * time.Second) - } - - // Connect to ArangoDB URL - conn, err := ahttp.NewConnection(ahttp.ConnectionConfig{ - Endpoints: []string{baseURL}, - TLSConfig: &tls.Config{ /*...*/ }, - }) - if err != nil { - log.Println("connectDB:", "Cannot Connect to ArangoDB!", err) - continue - } - - // Connect Driver to User - client, err := adriver.NewClient(adriver.ClientConfig{ - Connection: conn, - Authentication: adriver.BasicAuthentication(arangoUser, arangoPWD), - }) - if err != nil { - log.Println("connectDB:", "Cannot Authenticate ArangoDB User!", err) - continue - } - - // Create Context for Database Access - ctx = context.Background() - driver, err = client.Database(ctx, arangoDatabase) - if err != nil { - log.Println("connectDB:", "Cannot Load ArangoDB Database!", err) - continue - } - log.Println("connectDB:", "Connection Sucessful!") - return driver, ctx, nil - } - - return driver, ctx, fmt.Errorf("connectDB: FUCK HOW DID THIS EXCUTE?") -} -func InitDatabase() (err error) { - arangoDB, arangoCTX, err = ConnectDB("http://192.168.45.8:8529/", "gma-inator", "gma-inator", "gma-inator") - if err != nil { - return err - } - - colChunk, err = arangoDB.Collection(arangoCTX, "chunk") - if err != nil { - return err - } - colFile, err = arangoDB.Collection(arangoCTX, "file") - if err != nil { - return err - } - colFile2Chunk, err = arangoDB.Collection(arangoCTX, "file_chunk_map") - if err != nil { - return err - } - return nil -} - -func bla() error { - entries, err := os.ReadDir("/mnt/SC9000/storagePools/") - if err != nil { - return err - } - for _, e := range entries { - if strings.Contains(e.Name(), ".json") { - continue - } - if !strings.Contains(e.Name(), "78d8553f-d716-47e3-ad21-c94f00c5e19c") { - continue - } - - if !e.IsDir() { - fmt.Printf("Scanning For Local Pools, found %s:", e.Name()) - - tarFinalPath := filepath.Join("/mnt/SC9000/storagePools/", e.Name()) - stats, err := os.Stat(tarFinalPath) - if err != nil { - return err - } - parts := strings.Split(e.Name(), ".") - var dboChunk common.DB_Chunk - _, err = colChunk.ReadDocument(arangoCTX, parts[0], &dboChunk) - if err != nil { - return err - } - dboChunk.Finalized = true - dboChunk.NotReady = false - dboChunk.ReadOnly = true - dboChunk.Size = stats.Size() - - zstFile, err := os.Open(tarFinalPath) - if err != nil { - return err - } - shaHasher := sha256.New() - _, err = io.Copy(shaHasher, zstFile) - if err != nil { - return err - } - - jsonPath := filepath.Join("/mnt/SC9000/storagePools/", fmt.Sprintf("%s.json", parts[0])) - fmt.Print(jsonPath) - _, err = os.Stat(jsonPath) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - fmt.Println("json 404") - // rewrite json from db - - zstFile.Seek(0, 0) - decompressor, err := zstd.NewReader(zstFile) - if err != nil { - panic(err) - } - defer decompressor.Close() - - items := []string{} - tarReader := tar.NewReader(decompressor) - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - return err - } - items = append(items, header.Name) - } - - poolRecoveryData := chunk.PoolRecoveryData{ - PoolID: parts[0], - Size: uint64(stats.Size()), - Created: time.Now(), - Hash: fmt.Sprintf("%x", shaHasher.Sum(nil)), - ItemCount: len(items), - Items: items, - //RecoveryData, - } - dboChunk.Hash = poolRecoveryData.Hash - dboChunk.FileCount = len(items) - - //TODO: fetch RecoveryData from DB - poolRecoveryData.RecoveryData = make([]common.DB_File, len(items)) - _, _, err = colFile.ReadDocuments(arangoCTX, items, poolRecoveryData.RecoveryData) - if err != nil { - return fmt.Errorf("error @ReadDocuments %v", err) - } - - json, err := json.MarshalIndent(poolRecoveryData, "", "\t") - if err != nil { - return fmt.Errorf("error @json.MarshalIndent %v", err) - } - - recoveryFile, err := os.Create(jsonPath) - if err != nil { - return err - } - _, err = recoveryFile.Write(json) - if err != nil { - return fmt.Errorf("error @recoveryFile.Write %v", err) - } - } - } else { - var poolRecoveryData chunk.PoolRecoveryData - - readJSONFile, err := os.Open(jsonPath) - if err != nil { - return err - } - defer readJSONFile.Close() - - readBytes, err := io.ReadAll(readJSONFile) - if err != nil { - return err - } - err = json.Unmarshal(readBytes, &poolRecoveryData) - if err != nil { - return err - } - - poolRecoveryData.Size = uint64(stats.Size()) - poolRecoveryData.Created = time.Now() - poolRecoveryData.Hash = fmt.Sprintf("%x", shaHasher.Sum(nil)) - dboChunk.Hash = poolRecoveryData.Hash - json, err := json.MarshalIndent(poolRecoveryData, "", "\t") - if err != nil { - return fmt.Errorf("error @json.MarshalIndent %v", err) - } - recoveryFile, err := os.Create(jsonPath) - if err != nil { - return err - } - _, err = recoveryFile.Write(json) - if err != nil { - return fmt.Errorf("error @recoveryFile.Write %v", err) - } - } - _, err = colChunk.UpdateDocument(arangoCTX, parts[0], &dboChunk) - if err != nil { - return err - } - fmt.Printf(":%d\n", dboChunk.FileCount) - } - } - - return nil -} -func main() { - err := InitDatabase() - if err != nil { - panic(err) - } - - err = bla() - if err != nil { - panic(err) - } -} diff --git a/zstd-tar-test/tartest.go b/zstd-tar-test/tartest.go deleted file mode 100644 index 59505f4..0000000 --- a/zstd-tar-test/tartest.go +++ /dev/null @@ -1,76 +0,0 @@ -package main - -import ( - "archive/tar" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/klauspost/compress/zstd" -) - -func main() { - tarFile, err := os.Create("test.tar.zst") - if err != nil { - panic(err) - } - defer tarFile.Close() - - compressor, err := zstd.NewWriter(tarFile, zstd.WithEncoderLevel(4)) - if err != nil { - panic(err) - } - defer compressor.Close() - - tw := tar.NewWriter(compressor) - defer tw.Close() - - entries, err := os.ReadDir("/home/cheetah/dev/gma-puzzles/zstd-tar-test/testpayload") - if err != nil { - panic(err) - } - - for _, e := range entries { - originalPath := filepath.Join("/home/cheetah/dev/gma-puzzles/zstd-tar-test/testpayload", e.Name()) - - file, err := os.Open(originalPath) - if err != nil { - panic(err) - } - defer file.Close() - - info, err := file.Stat() - if err != nil { - panic(err) - } - - tarFileHeader, err := tar.FileInfoHeader(info, info.Name()) - if err != nil { - panic(err) - } - err = tw.WriteHeader(tarFileHeader) - if err != nil { - panic(err) - } - - _, err = io.Copy(tw, file) - if err != nil { - panic(err) - } - fmt.Println(info.Name()) - } - - err = tw.Close() - if err != nil { - panic(err) - } - err = compressor.Close() - if err != nil { - panic(err) - } - err = tarFile.Close() - if err != nil { - panic(err) - } -}