|
|
@ -3,7 +3,9 @@ package main
|
|
|
|
import (
|
|
|
|
import (
|
|
|
|
"archive/tar"
|
|
|
|
"archive/tar"
|
|
|
|
"context"
|
|
|
|
"context"
|
|
|
|
|
|
|
|
"crypto/sha256"
|
|
|
|
"crypto/tls"
|
|
|
|
"crypto/tls"
|
|
|
|
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io"
|
|
|
|
"log"
|
|
|
|
"log"
|
|
|
@ -12,9 +14,11 @@ import (
|
|
|
|
"path"
|
|
|
|
"path"
|
|
|
|
"path/filepath"
|
|
|
|
"path/filepath"
|
|
|
|
"strconv"
|
|
|
|
"strconv"
|
|
|
|
|
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"git.cheetah.cat/worksucc/gma-puzzles/common"
|
|
|
|
adriver "github.com/arangodb/go-driver"
|
|
|
|
adriver "github.com/arangodb/go-driver"
|
|
|
|
ahttp "github.com/arangodb/go-driver/http"
|
|
|
|
ahttp "github.com/arangodb/go-driver/http"
|
|
|
|
"github.com/labstack/echo/v4"
|
|
|
|
"github.com/labstack/echo/v4"
|
|
|
@ -32,13 +36,15 @@ type Pool struct {
|
|
|
|
Finalized bool `json:"finalized"`
|
|
|
|
Finalized bool `json:"finalized"`
|
|
|
|
ReadOnly bool `json:"readOnly"`
|
|
|
|
ReadOnly bool `json:"readOnly"`
|
|
|
|
Size uint64 `json:"size"`
|
|
|
|
Size uint64 `json:"size"`
|
|
|
|
folder string `json:"-"`
|
|
|
|
//folder string `json:"-"`
|
|
|
|
|
|
|
|
|
|
|
|
itemCount int
|
|
|
|
itemCount int
|
|
|
|
items []string
|
|
|
|
items []string
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
wormMode bool
|
|
|
|
|
|
|
|
filePath string
|
|
|
|
file *os.File
|
|
|
|
file *os.File
|
|
|
|
tarWriter *tar.Writer
|
|
|
|
//tarWriter *tar.Writer
|
|
|
|
tarReader *tar.Reader
|
|
|
|
tarReader *tar.Reader
|
|
|
|
}
|
|
|
|
}
|
|
|
|
type PoolFile struct {
|
|
|
|
type PoolFile struct {
|
|
|
@ -54,11 +60,154 @@ type PoolMaster struct {
|
|
|
|
|
|
|
|
|
|
|
|
LocalPools []Pool
|
|
|
|
LocalPools []Pool
|
|
|
|
FullPools []Pool
|
|
|
|
FullPools []Pool
|
|
|
|
|
|
|
|
WORMPools map[string]Pool
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
type PoolPackResult struct {
|
|
|
|
|
|
|
|
PoolID string
|
|
|
|
|
|
|
|
Files []string
|
|
|
|
|
|
|
|
FileCount int
|
|
|
|
|
|
|
|
Size int64
|
|
|
|
|
|
|
|
Hash string
|
|
|
|
|
|
|
|
outputFileName string
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var (
|
|
|
|
|
|
|
|
arangoDB adriver.Database
|
|
|
|
|
|
|
|
arangoCTX context.Context
|
|
|
|
|
|
|
|
colChunk adriver.Collection
|
|
|
|
|
|
|
|
colFile adriver.Collection
|
|
|
|
|
|
|
|
colFile2Chunk adriver.Collection
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// PoolMaster
|
|
|
|
|
|
|
|
poolMaster PoolMaster
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func ConnectDB(baseURL string, arangoUser string, arangoPWD string, arangoDatabase string) (driver adriver.Database, ctx context.Context, err error) {
|
|
|
|
|
|
|
|
log.Println("connectDB:", "Starting Connection Process...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Retry Loop for Failed Connections
|
|
|
|
|
|
|
|
for i := 0; i < 6; i++ {
|
|
|
|
|
|
|
|
if i == 5 {
|
|
|
|
|
|
|
|
return driver, ctx, fmt.Errorf("connectdb: unable to connect to database %d times!", i)
|
|
|
|
|
|
|
|
} else if i > 0 {
|
|
|
|
|
|
|
|
time.Sleep(30 * time.Second)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Connect to ArangoDB URL
|
|
|
|
|
|
|
|
conn, err := ahttp.NewConnection(ahttp.ConnectionConfig{
|
|
|
|
|
|
|
|
Endpoints: []string{baseURL},
|
|
|
|
|
|
|
|
TLSConfig: &tls.Config{ /*...*/ },
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
log.Println("connectDB:", "Cannot Connect to ArangoDB!", err)
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Connect Driver to User
|
|
|
|
|
|
|
|
client, err := adriver.NewClient(adriver.ClientConfig{
|
|
|
|
|
|
|
|
Connection: conn,
|
|
|
|
|
|
|
|
Authentication: adriver.BasicAuthentication(arangoUser, arangoPWD),
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
log.Println("connectDB:", "Cannot Authenticate ArangoDB User!", err)
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Create Context for Database Access
|
|
|
|
|
|
|
|
ctx = context.Background()
|
|
|
|
|
|
|
|
driver, err = client.Database(ctx, arangoDatabase)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
log.Println("connectDB:", "Cannot Load ArangoDB Database!", err)
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Println("connectDB:", "Connection Sucessful!")
|
|
|
|
|
|
|
|
return driver, ctx, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return driver, ctx, fmt.Errorf("connectDB: FUCK HOW DID THIS EXCUTE?")
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func InitDatabase() (err error) {
|
|
|
|
|
|
|
|
arangoDB, arangoCTX, err = ConnectDB("http://192.168.45.8:8529/", "gma-inator", "gma-inator", "gma-inator")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
colChunk, err = arangoDB.Collection(arangoCTX, "chunk")
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
colFile, err = arangoDB.Collection(arangoCTX, "file")
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
colFile2Chunk, err = arangoDB.Collection(arangoCTX, "file_chunk_map")
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func (p *Pool) OpenTar() (err error) {
|
|
|
|
|
|
|
|
p.wormMode = true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
outputDir := filepath.Join(poolMaster.cachePath, "worm", p.PoolID)
|
|
|
|
|
|
|
|
err = os.MkdirAll(outputDir, os.ModePerm)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
p.file, err = os.Open(p.filePath)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
p.items = []string{}
|
|
|
|
|
|
|
|
p.tarReader = tar.NewReader(p.file)
|
|
|
|
|
|
|
|
for {
|
|
|
|
|
|
|
|
header, err := p.tarReader.Next()
|
|
|
|
|
|
|
|
if err == io.EOF {
|
|
|
|
|
|
|
|
break
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
path := filepath.Join(outputDir, header.Name)
|
|
|
|
|
|
|
|
info := header.FileInfo()
|
|
|
|
|
|
|
|
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
defer file.Close()
|
|
|
|
|
|
|
|
_, err = io.Copy(file, p.tarReader)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
p.items = append(p.items, header.Name)
|
|
|
|
|
|
|
|
fmt.Print(".")
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *Pool) Fetch(id string, writer io.Writer) (err error) {
|
|
|
|
|
|
|
|
for _, poolItem := range p.items {
|
|
|
|
|
|
|
|
if poolItem == id {
|
|
|
|
|
|
|
|
fmt.Printf("Fetch WORMPool %s\n", id)
|
|
|
|
|
|
|
|
poolLocalFilePath := filepath.Join(poolMaster.cachePath, "worm", p.PoolID, id)
|
|
|
|
|
|
|
|
srcLocalFile, err := os.Open(poolLocalFilePath)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
defer srcLocalFile.Close()
|
|
|
|
|
|
|
|
if _, err = io.Copy(writer, srcLocalFile); err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return fmt.Errorf("%s not found", id)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func NewPoolMaster(finalPath string, cachePath string) (poolMaster PoolMaster, err error) {
|
|
|
|
func NewPoolMaster(finalPath string, cachePath string) (poolMaster PoolMaster, err error) {
|
|
|
|
poolMaster.finalPath = finalPath
|
|
|
|
poolMaster.finalPath = finalPath
|
|
|
|
poolMaster.cachePath = cachePath
|
|
|
|
poolMaster.cachePath = cachePath
|
|
|
|
|
|
|
|
poolMaster.WORMPools = make(map[string]Pool)
|
|
|
|
//poolMaster.lock = sync.Mutex{}
|
|
|
|
//poolMaster.lock = sync.Mutex{}
|
|
|
|
|
|
|
|
|
|
|
|
destPath := filepath.Join(poolMaster.cachePath, "pool")
|
|
|
|
destPath := filepath.Join(poolMaster.cachePath, "pool")
|
|
|
@ -66,6 +215,13 @@ func NewPoolMaster(finalPath string, cachePath string) (poolMaster PoolMaster, e
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return poolMaster, err
|
|
|
|
return poolMaster, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
destPath = filepath.Join(poolMaster.cachePath, "worm")
|
|
|
|
|
|
|
|
err = os.MkdirAll(destPath, os.ModePerm)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return poolMaster, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
err = os.MkdirAll(poolMaster.finalPath, os.ModePerm)
|
|
|
|
err = os.MkdirAll(poolMaster.finalPath, os.ModePerm)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return poolMaster, err
|
|
|
|
return poolMaster, err
|
|
|
@ -89,12 +245,14 @@ func (p *PoolMaster) NewPool() (pool *Pool, err error) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (p *PoolMaster) GetCurrentWriteablePool() (pool *Pool, err error) {
|
|
|
|
func (p *PoolMaster) GetCurrentWriteablePool() (pool *Pool, err error) {
|
|
|
|
|
|
|
|
//fmt.Printf("Current Pool %s, ItemCount = %d\n", pool.PoolID, pool.itemCount)
|
|
|
|
if p.CurrentPool != nil && p.CurrentPool.itemCount >= PoolMaxItems {
|
|
|
|
if p.CurrentPool != nil && p.CurrentPool.itemCount >= PoolMaxItems {
|
|
|
|
p.lock.Lock()
|
|
|
|
p.lock.Lock()
|
|
|
|
defer p.lock.Unlock()
|
|
|
|
defer p.lock.Unlock()
|
|
|
|
p.CurrentPool.ReadOnly = true
|
|
|
|
p.CurrentPool.ReadOnly = true
|
|
|
|
p.FullPools = append(p.FullPools, *p.CurrentPool)
|
|
|
|
p.FullPools = append(p.FullPools, *p.CurrentPool)
|
|
|
|
// queue for compression
|
|
|
|
// queue for compression
|
|
|
|
|
|
|
|
fmt.Printf("GetCurrentWriteablePool(): current Pool (%s) is full (%d), creating new one", p.CurrentPool.PoolID, p.CurrentPool.itemCount)
|
|
|
|
p.CurrentPool = nil
|
|
|
|
p.CurrentPool = nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if p.CurrentPool == nil {
|
|
|
|
if p.CurrentPool == nil {
|
|
|
@ -135,7 +293,37 @@ func (p *PoolMaster) ScanForLocalPools() (err error) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for _, e := range entries {
|
|
|
|
for _, e := range entries {
|
|
|
|
if e.IsDir() {
|
|
|
|
if e.IsDir() {
|
|
|
|
fmt.Printf("Scanning For Local Pools, found %s:\n", e.Name())
|
|
|
|
fmt.Printf("Scanning For Local Pools, found %s:", e.Name())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tarFinalPath := filepath.Join(p.finalPath, fmt.Sprintf("%s.tar", e.Name()))
|
|
|
|
|
|
|
|
_, err = os.Stat(tarFinalPath)
|
|
|
|
|
|
|
|
finalPathExists := false
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
if !errors.Is(err, os.ErrNotExist) {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
dboChunkExists, err := colChunk.DocumentExists(arangoCTX, e.Name())
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
if dboChunkExists {
|
|
|
|
|
|
|
|
var dboChunk common.DB_Chunk
|
|
|
|
|
|
|
|
_, err := colChunk.ReadDocument(arangoCTX, e.Name(), &dboChunk)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
finalPathExists = dboChunk.Finalized && dboChunk.ReadOnly && !dboChunk.NotReady
|
|
|
|
|
|
|
|
fmt.Printf("is in DB readonly %v finalized %v notready %v itemCount=%d size=%d hash=%s\n", dboChunk.ReadOnly, dboChunk.Finalized, dboChunk.NotReady, dboChunk.FileCount, dboChunk.Size, dboChunk.Hash)
|
|
|
|
|
|
|
|
if finalPathExists {
|
|
|
|
|
|
|
|
fmt.Println("skipping")
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if finalPathExists {
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
poolDirPath := filepath.Join(p.cachePath, "pool", e.Name())
|
|
|
|
poolDirPath := filepath.Join(p.cachePath, "pool", e.Name())
|
|
|
|
restoredPool, err := RestorePoolFromFolder(poolDirPath)
|
|
|
|
restoredPool, err := RestorePoolFromFolder(poolDirPath)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
@ -147,6 +335,189 @@ func (p *PoolMaster) ScanForLocalPools() (err error) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Pool Packing
|
|
|
|
|
|
|
|
func (p *PoolMaster) ImportPoolPackResult(packResult PoolPackResult) (err error) {
|
|
|
|
|
|
|
|
startTime := time.Now()
|
|
|
|
|
|
|
|
dboChunk := common.DB_Chunk{
|
|
|
|
|
|
|
|
ID: packResult.PoolID,
|
|
|
|
|
|
|
|
Hash: packResult.Hash,
|
|
|
|
|
|
|
|
Size: packResult.Size,
|
|
|
|
|
|
|
|
FileCount: packResult.FileCount,
|
|
|
|
|
|
|
|
NotReady: true,
|
|
|
|
|
|
|
|
ReadOnly: true,
|
|
|
|
|
|
|
|
Finalized: true,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
var dboChunk2File []common.DB_File2Chunk
|
|
|
|
|
|
|
|
for _, prFile := range packResult.Files {
|
|
|
|
|
|
|
|
dboChunk2File = append(dboChunk2File, common.DB_File2Chunk{
|
|
|
|
|
|
|
|
ID: prFile,
|
|
|
|
|
|
|
|
File: fmt.Sprintf("file/%s", prFile),
|
|
|
|
|
|
|
|
Chunk: fmt.Sprintf("chunk/%s", dboChunk.ID),
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
_, err = colChunk.CreateDocument(arangoCTX, dboChunk)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
chunkSize := 500
|
|
|
|
|
|
|
|
for {
|
|
|
|
|
|
|
|
if len(dboChunk2File) == 0 {
|
|
|
|
|
|
|
|
break
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// necessary check to avoid slicing beyond
|
|
|
|
|
|
|
|
// slice capacity
|
|
|
|
|
|
|
|
if len(dboChunk2File) < chunkSize {
|
|
|
|
|
|
|
|
chunkSize = len(dboChunk2File)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
_, errorSlice, _ := colFile2Chunk.CreateDocuments(arangoCTX, dboChunk2File[0:chunkSize])
|
|
|
|
|
|
|
|
//metaSlice, errorSlice, _ := colFile2Chunk.CreateDocuments(arangoCTX, dboChunk2File[0:chunkSize])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
//fmt.Println("Metaslice")
|
|
|
|
|
|
|
|
//fmt.Println(metaSlice)
|
|
|
|
|
|
|
|
/*for _, meta := range metaSlice {
|
|
|
|
|
|
|
|
if !meta.ID.IsEmpty() {
|
|
|
|
|
|
|
|
newUnknownFiles = append(newUnknownFiles, meta.Key)
|
|
|
|
|
|
|
|
fileIDs = append(fileIDs, meta.Key)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}*/
|
|
|
|
|
|
|
|
for _, createError := range errorSlice {
|
|
|
|
|
|
|
|
if createError != nil && strings.Contains(createError.Error(), "unique constraint violated - in index primary of type primary over '_key'") {
|
|
|
|
|
|
|
|
} else if createError != nil {
|
|
|
|
|
|
|
|
return createError
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dboChunk2File = dboChunk2File[chunkSize:]
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fmt.Printf("ImportPool Duration %dms\n", time.Since(startTime).Milliseconds())
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *PoolMaster) MovePoolPackToWORM(packResult PoolPackResult) (err error) {
|
|
|
|
|
|
|
|
startTime := time.Now()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
targetFileName := filepath.Join(p.finalPath, fmt.Sprintf("%s.tar", packResult.PoolID))
|
|
|
|
|
|
|
|
os.Rename(packResult.outputFileName, targetFileName)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tarFileCheck, err := os.Open(targetFileName)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
defer tarFileCheck.Close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
shaHasher := sha256.New()
|
|
|
|
|
|
|
|
_, err = io.Copy(shaHasher, tarFileCheck)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
wormHash := fmt.Sprintf("%x", shaHasher.Sum(nil))
|
|
|
|
|
|
|
|
fmt.Printf("WORMTarPool hash is %s , old is %s\n", wormHash, packResult.Hash)
|
|
|
|
|
|
|
|
if wormHash != packResult.Hash {
|
|
|
|
|
|
|
|
os.Remove(targetFileName)
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
fmt.Printf("MoveWORM Duration %dms\n", time.Since(startTime).Milliseconds())
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *PoolMaster) PackPool(poolID string) (packResult PoolPackResult, err error) {
|
|
|
|
|
|
|
|
startTime := time.Now()
|
|
|
|
|
|
|
|
packResult.PoolID = poolID
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
p.lock.Lock()
|
|
|
|
|
|
|
|
defer p.lock.Unlock()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
packResult.outputFileName = filepath.Join(p.cachePath, "pool", fmt.Sprintf("%s.tar", poolID))
|
|
|
|
|
|
|
|
tarFile, err := os.Create(packResult.outputFileName)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
defer tarFile.Close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tw := tar.NewWriter(tarFile)
|
|
|
|
|
|
|
|
defer tw.Close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
entries, err := os.ReadDir(filepath.Join(p.cachePath, "pool", poolID))
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
//fmt.Printf("len(entries) == %d\n", len(entries))
|
|
|
|
|
|
|
|
if len(entries) != PoolMaxItems {
|
|
|
|
|
|
|
|
return packResult, fmt.Errorf("Pool contains %d items, but there should be %d", len(entries), PoolMaxItems)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, e := range entries {
|
|
|
|
|
|
|
|
originalPath := filepath.Join(p.cachePath, "pool", poolID, e.Name())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
file, err := os.Open(originalPath)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
defer file.Close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
info, err := file.Stat()
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tarFileHeader, err := tar.FileInfoHeader(info, info.Name())
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
err = tw.WriteHeader(tarFileHeader)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
_, err = io.Copy(tw, file)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
packResult.FileCount++
|
|
|
|
|
|
|
|
packResult.Files = append(packResult.Files, e.Name())
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
err = tw.Flush()
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
err = tarFile.Close()
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// re-open and check
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tarFileCheck, err := os.Open(packResult.outputFileName)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
defer tarFileCheck.Close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
shaHasher := sha256.New()
|
|
|
|
|
|
|
|
hashedBytes, err := io.Copy(shaHasher, tarFileCheck)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
packResult.Hash = fmt.Sprintf("%x", shaHasher.Sum(nil))
|
|
|
|
|
|
|
|
fmt.Printf("PackPoolTar hash is %s\n", packResult.Hash)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
packFileStats, err := tarFileCheck.Stat()
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return packResult, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
packResult.Size = packFileStats.Size()
|
|
|
|
|
|
|
|
if hashedBytes != packResult.Size {
|
|
|
|
|
|
|
|
return packResult, fmt.Errorf("WORM Copy HashedBytes %d != FileSize %d", hashedBytes, packResult.Size)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
fmt.Printf("PackPool Duration %dms\n", time.Since(startTime).Milliseconds())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return packResult, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (p *PoolMaster) AcquireNewOrRecoverPool() (pool *Pool, err error) {
|
|
|
|
func (p *PoolMaster) AcquireNewOrRecoverPool() (pool *Pool, err error) {
|
|
|
|
// p.NewPool()
|
|
|
|
// p.NewPool()
|
|
|
|
for _, localPool := range p.LocalPools {
|
|
|
|
for _, localPool := range p.LocalPools {
|
|
|
@ -156,22 +527,31 @@ func (p *PoolMaster) AcquireNewOrRecoverPool() (pool *Pool, err error) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return p.NewPool()
|
|
|
|
return p.NewPool()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (p *PoolMaster) Lookup(id string) (exists bool) {
|
|
|
|
func (p *PoolMaster) Lookup(id string) (exists bool) {
|
|
|
|
if p.CurrentPool != nil {
|
|
|
|
// TODO: DB check
|
|
|
|
|
|
|
|
if p.CurrentPool != nil { // CurrentPool
|
|
|
|
for _, poolItem := range p.CurrentPool.items {
|
|
|
|
for _, poolItem := range p.CurrentPool.items {
|
|
|
|
if poolItem == id {
|
|
|
|
if poolItem == id {
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for _, fullPool := range p.FullPools {
|
|
|
|
for _, wormPool := range p.WORMPools { // WORM Pools
|
|
|
|
|
|
|
|
for _, poolItem := range wormPool.items {
|
|
|
|
|
|
|
|
if poolItem == id {
|
|
|
|
|
|
|
|
return true
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, fullPool := range p.FullPools { // Full Pools
|
|
|
|
for _, poolItem := range fullPool.items {
|
|
|
|
for _, poolItem := range fullPool.items {
|
|
|
|
if poolItem == id {
|
|
|
|
if poolItem == id {
|
|
|
|
return true
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for _, localPool := range p.LocalPools {
|
|
|
|
for _, localPool := range p.LocalPools { // Local Pools
|
|
|
|
for _, poolItem := range localPool.items {
|
|
|
|
for _, poolItem := range localPool.items {
|
|
|
|
if poolItem == id {
|
|
|
|
if poolItem == id {
|
|
|
|
return true
|
|
|
|
return true
|
|
|
@ -179,8 +559,59 @@ func (p *PoolMaster) Lookup(id string) (exists bool) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// TODO : DB Check
|
|
|
|
// TODO : DB Check
|
|
|
|
|
|
|
|
// ArangoDB
|
|
|
|
|
|
|
|
dboFile2ChunkExists, err := colFile2Chunk.DocumentExists(arangoCTX, id)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return false
|
|
|
|
return false
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return dboFile2ChunkExists
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func (p *PoolMaster) FetchLoadWORM(chunkID string, fileID string, writer io.Writer) (err error) {
|
|
|
|
|
|
|
|
fmt.Printf("FetchLoadWORM(chunkID %s, fileID %s, ...)\n", chunkID, fileID)
|
|
|
|
|
|
|
|
// search within loaded worm-pools
|
|
|
|
|
|
|
|
for wormID, wormPool := range p.WORMPools {
|
|
|
|
|
|
|
|
if wormID != chunkID {
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, poolItem := range wormPool.items {
|
|
|
|
|
|
|
|
if poolItem == fileID {
|
|
|
|
|
|
|
|
fmt.Printf("Fetch WORMPool %s file %s\n", wormID, fileID)
|
|
|
|
|
|
|
|
return wormPool.Fetch(fileID, writer)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
break
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// else load wormPool into disk-cache extract to "worm"
|
|
|
|
|
|
|
|
// wormMode
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
p.lock.Lock()
|
|
|
|
|
|
|
|
defer p.lock.Unlock()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var dboChunk common.DB_Chunk
|
|
|
|
|
|
|
|
_, err = colChunk.ReadDocument(arangoCTX, chunkID, &dboChunk)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
loadedWormPool := Pool{
|
|
|
|
|
|
|
|
PoolID: dboChunk.ID,
|
|
|
|
|
|
|
|
Size: uint64(dboChunk.Size),
|
|
|
|
|
|
|
|
ReadOnly: dboChunk.ReadOnly,
|
|
|
|
|
|
|
|
Finalized: dboChunk.Finalized,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
filePath: filepath.Join(p.finalPath, fmt.Sprintf("%s.tar", dboChunk.ID)),
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
fmt.Println("initialized loadedWormPool, Opening tar...")
|
|
|
|
|
|
|
|
err = loadedWormPool.OpenTar()
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
fmt.Println("extracted")
|
|
|
|
|
|
|
|
p.WORMPools[loadedWormPool.PoolID] = loadedWormPool
|
|
|
|
|
|
|
|
return loadedWormPool.Fetch(fileID, writer)
|
|
|
|
|
|
|
|
//return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (p *PoolMaster) Fetch(id string, writer io.Writer) (err error) {
|
|
|
|
func (p *PoolMaster) Fetch(id string, writer io.Writer) (err error) {
|
|
|
|
if p.CurrentPool != nil {
|
|
|
|
if p.CurrentPool != nil {
|
|
|
|
for _, poolItem := range p.CurrentPool.items {
|
|
|
|
for _, poolItem := range p.CurrentPool.items {
|
|
|
@ -203,20 +634,24 @@ func (p *PoolMaster) Fetch(id string, writer io.Writer) (err error) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, wormPool := range p.WORMPools {
|
|
|
|
|
|
|
|
for _, poolItem := range wormPool.items {
|
|
|
|
|
|
|
|
if poolItem == id {
|
|
|
|
|
|
|
|
fmt.Printf("Fetch WORMPool %s file %s\n", wormPool.PoolID, id)
|
|
|
|
|
|
|
|
return wormPool.Fetch(id, writer)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
for _, fullPool := range p.FullPools {
|
|
|
|
for _, fullPool := range p.FullPools {
|
|
|
|
for _, poolItem := range fullPool.items {
|
|
|
|
for _, poolItem := range fullPool.items {
|
|
|
|
if poolItem == id {
|
|
|
|
if poolItem == id {
|
|
|
|
fmt.Printf("Fetch FullPool %s\n", id)
|
|
|
|
fmt.Printf("Fetch FullPool %s\n", id)
|
|
|
|
poolLocalFilePath := filepath.Join(p.cachePath, "pool", fullPool.PoolID, id)
|
|
|
|
poolLocalFilePath := filepath.Join(p.cachePath, "pool", fullPool.PoolID, id)
|
|
|
|
//fmt.Println(poolLocalFilePath)
|
|
|
|
|
|
|
|
//fmt.Printf("%s %s\n", fullPool.PoolID, poolItem)
|
|
|
|
|
|
|
|
srcLocalFile, err := os.Open(poolLocalFilePath)
|
|
|
|
srcLocalFile, err := os.Open(poolLocalFilePath)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
//fmt.Println("Closer")
|
|
|
|
|
|
|
|
defer srcLocalFile.Close()
|
|
|
|
defer srcLocalFile.Close()
|
|
|
|
//fmt.Println("io.Copy")
|
|
|
|
|
|
|
|
if _, err = io.Copy(writer, srcLocalFile); err != nil {
|
|
|
|
if _, err = io.Copy(writer, srcLocalFile); err != nil {
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -229,15 +664,11 @@ func (p *PoolMaster) Fetch(id string, writer io.Writer) (err error) {
|
|
|
|
if poolItem == id {
|
|
|
|
if poolItem == id {
|
|
|
|
fmt.Printf("Fetch LocalPool %s\n", id)
|
|
|
|
fmt.Printf("Fetch LocalPool %s\n", id)
|
|
|
|
poolLocalFilePath := filepath.Join(p.cachePath, "pool", localPool.PoolID, id)
|
|
|
|
poolLocalFilePath := filepath.Join(p.cachePath, "pool", localPool.PoolID, id)
|
|
|
|
//fmt.Println(poolLocalFilePath)
|
|
|
|
|
|
|
|
//fmt.Printf("%s %s\n", localPool.PoolID, poolItem)
|
|
|
|
|
|
|
|
srcLocalFile, err := os.Open(poolLocalFilePath)
|
|
|
|
srcLocalFile, err := os.Open(poolLocalFilePath)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
//fmt.Println("Closer")
|
|
|
|
|
|
|
|
defer srcLocalFile.Close()
|
|
|
|
defer srcLocalFile.Close()
|
|
|
|
//fmt.Println("io.Copy")
|
|
|
|
|
|
|
|
if _, err = io.Copy(writer, srcLocalFile); err != nil {
|
|
|
|
if _, err = io.Copy(writer, srcLocalFile); err != nil {
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -245,6 +676,24 @@ func (p *PoolMaster) Fetch(id string, writer io.Writer) (err error) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// ArangoDB
|
|
|
|
|
|
|
|
dboFile2ChunkExists, err := colFile2Chunk.DocumentExists(arangoCTX, id)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
fmt.Printf("dboFile2ChunkExists %s = %v\n", id, dboFile2ChunkExists)
|
|
|
|
|
|
|
|
if dboFile2ChunkExists {
|
|
|
|
|
|
|
|
var dboFile2Chunk common.DB_File2Chunk
|
|
|
|
|
|
|
|
_, err = colFile2Chunk.ReadDocument(arangoCTX, id, &dboFile2Chunk)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
//FetchFromPoolPack
|
|
|
|
|
|
|
|
//dboFile2Chunk.Chunk <- which chunk i need to find
|
|
|
|
|
|
|
|
return p.FetchLoadWORM(dboFile2Chunk.Chunk[6:], id, writer)
|
|
|
|
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
func (p *PoolMaster) Store(id string, src io.Reader, targetSize int64) (err error) {
|
|
|
|
func (p *PoolMaster) Store(id string, src io.Reader, targetSize int64) (err error) {
|
|
|
@ -253,8 +702,15 @@ func (p *PoolMaster) Store(id string, src io.Reader, targetSize int64) (err erro
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if pool.ReadOnly {
|
|
|
|
|
|
|
|
return fmt.Errorf("WTF Pool %s is ReadOnly but GetCurrentWriteablePool returned it", pool.PoolID)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fmt.Printf("Store(%s)\n", id)
|
|
|
|
|
|
|
|
|
|
|
|
p.lock.Lock()
|
|
|
|
p.lock.Lock()
|
|
|
|
defer p.lock.Unlock()
|
|
|
|
defer p.lock.Unlock()
|
|
|
|
|
|
|
|
// figuring out paths
|
|
|
|
poolFolder := filepath.Join(p.cachePath, "pool", pool.PoolID)
|
|
|
|
poolFolder := filepath.Join(p.cachePath, "pool", pool.PoolID)
|
|
|
|
destPath := filepath.Join(poolFolder, id)
|
|
|
|
destPath := filepath.Join(poolFolder, id)
|
|
|
|
dst, err := os.Create(destPath)
|
|
|
|
dst, err := os.Create(destPath)
|
|
|
@ -263,7 +719,7 @@ func (p *PoolMaster) Store(id string, src io.Reader, targetSize int64) (err erro
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
defer dst.Close()
|
|
|
|
defer dst.Close()
|
|
|
|
|
|
|
|
// copy from ioReader to file
|
|
|
|
writtenBytes, err := io.Copy(dst, src)
|
|
|
|
writtenBytes, err := io.Copy(dst, src)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
_ = os.Remove(destPath)
|
|
|
|
_ = os.Remove(destPath)
|
|
|
@ -273,6 +729,18 @@ func (p *PoolMaster) Store(id string, src io.Reader, targetSize int64) (err erro
|
|
|
|
_ = os.Remove(destPath)
|
|
|
|
_ = os.Remove(destPath)
|
|
|
|
return err
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// check transferred data
|
|
|
|
|
|
|
|
dst.Seek(0, 0)
|
|
|
|
|
|
|
|
shaHasher := sha256.New()
|
|
|
|
|
|
|
|
if _, err := io.Copy(shaHasher, dst); err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
outputHash := fmt.Sprintf("%x", shaHasher.Sum(nil))
|
|
|
|
|
|
|
|
if outputHash != id {
|
|
|
|
|
|
|
|
return fmt.Errorf("Store() Sha256 Hash Mismatch")
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pool.itemCount++
|
|
|
|
pool.itemCount++
|
|
|
|
pool.items = append(pool.items, id)
|
|
|
|
pool.items = append(pool.items, id)
|
|
|
|
fmt.Printf("Current Pool %s, ItemCount = %d\n", pool.PoolID, pool.itemCount)
|
|
|
|
fmt.Printf("Current Pool %s, ItemCount = %d\n", pool.PoolID, pool.itemCount)
|
|
|
@ -297,99 +765,40 @@ func (p *PoolMaster) Store(id string, src io.Reader, targetSize int64) (err erro
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
var (
|
|
|
|
func main() {
|
|
|
|
arangoDB adriver.Database
|
|
|
|
err := InitDatabase()
|
|
|
|
arangoCTX context.Context
|
|
|
|
|
|
|
|
colChunk adriver.Collection
|
|
|
|
|
|
|
|
colFile adriver.Collection
|
|
|
|
|
|
|
|
colFile2Chunk adriver.Collection
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func ConnectDB(baseURL string, arangoUser string, arangoPWD string, arangoDatabase string) (driver adriver.Database, ctx context.Context, err error) {
|
|
|
|
|
|
|
|
log.Println("connectDB:", "Starting Connection Process...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Retry Loop for Failed Connections
|
|
|
|
|
|
|
|
for i := 0; i < 6; i++ {
|
|
|
|
|
|
|
|
if i == 5 {
|
|
|
|
|
|
|
|
return driver, ctx, fmt.Errorf("connectdb: unable to connect to database %d times!", i)
|
|
|
|
|
|
|
|
} else if i > 0 {
|
|
|
|
|
|
|
|
time.Sleep(30 * time.Second)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Connect to ArangoDB URL
|
|
|
|
|
|
|
|
conn, err := ahttp.NewConnection(ahttp.ConnectionConfig{
|
|
|
|
|
|
|
|
Endpoints: []string{baseURL},
|
|
|
|
|
|
|
|
TLSConfig: &tls.Config{ /*...*/ },
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
log.Println("connectDB:", "Cannot Connect to ArangoDB!", err)
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Connect Driver to User
|
|
|
|
|
|
|
|
client, err := adriver.NewClient(adriver.ClientConfig{
|
|
|
|
|
|
|
|
Connection: conn,
|
|
|
|
|
|
|
|
Authentication: adriver.BasicAuthentication(arangoUser, arangoPWD),
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
log.Println("connectDB:", "Cannot Authenticate ArangoDB User!", err)
|
|
|
|
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Create Context for Database Access
|
|
|
|
|
|
|
|
ctx = context.Background()
|
|
|
|
|
|
|
|
driver, err = client.Database(ctx, arangoDatabase)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
log.Println("connectDB:", "Cannot Load ArangoDB Database!", err)
|
|
|
|
panic(err)
|
|
|
|
continue
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
log.Println("connectDB:", "Connection Sucessful!")
|
|
|
|
|
|
|
|
return driver, ctx, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
return driver, ctx, fmt.Errorf("connectDB: FUCK HOW DID THIS EXCUTE?")
|
|
|
|
poolMaster, err = NewPoolMaster(PoolPathFinal, PoolPathTemp)
|
|
|
|
}
|
|
|
|
|
|
|
|
func InitDatabase() (err error) {
|
|
|
|
|
|
|
|
arangoDB, arangoCTX, err = ConnectDB("http://192.168.45.8:8529/", "gma-inator", "gma-inator", "gma-inator")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
colChunk, err = arangoDB.Collection(arangoCTX, "chunk")
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
colFile, err = arangoDB.Collection(arangoCTX, "file")
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
colFile2Chunk, err = arangoDB.Collection(arangoCTX, "file_chunk_map")
|
|
|
|
// Scan for local existing Pools
|
|
|
|
|
|
|
|
err = poolMaster.ScanForLocalPools()
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var (
|
|
|
|
|
|
|
|
poolMaster PoolMaster
|
|
|
|
|
|
|
|
//poolFiles = []PoolFile{}
|
|
|
|
|
|
|
|
//seq = 1
|
|
|
|
|
|
|
|
//lock = sync.Mutex{}
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func main() {
|
|
|
|
for _, localPool := range poolMaster.LocalPools {
|
|
|
|
err := InitDatabase()
|
|
|
|
if localPool.ReadOnly {
|
|
|
|
|
|
|
|
fmt.Printf("Packing Pool %s\n", localPool.PoolID)
|
|
|
|
|
|
|
|
packResult, err := poolMaster.PackPool(localPool.PoolID)
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
err = poolMaster.ImportPoolPackResult(packResult)
|
|
|
|
poolMaster, err = NewPoolMaster(PoolPathFinal, PoolPathTemp)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Scan for local existing Pools
|
|
|
|
err = poolMaster.MovePoolPackToWORM(packResult)
|
|
|
|
err = poolMaster.ScanForLocalPools()
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
// packResult.FileCount
|
|
|
|
|
|
|
|
}
|
|
|
|
e := echo.New()
|
|
|
|
e := echo.New()
|
|
|
|
//e.Use(middleware.Logger())
|
|
|
|
//e.Use(middleware.Logger())
|
|
|
|
|
|
|
|
|
|
|
|