|
|
|
package common
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"mime/multipart"
|
|
|
|
"net/http"
|
|
|
|
"os"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"git.cheetah.cat/worksucc/gma-puzzles/gma"
|
|
|
|
)
|
|
|
|
|
|
|
|
type DB_GMA struct {
|
|
|
|
ID string `json:"_key"`
|
|
|
|
BatchID string `json:"batch"`
|
|
|
|
|
|
|
|
ProcessingStart time.Time `json:"processingStart"`
|
|
|
|
ProcessingEnd time.Time `json:"processingEnd"`
|
|
|
|
ProcessingDuration int64 `json:"processingDuration"`
|
|
|
|
|
|
|
|
OriginalPath string `json:"originalPath"`
|
|
|
|
StatModTime time.Time `json:"statModTime"`
|
|
|
|
GMASize int64 `json:"archiveSize"`
|
|
|
|
OptimizedSize int64 `json:"gmasmSize"`
|
|
|
|
GMAHash string `json:"archiveHash"`
|
|
|
|
FooterAddonCRC uint32 `json:"footerCRC"`
|
|
|
|
Header gma.GMAHeader `json:"header"`
|
|
|
|
FirstType int32 `json:"firstType"`
|
|
|
|
Success bool `json:"success"`
|
|
|
|
RetryCounter int `json:"retries"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type DB_File struct {
|
|
|
|
ID string `json:"_key"`
|
|
|
|
BatchID string `json:"batch"`
|
|
|
|
InitialPath string `json:"initialPath"`
|
|
|
|
Extension string `json:"extension"`
|
|
|
|
Created time.Time `json:"created"`
|
|
|
|
Size int64 `json:"size"`
|
|
|
|
CRC uint32 `json:"crc"`
|
|
|
|
Hash string `json:"hash"`
|
|
|
|
G2FRef string `json:"-"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type DB_GMA2File struct {
|
|
|
|
ID string `json:"_key"`
|
|
|
|
BatchID string `json:"batch"`
|
|
|
|
File string `json:"_to"`
|
|
|
|
GMA string `json:"_from"`
|
|
|
|
|
|
|
|
FileNumber int32 `json:"fileNumber"`
|
|
|
|
FileName string `json:"fileName"`
|
|
|
|
Offset int64 `json:"offset"`
|
|
|
|
FileSize int64 `json:"size"`
|
|
|
|
CRC uint32 `json:"crc"`
|
|
|
|
CRCMatch bool `json:"crcMatch"`
|
|
|
|
NextType uint32 `json:"nextType"`
|
|
|
|
|
|
|
|
LocalFileName string `json:"-"`
|
|
|
|
UploadID string `json:"-"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type DB_Chunk struct {
|
|
|
|
ID string `json:"_key"`
|
|
|
|
|
|
|
|
NotReady bool `json:"notReady"`
|
|
|
|
Finalized bool `json:"finalized"`
|
|
|
|
ReadOnly bool `json:"readOnly"`
|
|
|
|
Created time.Time `json:"created"`
|
|
|
|
|
|
|
|
FileCount int `json:"fileCount"`
|
|
|
|
Size int64 `json:"size"`
|
|
|
|
Hash string `json:"hash"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type DB_File2Chunk struct {
|
|
|
|
ID string `json:"_key"`
|
|
|
|
Chunk string `json:"_to"`
|
|
|
|
File string `json:"_from"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type JSON_GMARecovery struct {
|
|
|
|
GMA DB_GMA `json:"gma"`
|
|
|
|
Refs []DB_GMA2File `json:"refs"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func MultipartUpload(client *http.Client, url string, path string, jsonBytes []byte, workerID string) (err error) {
|
|
|
|
//fmt.Printf("\nMultipartUpload(%s, %s)\n", url, path)
|
|
|
|
file, err := os.Open(path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
fileContents, err := io.ReadAll(file)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
fi, err := file.Stat()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
file.Close()
|
|
|
|
|
|
|
|
//body := new(bytes.Buffer)
|
|
|
|
pr, pw := io.Pipe()
|
|
|
|
writer := multipart.NewWriter(pw)
|
|
|
|
go func() {
|
|
|
|
defer pw.Close()
|
|
|
|
err = writer.WriteField("info", string(jsonBytes))
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
err = writer.WriteField("worker", fmt.Sprintf("gma-%s", workerID))
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
part, err := writer.CreateFormFile("file", fi.Name())
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
part.Write(fileContents)
|
|
|
|
|
|
|
|
//defer body.Reset()
|
|
|
|
|
|
|
|
/*for key, val := range params {
|
|
|
|
_ = writer.WriteField(key, val)
|
|
|
|
}*/
|
|
|
|
err = writer.Close()
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
req, err := http.NewRequest("POST", url, pr)
|
|
|
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Submit the request
|
|
|
|
res, err := client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check the response
|
|
|
|
if res.StatusCode == http.StatusAlreadyReported {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if res.StatusCode != http.StatusOK {
|
|
|
|
return fmt.Errorf("bad status: %s", res.Status)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Discard Response Body, to clean up tcp socket
|
|
|
|
defer res.Body.Close()
|
|
|
|
_, err = io.Copy(io.Discard, res.Body)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
func MoveFile(sourcePath, destPath string) error {
|
|
|
|
inputFile, err := os.Open(sourcePath)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("couldn't open source file: %s", err)
|
|
|
|
}
|
|
|
|
outputFile, err := os.Create(destPath)
|
|
|
|
if err != nil {
|
|
|
|
inputFile.Close()
|
|
|
|
return fmt.Errorf("couldn't open dest file: %s", err)
|
|
|
|
}
|
|
|
|
defer outputFile.Close()
|
|
|
|
_, err = io.Copy(outputFile, inputFile)
|
|
|
|
inputFile.Close()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("writing to output file failed: %s", err)
|
|
|
|
}
|
|
|
|
// The copy was successful, so now delete the original file
|
|
|
|
err = os.Remove(sourcePath)
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("failed removing original file: %s", err)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
type Semaphore interface {
|
|
|
|
Acquire()
|
|
|
|
Release()
|
|
|
|
Close()
|
|
|
|
}
|
|
|
|
|
|
|
|
type semaphore struct {
|
|
|
|
semC chan struct{}
|
|
|
|
}
|
|
|
|
|
|
|
|
func NewSemaphore(maxConcurrency int) Semaphore {
|
|
|
|
return &semaphore{
|
|
|
|
semC: make(chan struct{}, maxConcurrency),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *semaphore) Acquire() {
|
|
|
|
s.semC <- struct{}{}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *semaphore) Release() {
|
|
|
|
<-s.semC
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *semaphore) Close() {
|
|
|
|
close(s.semC)
|
|
|
|
}
|