package common import ( "fmt" "io" "mime/multipart" "net/http" "os" "path" "strings" "time" "git.cheetah.cat/worksucc/gma-puzzles/gma" ) type DB_GMA struct { ID string `json:"_key"` BatchID string `json:"batch"` ProcessingStart time.Time `json:"processingStart"` ProcessingEnd time.Time `json:"processingEnd"` ProcessingDuration int64 `json:"processingDuration"` MigrationID string `json:"migrationID"` OriginalPath string `json:"originalPath"` StatModTime time.Time `json:"statModTime"` GMASize int64 `json:"archiveSize"` OptimizedSize int64 `json:"gmasmSize"` GMAHash string `json:"archiveHash"` FooterAddonCRC uint32 `json:"footerCRC"` Header gma.GMAHeader `json:"header"` FirstType int32 `json:"firstType"` Success bool `json:"success"` RetryCounter int `json:"retries"` } type DB_GMA_Alias struct { MigrationID string `json:"migrationID"` Path string `json:"path"` Hash string `json:"hash"` Deleted bool `json:"deleted"` } type DB_GMADeletionData struct { Deleted bool `json:"deleted"` } type DB_File struct { ID string `json:"_key"` BatchID string `json:"batch"` InitialPath string `json:"initialPath"` Extension string `json:"extension"` Created time.Time `json:"created"` Size int64 `json:"size"` CRC uint32 `json:"crc"` Hash string `json:"hash"` G2FRef string `json:"-"` } type DB_GMA2File struct { ID string `json:"_key"` BatchID string `json:"batch"` File string `json:"_to"` GMA string `json:"_from"` FileNumber int32 `json:"fileNumber"` FileName string `json:"fileName"` Offset int64 `json:"offset"` FileSize int64 `json:"size"` CRC uint32 `json:"crc"` CRCMatch bool `json:"crcMatch"` NextType uint32 `json:"nextType"` LocalFileName string `json:"-"` UploadID string `json:"-"` } type DB_Chunk struct { ID string `json:"_key"` NotReady bool `json:"notReady"` Finalized bool `json:"finalized"` ReadOnly bool `json:"readOnly"` Created time.Time `json:"created"` FileCount int `json:"fileCount"` Size int64 `json:"size"` Hash string `json:"hash"` } type DB_File2Chunk struct { ID string `json:"_key"` Chunk string `json:"_to"` File string `json:"_from"` } type JSON_GMARecovery struct { GMA DB_GMA `json:"gma"` Refs []DB_GMA2File `json:"refs"` } func MultipartUpload(client *http.Client, url string, path string, jsonBytes []byte, workerID string) (err error) { //fmt.Printf("\nMultipartUpload(%s, %s)\n", url, path) file, err := os.Open(path) if err != nil { return err } fileContents, err := io.ReadAll(file) if err != nil { return err } fi, err := file.Stat() if err != nil { return err } file.Close() //body := new(bytes.Buffer) pr, pw := io.Pipe() writer := multipart.NewWriter(pw) go func() { defer pw.Close() err = writer.WriteField("info", string(jsonBytes)) if err != nil { return } err = writer.WriteField("worker", fmt.Sprintf("gma-%s", workerID)) if err != nil { return } part, err := writer.CreateFormFile("file", fi.Name()) if err != nil { return } part.Write(fileContents) //defer body.Reset() /*for key, val := range params { _ = writer.WriteField(key, val) }*/ err = writer.Close() if err != nil { return } }() req, err := http.NewRequest("POST", url, pr) req.Header.Set("Content-Type", writer.FormDataContentType()) if err != nil { return err } // Submit the request res, err := client.Do(req) if err != nil { return err } // Check the response if res.StatusCode == http.StatusAlreadyReported { return } if res.StatusCode != http.StatusOK { return fmt.Errorf("bad status: %s", res.Status) } // Discard Response Body, to clean up tcp socket defer res.Body.Close() _, err = io.Copy(io.Discard, res.Body) if err != nil { return err } return } func MoveFile(sourcePath, destPath string) error { inputFile, err := os.Open(sourcePath) if err != nil { return fmt.Errorf("couldn't open source file: %s", err) } outputFile, err := os.Create(destPath) if err != nil { inputFile.Close() return fmt.Errorf("couldn't open dest file: %s", err) } defer outputFile.Close() _, err = io.Copy(outputFile, inputFile) inputFile.Close() if err != nil { return fmt.Errorf("writing to output file failed: %s", err) } // The copy was successful, so now delete the original file err = os.Remove(sourcePath) if err != nil { return fmt.Errorf("failed removing original file: %s", err) } return nil } type Semaphore interface { Acquire() Release() Close() } type semaphore struct { semC chan struct{} } func NewSemaphore(maxConcurrency int) Semaphore { return &semaphore{ semC: make(chan struct{}, maxConcurrency), } } func (s *semaphore) Acquire() { s.semC <- struct{}{} } func (s *semaphore) Release() { <-s.semC } func (s *semaphore) Close() { close(s.semC) } func CreateMigrationID(filePath string) string { return strings.ReplaceAll(strings.ReplaceAll(path.Base(filePath), ".gma", ""), ".", "_") }