gma write/reader + common stuff
parent
429dc0a704
commit
b0941f1e38
@ -0,0 +1,130 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"git.cheetah.cat/worksucc/gma-puzzles/gma"
|
||||
)
|
||||
|
||||
type DB_GMA struct {
|
||||
ID string `json:"_key"`
|
||||
BatchID string `json:"batch"`
|
||||
|
||||
ProcessingStart time.Time `json:"processingStart"`
|
||||
ProcessingEnd time.Time `json:"processingEnd"`
|
||||
ProcessingDuration int64 `json:"processingDuration"`
|
||||
|
||||
OriginalPath string `json:"originalPath"`
|
||||
StatModTime time.Time `json:"statModTime"`
|
||||
GMASize int64 `json:"archiveSize"`
|
||||
OptimizedSize int64 `json:"gmasmSize"`
|
||||
GMAHash string `json:"archiveHash"`
|
||||
FooterAddonCRC uint32 `json:"footerCRC"`
|
||||
Header gma.GMAHeader `json:"header"`
|
||||
FirstType int32 `json:"firstType"`
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
|
||||
type DB_File struct {
|
||||
ID string `json:"_key"`
|
||||
BatchID string `json:"batch"`
|
||||
InitialPath string `json:"initialPath"`
|
||||
Extension string `json:"extension"`
|
||||
Size int64 `json:"size"`
|
||||
CRC uint32 `json:"crc"`
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
type DB_GMA2File struct {
|
||||
ID string `json:"_key"`
|
||||
BatchID string `json:"batch"`
|
||||
File string `json:"_to"`
|
||||
GMA string `json:"_from"`
|
||||
|
||||
FileNumber int32 `json:"fileNumber"`
|
||||
FileName string `json:"fileName"`
|
||||
Offset int64 `json:"offset"`
|
||||
FileSize int64 `json:"size"`
|
||||
CRC uint32 `json:"crc"`
|
||||
CRCMatch bool `json:"crcMatch"`
|
||||
NextType int32 `json:"nextType"`
|
||||
|
||||
LocalFileName string `json:"-"`
|
||||
UploadID string `json:"-"`
|
||||
}
|
||||
|
||||
type DB_Chunk struct {
|
||||
ID string `json:"_key"`
|
||||
|
||||
Finalized bool `json:"finalized"`
|
||||
ReadOnly bool `json:"readOnly"`
|
||||
|
||||
Size int64 `json:"size"`
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
type DB_File2Chunk struct {
|
||||
ID string `json:"_key"`
|
||||
Chunk string `json:"_to"`
|
||||
File string `json:"_from"`
|
||||
}
|
||||
|
||||
func MultipartUpload(client *http.Client, url string, path string) (err error) {
|
||||
//fmt.Printf("\nMultipartUpload(%s, %s)\n", url, path)
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileContents, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fi, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
file.Close()
|
||||
|
||||
body := new(bytes.Buffer)
|
||||
writer := multipart.NewWriter(body)
|
||||
part, err := writer.CreateFormFile("file", fi.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
part.Write(fileContents)
|
||||
|
||||
/*for key, val := range params {
|
||||
_ = writer.WriteField(key, val)
|
||||
}*/
|
||||
err = writer.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", url, body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Submit the request
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check the response
|
||||
if res.StatusCode == http.StatusAlreadyReported {
|
||||
return
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("bad status: %s", res.Status)
|
||||
}
|
||||
return
|
||||
}
|
||||
@ -0,0 +1,253 @@
|
||||
package gma
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type GMAReader struct {
|
||||
FileHandle *os.File
|
||||
//gmaStream io.Reader
|
||||
gmaStreamReader *bufio.Reader
|
||||
cursorOffset uint32
|
||||
//h header
|
||||
}
|
||||
|
||||
func NewReader(fileName string) (_ GMAReader, err error) {
|
||||
return GMAReader{}.NewReader(fileName)
|
||||
}
|
||||
func (r GMAReader) NewReader(fileName string) (_ GMAReader, err error) {
|
||||
r.FileHandle, err = os.Open(fileName)
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
r.gmaStreamReader = bufio.NewReader(r.FileHandle)
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (r *GMAReader) IsValidGMAD() (isValid bool, err error) {
|
||||
gmadHeader, err := r.gmaStreamReader.Peek(4)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return string(gmadHeader) == "GMAD", nil
|
||||
}
|
||||
func (r *GMAReader) IsCompressed() (isValid bool, err error) {
|
||||
gmadHeader, err := r.gmaStreamReader.Peek(4)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return gmadHeader[0] == 93 && gmadHeader[1] == 0 && gmadHeader[1] == gmadHeader[2] && gmadHeader[1] == gmadHeader[3], nil
|
||||
}
|
||||
|
||||
func (r *GMAReader) Close() {
|
||||
r.FileHandle.Close()
|
||||
}
|
||||
|
||||
func (r *GMAReader) ReadHeader() (_ GMAHeader, err error) {
|
||||
header := GMAHeader{}
|
||||
|
||||
r.gmaStreamReader.Discard(4)
|
||||
r.cursorOffset += 4
|
||||
if r.cursorOffset != 4 {
|
||||
return header, fmt.Errorf("%d invalid offset", r.cursorOffset)
|
||||
}
|
||||
|
||||
// Read the format version
|
||||
header.FormatVersion, err = r.gmaStreamReader.ReadByte()
|
||||
if err != nil {
|
||||
return header, err
|
||||
}
|
||||
r.cursorOffset++
|
||||
|
||||
// Read the SteamID
|
||||
steamIDBytes := make([]byte, 8)
|
||||
_, err = r.gmaStreamReader.Read(steamIDBytes)
|
||||
if err != nil {
|
||||
return header, err
|
||||
}
|
||||
r.cursorOffset += 8
|
||||
header.SteamID = binary.LittleEndian.Uint64(steamIDBytes)
|
||||
|
||||
// Read the Timestamp
|
||||
timestampBytes := make([]byte, 8)
|
||||
_, err = r.gmaStreamReader.Read(timestampBytes)
|
||||
if err != nil {
|
||||
return header, err
|
||||
}
|
||||
r.cursorOffset += 8
|
||||
header.Timestamp = binary.LittleEndian.Uint64(timestampBytes)
|
||||
|
||||
if header.FormatVersion > 1 {
|
||||
header.FormatVersionDiscardByte, err = r.gmaStreamReader.ReadByte()
|
||||
if err != nil {
|
||||
return header, err
|
||||
}
|
||||
r.cursorOffset++
|
||||
}
|
||||
|
||||
// Read the Title
|
||||
header.Title, err = r.gmaStreamReader.ReadString(byte(0))
|
||||
if err != nil {
|
||||
return header, err
|
||||
}
|
||||
header.Title = header.Title[:len(header.Title)-1] // remove nullbyte
|
||||
r.cursorOffset += uint32(len(header.Title) + 1) // Add title length + null byte
|
||||
|
||||
// Read the Description
|
||||
header.Description, err = r.gmaStreamReader.ReadString(byte(0))
|
||||
if err != nil {
|
||||
return header, err
|
||||
}
|
||||
header.Description = header.Description[:len(header.Description)-1] // remove nullbyte
|
||||
//fmt.Printf("Desc Start %d\n", r.cursorOffset)
|
||||
r.cursorOffset += uint32(len(header.Description) + 1) // Add description length + null byte
|
||||
//fmt.Printf("Desc End %d\n", r.cursorOffset)
|
||||
|
||||
// Read the Author
|
||||
header.Author, err = r.gmaStreamReader.ReadString(byte(0))
|
||||
if err != nil {
|
||||
return header, err
|
||||
}
|
||||
header.Author = header.Author[:len(header.Author)-1] // remove nullbyte
|
||||
r.cursorOffset += uint32(len(header.Author) + 1) // Add author length + null byte
|
||||
|
||||
// Read the AddonVersion
|
||||
addonVersionBytes := make([]byte, 4)
|
||||
_, err = r.gmaStreamReader.Read(addonVersionBytes)
|
||||
if err != nil {
|
||||
return header, err
|
||||
}
|
||||
r.cursorOffset += 4
|
||||
header.AddonVersion = int32(binary.LittleEndian.Uint32(addonVersionBytes))
|
||||
|
||||
return header, nil
|
||||
}
|
||||
|
||||
func (r *GMAReader) readFileMetadata() (GMAFileMetadata, error) {
|
||||
metadata := GMAFileMetadata{}
|
||||
|
||||
// Read the file name
|
||||
fileName, err := r.gmaStreamReader.ReadString(byte(0))
|
||||
if err != nil {
|
||||
return metadata, err
|
||||
}
|
||||
fileName = fileName[:len(fileName)-1] // remove nullbyte
|
||||
r.cursorOffset += uint32(len(fileName) + 1) // Add name length + null byte
|
||||
metadata.FileName = fileName
|
||||
|
||||
// Read the file size
|
||||
fileSizeBytes := make([]byte, 8)
|
||||
_, err = r.gmaStreamReader.Read(fileSizeBytes)
|
||||
if err != nil {
|
||||
return metadata, err
|
||||
}
|
||||
r.cursorOffset += 8
|
||||
metadata.FileSize = int64(binary.LittleEndian.Uint64(fileSizeBytes))
|
||||
|
||||
// Read the file crc
|
||||
crcBytes := make([]byte, 4)
|
||||
_, err = r.gmaStreamReader.Read(crcBytes)
|
||||
if err != nil {
|
||||
return metadata, err
|
||||
}
|
||||
r.cursorOffset += 4
|
||||
metadata.CRC = binary.LittleEndian.Uint32(crcBytes)
|
||||
|
||||
// Read the next type
|
||||
nextTypeBytes := make([]byte, 4)
|
||||
_, err = r.gmaStreamReader.Read(nextTypeBytes)
|
||||
if err != nil {
|
||||
return metadata, err
|
||||
}
|
||||
r.cursorOffset += 4
|
||||
metadata.NextType = int32(binary.LittleEndian.Uint32(nextTypeBytes))
|
||||
|
||||
return metadata, nil
|
||||
}
|
||||
func (r *GMAReader) ReadAddonCRC(lastOffset int64) (crc uint32, err error) {
|
||||
|
||||
limitReader := io.NewSectionReader(r.FileHandle, int64(r.cursorOffset)+lastOffset, int64(4))
|
||||
|
||||
crcBytes := make([]byte, 4)
|
||||
_, err = limitReader.Read(crcBytes)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
r.cursorOffset += 4
|
||||
CRC := binary.LittleEndian.Uint32(crcBytes)
|
||||
return CRC, nil
|
||||
}
|
||||
|
||||
func (r *GMAReader) ReadFiles() (firstType int32, files []GMAFileMetadata, err error) {
|
||||
// read nType 4byte
|
||||
firstTypeBytes := make([]byte, 4)
|
||||
_, err = r.gmaStreamReader.Read(firstTypeBytes)
|
||||
if err != nil {
|
||||
return 0, files, err
|
||||
}
|
||||
r.cursorOffset += 4
|
||||
firstType = int32(binary.LittleEndian.Uint32(firstTypeBytes))
|
||||
|
||||
if firstType == 0 {
|
||||
return 0, files, nil
|
||||
}
|
||||
fileOffset := int64(0)
|
||||
fileNumber := int32(1)
|
||||
for {
|
||||
fileMeta, err := r.readFileMetadata()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return firstType, files, err
|
||||
}
|
||||
fileMeta.FileNumber = fileNumber
|
||||
fileMeta.Offset = fileOffset
|
||||
//fmt.Printf("%s CRC: %d Offset: %d Size: %d\n", fileMeta.FileName, fileMeta.CRC, fileMeta.Offset, fileMeta.FileSize)
|
||||
//fmt.Printf("[% x]\n", fileMeta.FileName)
|
||||
files = append(files, fileMeta)
|
||||
fileOffset += fileMeta.FileSize
|
||||
fileNumber++
|
||||
if fileMeta.NextType == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return firstType, files, nil
|
||||
}
|
||||
func (r *GMAReader) GetOffset() (offset uint32) {
|
||||
return r.cursorOffset
|
||||
}
|
||||
func (r *GMAReader) ExtractFileTo(fileMeta GMAFileMetadata, writer io.Writer) (extractMeta GMAExtractionMeta, err error) {
|
||||
extractMeta.OriginalMeta = fileMeta
|
||||
// Seek to the specified offset in the reader
|
||||
limitReader := io.NewSectionReader(r.FileHandle, int64(r.cursorOffset)+fileMeta.Offset, int64(fileMeta.FileSize))
|
||||
// Copy the specified length of data from the reader to the output file
|
||||
buf := bytes.NewBuffer(nil)
|
||||
_, err = io.CopyN(buf, limitReader, int64(fileMeta.FileSize))
|
||||
if err != nil {
|
||||
return extractMeta, err
|
||||
}
|
||||
shaHasher := sha256.New()
|
||||
|
||||
extractMeta.ExtractedCRC = crc32.Checksum(buf.Bytes(), crc32.MakeTable(crc32.IEEE))
|
||||
shaHasher.Write(buf.Bytes())
|
||||
extractMeta.ExtractedSHA256 = fmt.Sprintf("%x", shaHasher.Sum(nil))
|
||||
buf.WriteTo(writer)
|
||||
|
||||
return extractMeta, nil
|
||||
}
|
||||
func (r *GMAReader) GetSHA256() (hash string, err error) {
|
||||
shaHasher := sha256.New()
|
||||
if _, err := io.Copy(shaHasher, r.FileHandle); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", shaHasher.Sum(nil)), nil
|
||||
}
|
||||
@ -0,0 +1,132 @@
|
||||
package gma
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
type GMAWriter struct {
|
||||
FileHandle *os.File
|
||||
gmaStreamWriter *bufio.Writer
|
||||
}
|
||||
|
||||
func NewWriter(fileName string) (_ GMAWriter, err error) {
|
||||
return GMAWriter{}.NewWriter(fileName)
|
||||
}
|
||||
func (w GMAWriter) NewWriter(fileName string) (_ GMAWriter, err error) {
|
||||
w.FileHandle, err = os.Create(fileName)
|
||||
if err != nil {
|
||||
return w, err
|
||||
}
|
||||
w.gmaStreamWriter = bufio.NewWriter(w.FileHandle)
|
||||
return w, nil
|
||||
}
|
||||
func (w *GMAWriter) WriteHeader(header GMAHeader) (err error) {
|
||||
w.gmaStreamWriter.WriteByte('G')
|
||||
w.gmaStreamWriter.WriteByte('M')
|
||||
w.gmaStreamWriter.WriteByte('A')
|
||||
w.gmaStreamWriter.WriteByte('D')
|
||||
|
||||
w.gmaStreamWriter.WriteByte(header.FormatVersion)
|
||||
|
||||
steamIDBytes := make([]byte, 8)
|
||||
binary.LittleEndian.PutUint64(steamIDBytes, header.SteamID)
|
||||
w.gmaStreamWriter.Write(steamIDBytes)
|
||||
|
||||
timestampBytes := make([]byte, 8)
|
||||
binary.LittleEndian.PutUint64(timestampBytes, header.Timestamp)
|
||||
w.gmaStreamWriter.Write(timestampBytes)
|
||||
|
||||
if header.FormatVersion > 1 {
|
||||
w.gmaStreamWriter.WriteByte(header.FormatVersionDiscardByte)
|
||||
}
|
||||
|
||||
w.gmaStreamWriter.WriteString(header.Title)
|
||||
w.gmaStreamWriter.WriteByte(0)
|
||||
w.gmaStreamWriter.WriteString(header.Description)
|
||||
w.gmaStreamWriter.WriteByte(0)
|
||||
w.gmaStreamWriter.WriteString(header.Author)
|
||||
w.gmaStreamWriter.WriteByte(0)
|
||||
|
||||
addonVersionBytes := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(addonVersionBytes, uint32(header.AddonVersion))
|
||||
w.gmaStreamWriter.Write(addonVersionBytes)
|
||||
|
||||
w.gmaStreamWriter.Flush()
|
||||
|
||||
return nil
|
||||
}
|
||||
func (w *GMAWriter) Close() (err error) {
|
||||
return w.FileHandle.Close()
|
||||
}
|
||||
func (w *GMAWriter) WriteFooterCRC(footerCRC uint32) (err error) {
|
||||
crcBytes := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(crcBytes, footerCRC)
|
||||
w.gmaStreamWriter.Write(crcBytes)
|
||||
w.gmaStreamWriter.Flush()
|
||||
|
||||
return nil
|
||||
}
|
||||
func (w *GMAWriter) WriteFirstType(firstType int32) (err error) {
|
||||
firstTypeBytes := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(firstTypeBytes, uint32(firstType))
|
||||
w.gmaStreamWriter.Write(firstTypeBytes)
|
||||
w.gmaStreamWriter.Flush()
|
||||
|
||||
return nil
|
||||
}
|
||||
func (w *GMAWriter) WriteFileIndex(fileName string, fileSize int64, fileCRC uint32, nextType int32) (err error) {
|
||||
w.gmaStreamWriter.WriteString(fileName)
|
||||
w.gmaStreamWriter.WriteByte(0)
|
||||
|
||||
fileSizeBytes := make([]byte, 8)
|
||||
binary.LittleEndian.PutUint64(fileSizeBytes, uint64(fileSize))
|
||||
w.gmaStreamWriter.Write(fileSizeBytes)
|
||||
|
||||
crcBytes := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(crcBytes, fileCRC)
|
||||
w.gmaStreamWriter.Write(crcBytes)
|
||||
|
||||
nextTypeBytes := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(nextTypeBytes, uint32(nextType))
|
||||
w.gmaStreamWriter.Write(nextTypeBytes)
|
||||
|
||||
w.gmaStreamWriter.Flush()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *GMAWriter) WriteFile(reader io.ReadCloser) (err error) {
|
||||
//body := new(bytes.Buffer)
|
||||
|
||||
if _, err = io.Copy(w.gmaStreamWriter, reader); err != nil {
|
||||
return err
|
||||
}
|
||||
/*
|
||||
byteCount, err := w.gmaStreamWriter.Write(body.Bytes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if byteCount != body.Len() {
|
||||
return fmt.Errorf("Only wrote %d bytes from %d", byteCount, body.Len())
|
||||
}
|
||||
*/
|
||||
err = w.gmaStreamWriter.Flush()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w GMAWriter) GetSHA256() (hash string, err error) {
|
||||
shaHasher := sha256.New()
|
||||
if _, err := io.Copy(shaHasher, w.FileHandle); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%x", shaHasher.Sum(nil)), nil
|
||||
}
|
||||
Loading…
Reference in New Issue