thelilfrog/cloudsave
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 8 Wiki Activity

push backup

Browse Source
This commit is contained in:
Aurélie DELHAIE
2025-07-30 15:14:01 +02:00
parent c6edb91f29
commit 30b76e1887
11 changed files with 290 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.vscode/launch.json vendored
View File

@@ -10,7 +10,7 @@
"type": "go",
"request": "launch",
"mode": "auto",
"args": ["list", "-a", "http://localhost:8080"],
"args": ["run"],
"console": "integratedTerminal",
"program": "${workspaceFolder}/cmd/cli"
}

15
cmd/cli/commands/run/run.go
View File

@@ -37,11 +37,7 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
return subcommands.ExitFailure
}
pg := progressbar.New(len(datastore))
defer pg.Close()
for _, metadata := range datastore {
pg.Describe("Scanning " + metadata.Name + "...")
metadataPath := filepath.Join(repository.DatastorePath(), metadata.ID)
//todo transaction
err := archiveIfChanged(metadata.ID, metadata.Path, filepath.Join(metadataPath, "data.tar.gz"), filepath.Join(metadataPath, ".last_run"))
@@ -57,11 +53,8 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
fmt.Fprintf(os.Stderr, "error: cannot process the data of %s: %s\n", metadata.ID, err)
return subcommands.ExitFailure
}
pg.Add(1)
}
pg.Finish()
return subcommands.ExitSuccess
}
@@ -69,6 +62,11 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
// in srcDir has a modification time > the last run time stored in stateFile.
// After archiving, it updates stateFile to the current time.
func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error {
pg := progressbar.New(-1)
defer pg.Close()
pg.Describe("Scanning " + gameID + "...")
// load last run time
var lastRun time.Time
data, err := os.ReadFile(stateFile)
@@ -99,15 +97,18 @@ func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error {
}
if !changed {
pg.Finish()
return nil
}
// make a backup
pg.Describe("Backup current data...")
if err := repository.Archive(gameID); err != nil {
return fmt.Errorf("failed to archive data: %w", err)
}
// create archive
pg.Describe("Archiving new data...")
f, err := os.Create(destTarGz)
if err != nil {
return fmt.Errorf("failed to creating archive file: %w", err)

43
cmd/cli/commands/sync/sync.go
View File

@@ -65,10 +65,13 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
}
if !exists {
if err := push(r.GameID, g, cli); err != nil {
if err := push(g, cli); err != nil {
fmt.Fprintln(os.Stderr, "failed to push:", err)
return subcommands.ExitFailure
}
if err := pushBackup(g, cli); err != nil {
slog.Warn("failed to push backup files", "err", err)
}
continue
}
@@ -96,6 +99,10 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
continue
}
if err := pushBackup(g, cli); err != nil {
slog.Warn("failed to push backup files", "err", err)
}
if hlocal == hremote {
if vlocal != remoteMetadata.Version {
slog.Debug("version is not the same, but the hash is equal. Updating local database")
@@ -109,7 +116,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
}
if vlocal > remoteMetadata.Version {
if err := push(r.GameID, g, cli); err != nil {
if err := push(g, cli); err != nil {
fmt.Fprintln(os.Stderr, "failed to push:", err)
return subcommands.ExitFailure
}
@@ -164,7 +171,7 @@ func conflict(gameID string, m, remoteMetadata repository.Metadata, cli *client.
switch res {
case prompt.My:
{
if err := push(gameID, m, cli); err != nil {
if err := push(m, cli); err != nil {
return fmt.Errorf("failed to push: %w", err)
}
}
@@ -185,10 +192,34 @@ func conflict(gameID string, m, remoteMetadata repository.Metadata, cli *client.
return nil
}
func push(gameID string, m repository.Metadata, cli *client.Client) error {
archivePath := filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz")
func push(m repository.Metadata, cli *client.Client) error {
archivePath := filepath.Join(repository.DatastorePath(), m.ID, "data.tar.gz")
return cli.Push(gameID, archivePath, m)
return cli.PushSave(archivePath, m)
}
func pushBackup(m repository.Metadata, cli *client.Client) error {
bs, err := repository.Archives(m.ID)
if err != nil {
return err
}
for _, b := range bs {
binfo, err := cli.ArchiveInfo(m.ID, b.UUID)
if err != nil {
if !errors.Is(err, client.ErrNotFound) {
return fmt.Errorf("failed to get remote information about the backup file: %w", err)
}
}
if binfo.MD5 != b.MD5 {
if err := cli.PushBackup(b, m); err != nil {
return fmt.Errorf("failed to push backup: %w", err)
}
}
}
return nil
}
func pull(gameID string, cli *client.Client) error {

52
cmd/server/api/api.go
View File

@@ -61,7 +61,8 @@ func NewServer(documentRoot string, creds map[string]string, port int) *HTTPServ
// Data routes
gamesRouter.Group(func(saveRouter chi.Router) {
saveRouter.Post("/{id}/data", s.upload)
saveRouter.Post("/{id}/hist/data", s.histUpload)
saveRouter.Post("/{id}/hist/{uuid}/data", s.histUpload)
saveRouter.Get("/{id}/hist/{uuid}/info", s.histExists)
saveRouter.Get("/{id}/data", s.download)
saveRouter.Get("/{id}/hash", s.hash)
saveRouter.Get("/{id}/metadata", s.metadata)
@@ -215,19 +216,14 @@ func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) {
sizeLimit int64 = 500 << 20 // 500 MB
)
id := chi.URLParam(r, "id")
dt, err := formParseDate("date", r.MultipartForm.Value)
if err != nil {
fmt.Fprintln(os.Stderr, "error: failed to load payload:", err)
badRequest("bad payload", w, r)
return
}
gameID := chi.URLParam(r, "id")
uuid := chi.URLParam(r, "uuid")
// Limit max upload size
r.Body = http.MaxBytesReader(w, r.Body, sizeLimit)
// Parse multipart form
err = r.ParseMultipartForm(sizeLimit)
err := r.ParseMultipartForm(sizeLimit)
if err != nil {
fmt.Fprintln(os.Stderr, "error: failed to load payload:", err)
badRequest("bad payload", w, r)
@@ -243,7 +239,7 @@ func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) {
}
defer file.Close()
if err := data.WriteHist(id, s.documentRoot, dt, file); err != nil {
if err := data.WriteHist(gameID, s.documentRoot, uuid, file); err != nil {
fmt.Fprintln(os.Stderr, "error: failed to write file to disk:", err)
internalServerError(w, r)
return
@@ -253,6 +249,24 @@ func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusCreated)
}
func (s HTTPServer) histExists(w http.ResponseWriter, r *http.Request) {
gameID := chi.URLParam(r, "id")
uuid := chi.URLParam(r, "uuid")
finfo, err := data.ArchiveInfo(gameID, s.documentRoot, uuid)
if err != nil {
if errors.Is(err, data.ErrBackupNotExists) {
notFound("backup not found", w, r)
return
}
fmt.Fprintln(os.Stderr, "error: failed to read data:", err)
internalServerError(w, r)
return
}
ok(finfo, w, r)
}
func (s HTTPServer) hash(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
path := filepath.Clean(filepath.Join(s.documentRoot, "data", id))
@@ -374,21 +388,3 @@ func parseFormMetadata(gameID string, values map[string][]string) (repository.Me
Date: date,
}, nil
}
func formParseDate(key string, values map[string][]string) (time.Time, error) {
var date time.Time
if v, ok := values[key]; ok {
if len(v) == 0 {
return time.Time{}, fmt.Errorf("error: corrupted metadata")
}
if v, err := time.Parse(time.RFC3339, v[0]); err == nil {
date = v
} else {
return time.Time{}, err
}
} else {
return time.Time{}, fmt.Errorf("error: cannot find metadata in the form")
}
return date, nil
}

42
cmd/server/data/data.go
View File

@@ -2,12 +2,17 @@ package data
import (
"cloudsave/pkg/repository"
"cloudsave/pkg/tools/hash"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"time"
)
var (
ErrBackupNotExists error = errors.New("backup not found")
)
func Write(gameID, documentRoot string, r io.Reader) error {
@@ -40,15 +45,19 @@ func Write(gameID, documentRoot string, r io.Reader) error {
return nil
}
func WriteHist(gameID, documentRoot string, dt time.Time, r io.Reader) error {
dataFolderPath := filepath.Join(documentRoot, "data", gameID, "hist")
partPath := filepath.Join(dataFolderPath, dt.Format("2006-01-02T15-04-05Z07-00")+".data.tar.gz.part")
finalFilePath := filepath.Join(dataFolderPath, dt.Format("2006-01-02T15-04-05Z07-00")+".data.tar.gz")
func WriteHist(gameID, documentRoot, uuid string, r io.Reader) error {
dataFolderPath := filepath.Join(documentRoot, "data", gameID, "hist", uuid)
partPath := filepath.Join(dataFolderPath, "data.tar.gz.part")
finalFilePath := filepath.Join(dataFolderPath, "data.tar.gz")
if err := makeDataFolder(gameID, documentRoot); err != nil {
return err
}
if err := os.MkdirAll(dataFolderPath, 0740); err != nil {
return err
}
f, err := os.OpenFile(partPath, os.O_CREATE|os.O_WRONLY, 0740)
if err != nil {
return err
@@ -86,6 +95,29 @@ func UpdateMetadata(gameID, documentRoot string, m repository.Metadata) error {
return e.Encode(m)
}
func ArchiveInfo(gameID, documentRoot, uuid string) (repository.Backup, error) {
dataFolderPath := filepath.Join(documentRoot, "data", gameID, "hist", uuid, "data.tar.gz")
finfo, err := os.Stat(dataFolderPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return repository.Backup{}, ErrBackupNotExists
}
return repository.Backup{}, err
}
h, err := hash.FileMD5(dataFolderPath)
if err != nil {
return repository.Backup{}, fmt.Errorf("failed to calculate file md5: %w", err)
}
return repository.Backup{
CreatedAt: finfo.ModTime(),
UUID: uuid,
MD5: h,
}, nil
}
func makeDataFolder(gameID, documentRoot string) error {
if err := os.MkdirAll(filepath.Join(documentRoot, "data", gameID), 0740); err != nil {
return err

1
go.mod
View File

@@ -5,6 +5,7 @@ go 1.24
require (
github.com/go-chi/chi/v5 v5.2.1
github.com/google/subcommands v1.2.0
github.com/google/uuid v1.6.0
github.com/schollz/progressbar/v3 v3.18.0
golang.org/x/crypto v0.38.0
golang.org/x/term v0.32.0

2
go.sum
View File

@@ -6,6 +6,8 @@ github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8=
github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE=
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=

118
pkg/remote/client/client.go
View File

@@ -35,6 +35,10 @@ type (
}
)
var (
ErrNotFound error = errors.New("not found")
)
func New(baseURL, username, password string) *Client {
return &Client{
baseURL: baseURL,
@@ -141,59 +145,45 @@ func (c *Client) Metadata(gameID string) (repository.Metadata, error) {
return repository.Metadata{}, errors.New("invalid payload sent by the server")
}
func (c *Client) Push(gameID, archivePath string, m repository.Metadata) error {
u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "data")
func (c *Client) PushSave(archivePath string, m repository.Metadata) error {
u, err := url.JoinPath(c.baseURL, "api", "v1", "games", m.ID, "data")
if err != nil {
return err
}
f, err := os.OpenFile(archivePath, os.O_RDONLY, 0)
if err != nil {
return fmt.Errorf("failed to open file: %w", err)
}
defer f.Close()
return c.push(u, archivePath, m)
}
buf := new(bytes.Buffer)
writer := multipart.NewWriter(buf)
part, err := writer.CreateFormFile("payload", "data.tar.gz")
func (c *Client) PushBackup(archiveMetadata repository.Backup, m repository.Metadata) error {
u, err := url.JoinPath(c.baseURL, "api", "v1", "games", m.ID, "hist", archiveMetadata.UUID, "data")
if err != nil {
return err
}
if _, err := io.Copy(part, f); err != nil {
return fmt.Errorf("failed to copy data: %w", err)
}
return c.push(u, archiveMetadata.ArchivePath, m)
}
writer.WriteField("name", m.Name)
writer.WriteField("version", strconv.Itoa(m.Version))
writer.WriteField("date", m.Date.Format(time.RFC3339))
if err := writer.Close(); err != nil {
return err
}
cli := http.Client{}
req, err := http.NewRequest("POST", u, buf)
func (c *Client) ArchiveInfo(gameID, uuid string) (repository.Backup, error) {
u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "hist", uuid, "info")
if err != nil {
return err
return repository.Backup{}, err
}
req.SetBasicAuth(c.username, c.password)
req.Header.Set("Content-Type", writer.FormDataContentType())
res, err := cli.Do(req)
o, err := c.get(u)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 201 {
return fmt.Errorf("server returns an unexpected status code: %s (expected 201)", res.Status)
return repository.Backup{}, err
}
return nil
if m, ok := (o.Data).(map[string]any); ok {
b := repository.Backup{
UUID: m["uuid"].(string),
CreatedAt: customtime.MustParse(time.RFC3339, m["created_at"].(string)),
MD5: m["md5"].(string),
}
return b, nil
}
return repository.Backup{}, errors.New("invalid payload sent by the server")
}
func (c *Client) Pull(gameID, archivePath string) error {
@@ -318,6 +308,10 @@ func (c *Client) get(url string) (obj.HTTPObject, error) {
}
defer res.Body.Close()
if res.StatusCode == 404 {
return obj.HTTPObject{}, ErrNotFound
}
if res.StatusCode != 200 {
return obj.HTTPObject{}, fmt.Errorf("server returns an unexpected status code: %d %s (expected 200)", res.StatusCode, res.Status)
}
@@ -331,3 +325,53 @@ func (c *Client) get(url string) (obj.HTTPObject, error) {
return httpObject, nil
}
func (c *Client) push(u, archivePath string, m repository.Metadata) error {
f, err := os.OpenFile(archivePath, os.O_RDONLY, 0)
if err != nil {
return fmt.Errorf("failed to open file: %w", err)
}
defer f.Close()
buf := new(bytes.Buffer)
writer := multipart.NewWriter(buf)
part, err := writer.CreateFormFile("payload", "data.tar.gz")
if err != nil {
return err
}
if _, err := io.Copy(part, f); err != nil {
return fmt.Errorf("failed to copy data: %w", err)
}
writer.WriteField("name", m.Name)
writer.WriteField("version", strconv.Itoa(m.Version))
writer.WriteField("date", m.Date.Format(time.RFC3339))
if err := writer.Close(); err != nil {
return err
}
cli := http.Client{}
req, err := http.NewRequest("POST", u, buf)
if err != nil {
return err
}
req.SetBasicAuth(c.username, c.password)
req.Header.Set("Content-Type", writer.FormDataContentType())
res, err := cli.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 201 {
return fmt.Errorf("server returns an unexpected status code: %s (expected 201)", res.Status)
}
return nil
}

102
pkg/repository/repository.go
View File

@@ -1,16 +1,17 @@
package repository
import (
"cloudsave/pkg/tools/hash"
"cloudsave/pkg/tools/id"
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"time"
"github.com/google/uuid"
)
type (
@@ -21,6 +22,13 @@ type (
Version int `json:"version"`
Date time.Time `json:"date"`
}
Backup struct {
CreatedAt time.Time `json:"created_at"`
MD5 string `json:"md5"`
UUID string `json:"uuid"`
ArchivePath string `json:"-"`
}
)
var (
@@ -142,46 +150,20 @@ func Archive(gameID string) error {
// open old
f, err := os.OpenFile(path, os.O_RDONLY, 0)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return fmt.Errorf("failed to open old file: %w", err)
}
defer f.Close()
histDirPath := filepath.Join(datastorepath, gameID, "hist")
histDirPath := filepath.Join(datastorepath, gameID, "hist", uuid.NewString())
if err := os.MkdirAll(histDirPath, 0740); err != nil {
return fmt.Errorf("failed to make 'hist' directory")
}
d, err := os.ReadDir(histDirPath)
if err != nil {
return fmt.Errorf("failed to open 'hist' directory")
}
// keep the dir under 6 files
if len(d) > 5 {
var oldest *fs.FileInfo
for _, hfile := range d {
finfo, err := hfile.Info()
if err != nil {
return fmt.Errorf("failed to read backup file: %w", err)
}
if oldest == nil {
oldest = &finfo
continue
}
if finfo.ModTime().Before((*oldest).ModTime()) {
oldest = &finfo
}
}
if err := os.Remove((*oldest).Name()); err != nil {
return fmt.Errorf("failed to remove the oldest backup file: %w", err)
}
return fmt.Errorf("failed to make directory: %w", err)
}
// open new
nf, err := os.OpenFile(filepath.Join(datastorepath, gameID, "hist", time.Now().Format("2006-01-02T15-04-05Z07-00")+".data.tar.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740)
nf, err := os.OpenFile(filepath.Join(histDirPath, "data.tar.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740)
if err != nil {
return fmt.Errorf("failed to open new file: %w", err)
}
@@ -195,6 +177,43 @@ func Archive(gameID string) error {
return nil
}
func Archives(gameID string) ([]Backup, error) {
histDirPath := filepath.Join(datastorepath, gameID, "hist")
if err := os.MkdirAll(histDirPath, 0740); err != nil {
return nil, fmt.Errorf("failed to make 'hist' directory")
}
d, err := os.ReadDir(histDirPath)
if err != nil {
return nil, fmt.Errorf("failed to open 'hist' directory")
}
var res []Backup
for _, f := range d {
finfo, err := f.Info()
if err != nil {
return nil, fmt.Errorf("corrupted datastore: %w", err)
}
path := filepath.Join(histDirPath, finfo.Name())
archivePath := filepath.Join(path, "data.tar.gz")
h, err := hash.FileMD5(archivePath)
if err != nil {
return nil, fmt.Errorf("failed to calculate md5 hash: %w", err)
}
b := Backup{
CreatedAt: finfo.ModTime(),
UUID: filepath.Base(finfo.Name()),
MD5: h,
ArchivePath: archivePath,
}
res = append(res, b)
}
return res, nil
}
func DatastorePath() string {
return datastorepath
}
@@ -210,18 +229,7 @@ func Remove(gameID string) error {
func Hash(gameID string) (string, error) {
path := filepath.Join(datastorepath, gameID, "data.tar.gz")
f, err := os.OpenFile(path, os.O_RDONLY, 0)
if err != nil {
return "", err
}
defer f.Close()
hasher := md5.New()
if _, err := io.Copy(hasher, f); err != nil {
return "", err
}
sum := hasher.Sum(nil)
return hex.EncodeToString(sum), nil
return hash.FileMD5(path)
}
func Version(gameID string) (int, error) {

38
pkg/tools/archive/archive.go
View File

@@ -73,7 +73,7 @@ func Untar(file io.Reader, path string) error {
}
}
func Tar(file io.Writer, path string) error {
func Tar(file io.Writer, root string) error {
gw := gzip.NewWriter(file)
defer gw.Close()
@@ -81,34 +81,38 @@ func Tar(file io.Writer, path string) error {
defer tw.Close()
// Walk again to add files
err := filepath.Walk(path, func(path string, info os.FileInfo, walkErr error) error {
err := filepath.Walk(root, func(path string, info os.FileInfo, walkErr error) error {
if walkErr != nil {
return walkErr
}
path, err := filepath.Rel(root, path)
if err != nil {
return err
}
// Create tar header
header, err := tar.FileInfoHeader(info, path)
if err != nil {
return err
}
// Preserve directory structure relative to srcDir
relPath, err := filepath.Rel(filepath.Dir(path), path)
if err != nil {
return err
}
header.Name = relPath
header.Name = path
if err := tw.WriteHeader(header); err != nil {
return err
}
if info.Mode().IsRegular() {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
if _, err := io.Copy(tw, file); err != nil {
return err
}
if !info.Mode().IsRegular() {
return nil
}
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
if _, err := io.Copy(tw, file); err != nil {
return err
}
return nil
})

23
pkg/tools/hash/hash.go Normal file
View File

@@ -0,0 +1,23 @@
package hash
import (
"crypto/md5"
"encoding/hex"
"io"
"os"
)
func FileMD5(fp string) (string, error) {
f, err := os.OpenFile(fp, os.O_RDONLY, 0)
if err != nil {
return "", err
}
defer f.Close()
hasher := md5.New()
if _, err := io.Copy(hasher, f); err != nil {
return "", err
}
sum := hasher.Sum(nil)
return hex.EncodeToString(sum), nil
}