From c6edb91f297b871ceb69f065393528771c1b505a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Wed, 30 Jul 2025 00:49:22 +0200 Subject: [PATCH 1/5] starting 0.0.2 dev --- build.sh | 2 +- cmd/cli/commands/add/add.go | 4 +- cmd/cli/commands/list/list.go | 6 +- cmd/cli/commands/pull/pull.go | 8 +- cmd/cli/commands/remote/remote.go | 4 +- cmd/cli/commands/remove/remove.go | 4 +- cmd/cli/commands/run/run.go | 75 +++++----------- cmd/cli/commands/sync/sync.go | 30 +++---- cmd/cli/commands/version/version.go | 2 +- .../tools/prompt/credentials/credentials.go | 0 cmd/server/api/api.go | 90 ++++++++++++++++--- cmd/server/data/data.go | 45 +++++++++- pkg/constants/constants.go | 2 +- pkg/remote/client/client.go | 20 ++--- .../game.go => repository/repository.go} | 62 ++++++++++++- pkg/tools/archive/archive.go | 47 ++++++++++ 16 files changed, 287 insertions(+), 114 deletions(-) rename {pkg => cmd/cli}/tools/prompt/credentials/credentials.go (100%) rename pkg/{game/game.go => repository/repository.go} (77%) diff --git a/build.sh b/build.sh index c0ad24f..5eea3c2 100755 --- a/build.sh +++ b/build.sh @@ -1,7 +1,7 @@ #!/bin/bash MAKE_PACKAGE=false -VERSION=0.0.1 +VERSION=0.0.2 usage() { echo "Usage: $0 [OPTIONS]" diff --git a/cmd/cli/commands/add/add.go b/cmd/cli/commands/add/add.go index 84ee030..5e1c629 100644 --- a/cmd/cli/commands/add/add.go +++ b/cmd/cli/commands/add/add.go @@ -1,7 +1,7 @@ package add import ( - "cloudsave/pkg/game" + "cloudsave/pkg/repository" "context" "flag" "fmt" @@ -44,7 +44,7 @@ func (p *AddCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s p.name = filepath.Base(path) } - m, err := game.Add(p.name, path) + m, err := repository.Add(p.name, path) if err != nil { fmt.Fprintln(os.Stderr, "error: failed to add game reference:", err) return subcommands.ExitFailure diff --git a/cmd/cli/commands/list/list.go b/cmd/cli/commands/list/list.go index 3a763b5..805c44c 100644 --- a/cmd/cli/commands/list/list.go +++ b/cmd/cli/commands/list/list.go @@ -1,9 +1,9 @@ package list import ( - "cloudsave/pkg/game" "cloudsave/pkg/remote/client" - "cloudsave/pkg/tools/prompt/credentials" + "cloudsave/pkg/repository" + "cloudsave/cmd/cli/tools/prompt/credentials" "context" "flag" "fmt" @@ -57,7 +57,7 @@ func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } func local() error { - games, err := game.All() + games, err := repository.All() if err != nil { return fmt.Errorf("failed to load datastore: %w", err) } diff --git a/cmd/cli/commands/pull/pull.go b/cmd/cli/commands/pull/pull.go index a20d4ef..5a0da4e 100644 --- a/cmd/cli/commands/pull/pull.go +++ b/cmd/cli/commands/pull/pull.go @@ -1,10 +1,10 @@ package pull import ( - "cloudsave/pkg/game" "cloudsave/pkg/remote/client" + "cloudsave/pkg/repository" "cloudsave/pkg/tools/archive" - "cloudsave/pkg/tools/prompt/credentials" + "cloudsave/cmd/cli/tools/prompt/credentials" "context" "flag" "fmt" @@ -54,7 +54,7 @@ func (p *PullCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitFailure } - archivePath := filepath.Join(game.DatastorePath(), gameID, "data.tar.gz") + archivePath := filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz") m, err := cli.Metadata(gameID) if err != nil { @@ -62,7 +62,7 @@ func (p *PullCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitFailure } - err = game.Register(m, path) + err = repository.Register(m, path) if err != nil { fmt.Fprintf(os.Stderr, "failed to register local metadata: %s", err) return subcommands.ExitFailure diff --git a/cmd/cli/commands/remote/remote.go b/cmd/cli/commands/remote/remote.go index d1eebb1..e47cfc5 100644 --- a/cmd/cli/commands/remote/remote.go +++ b/cmd/cli/commands/remote/remote.go @@ -1,9 +1,9 @@ package remote import ( - "cloudsave/pkg/game" "cloudsave/pkg/remote" "cloudsave/pkg/remote/client" + "cloudsave/pkg/repository" "context" "flag" "fmt" @@ -62,7 +62,7 @@ func (p *RemoteCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface } func list() error { - games, err := game.All() + games, err := repository.All() if err != nil { return fmt.Errorf("failed to load datastore: %w", err) } diff --git a/cmd/cli/commands/remove/remove.go b/cmd/cli/commands/remove/remove.go index ee051c7..2459616 100644 --- a/cmd/cli/commands/remove/remove.go +++ b/cmd/cli/commands/remove/remove.go @@ -1,7 +1,7 @@ package remove import ( - "cloudsave/pkg/game" + "cloudsave/pkg/repository" "context" "flag" "fmt" @@ -31,7 +31,7 @@ func (p *RemoveCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} return subcommands.ExitUsageError } - err := game.Remove(f.Arg(0)) + err := repository.Remove(f.Arg(0)) if err != nil { fmt.Fprintln(os.Stderr, "error: failed to unregister the game:", err) return subcommands.ExitFailure diff --git a/cmd/cli/commands/run/run.go b/cmd/cli/commands/run/run.go index c9d0684..42aec35 100644 --- a/cmd/cli/commands/run/run.go +++ b/cmd/cli/commands/run/run.go @@ -1,9 +1,8 @@ package run import ( - "archive/tar" - "cloudsave/pkg/game" - "compress/gzip" + "cloudsave/pkg/repository" + "cloudsave/pkg/tools/archive" "context" "flag" "fmt" @@ -32,7 +31,7 @@ func (*RunCmd) Usage() string { func (p *RunCmd) SetFlags(f *flag.FlagSet) {} func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { - datastore, err := game.All() + datastore, err := repository.All() if err != nil { fmt.Fprintln(os.Stderr, "error: failed to load datastore:", err) return subcommands.ExitFailure @@ -43,18 +42,18 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s for _, metadata := range datastore { pg.Describe("Scanning " + metadata.Name + "...") - metadataPath := filepath.Join(game.DatastorePath(), metadata.ID) + metadataPath := filepath.Join(repository.DatastorePath(), metadata.ID) //todo transaction err := archiveIfChanged(metadata.ID, metadata.Path, filepath.Join(metadataPath, "data.tar.gz"), filepath.Join(metadataPath, ".last_run")) if err != nil { fmt.Fprintf(os.Stderr, "error: cannot process the data of %s: %s\n", metadata.ID, err) return subcommands.ExitFailure } - if err := game.SetVersion(metadata.ID, metadata.Version+1); err != nil { + if err := repository.SetVersion(metadata.ID, metadata.Version+1); err != nil { fmt.Fprintf(os.Stderr, "error: cannot process the data of %s: %s\n", metadata.ID, err) return subcommands.ExitFailure } - if err := game.SetDate(metadata.ID, time.Now()); err != nil { + if err := repository.SetDate(metadata.ID, time.Now()); err != nil { fmt.Fprintf(os.Stderr, "error: cannot process the data of %s: %s\n", metadata.ID, err) return subcommands.ExitFailure } @@ -69,12 +68,12 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s // archiveIfChanged will archive srcDir into destTarGz only if any file // in srcDir has a modification time > the last run time stored in stateFile. // After archiving, it updates stateFile to the current time. -func archiveIfChanged(id, srcDir, destTarGz, stateFile string) error { - // 1) Load last run time +func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error { + // load last run time var lastRun time.Time data, err := os.ReadFile(stateFile) if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("reading state file: %w", err) + return fmt.Errorf("failed to reading state file: %w", err) } if err == nil { lastRun, err = time.Parse(time.RFC3339, string(data)) @@ -83,7 +82,7 @@ func archiveIfChanged(id, srcDir, destTarGz, stateFile string) error { } } - // 2) Check for changes + // check for changes changed := false err = filepath.Walk(srcDir, func(path string, info os.FileInfo, walkErr error) error { if walkErr != nil { @@ -96,63 +95,29 @@ func archiveIfChanged(id, srcDir, destTarGz, stateFile string) error { return nil }) if err != nil && err != io.EOF { - return fmt.Errorf("scanning source directory: %w", err) + return fmt.Errorf("failed to scanning source directory: %w", err) } if !changed { return nil } - // 3) Create tar.gz + // make a backup + if err := repository.Archive(gameID); err != nil { + return fmt.Errorf("failed to archive data: %w", err) + } + + // create archive f, err := os.Create(destTarGz) if err != nil { - return fmt.Errorf("creating archive file: %w", err) + return fmt.Errorf("failed to creating archive file: %w", err) } defer f.Close() - gw := gzip.NewWriter(f) - defer gw.Close() - - tw := tar.NewWriter(gw) - defer tw.Close() - - // Walk again to add files - err = filepath.Walk(srcDir, func(path string, info os.FileInfo, walkErr error) error { - if walkErr != nil { - return walkErr - } - // Create tar header - header, err := tar.FileInfoHeader(info, path) - if err != nil { - return err - } - // Preserve directory structure relative to srcDir - relPath, err := filepath.Rel(filepath.Dir(srcDir), path) - if err != nil { - return err - } - header.Name = relPath - - if err := tw.WriteHeader(header); err != nil { - return err - } - if info.Mode().IsRegular() { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - if _, err := io.Copy(tw, file); err != nil { - return err - } - } - return nil - }) - if err != nil { - return fmt.Errorf("writing tar entries: %w", err) + if err := archive.Tar(f, srcDir); err != nil { + return fmt.Errorf("failed archiving files") } - // 4) Update state file now := time.Now().UTC().Format(time.RFC3339) if err := os.WriteFile(stateFile, []byte(now), 0644); err != nil { return fmt.Errorf("updating state file: %w", err) diff --git a/cmd/cli/commands/sync/sync.go b/cmd/cli/commands/sync/sync.go index 8be11bd..146a2a5 100644 --- a/cmd/cli/commands/sync/sync.go +++ b/cmd/cli/commands/sync/sync.go @@ -2,10 +2,10 @@ package sync import ( "cloudsave/cmd/cli/tools/prompt" - "cloudsave/pkg/game" + "cloudsave/cmd/cli/tools/prompt/credentials" "cloudsave/pkg/remote" "cloudsave/pkg/remote/client" - "cloudsave/pkg/tools/prompt/credentials" + "cloudsave/pkg/repository" "context" "errors" "flag" @@ -35,7 +35,7 @@ func (p *SyncCmd) SetFlags(f *flag.FlagSet) { } func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { - games, err := game.All() + games, err := repository.All() if err != nil { fmt.Fprintln(os.Stderr, "error: failed to load datastore:", err) return subcommands.ExitFailure @@ -72,7 +72,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) continue } - hlocal, err := game.Hash(r.GameID) + hlocal, err := repository.Hash(r.GameID) if err != nil { slog.Error(err.Error()) continue @@ -84,7 +84,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) continue } - vlocal, err := game.Version(r.GameID) + vlocal, err := repository.Version(r.GameID) if err != nil { slog.Error(err.Error()) continue @@ -99,7 +99,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) if hlocal == hremote { if vlocal != remoteMetadata.Version { slog.Debug("version is not the same, but the hash is equal. Updating local database") - if err := game.SetVersion(r.GameID, remoteMetadata.Version); err != nil { + if err := repository.SetVersion(r.GameID, remoteMetadata.Version); err != nil { fmt.Fprintln(os.Stderr, "error: failed to synchronize version number:", err) continue } @@ -121,11 +121,11 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } - if err := game.SetVersion(r.GameID, remoteMetadata.Version); err != nil { + if err := repository.SetVersion(r.GameID, remoteMetadata.Version); err != nil { fmt.Fprintln(os.Stderr, "error: failed to synchronize version number:", err) continue } - if err := game.SetDate(r.GameID, remoteMetadata.Date); err != nil { + if err := repository.SetDate(r.GameID, remoteMetadata.Date); err != nil { fmt.Fprintln(os.Stderr, "error: failed to synchronize date:", err) continue } @@ -144,8 +144,8 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitSuccess } -func conflict(gameID string, m, remoteMetadata game.Metadata, cli *client.Client) error { - g, err := game.One(gameID) +func conflict(gameID string, m, remoteMetadata repository.Metadata, cli *client.Client) error { + g, err := repository.One(gameID) if err != nil { slog.Warn("a conflict was found but the game is not found in the database") slog.Debug("debug info", "gameID", gameID) @@ -174,10 +174,10 @@ func conflict(gameID string, m, remoteMetadata game.Metadata, cli *client.Client if err := pull(gameID, cli); err != nil { return fmt.Errorf("failed to push: %w", err) } - if err := game.SetVersion(gameID, remoteMetadata.Version); err != nil { + if err := repository.SetVersion(gameID, remoteMetadata.Version); err != nil { return fmt.Errorf("failed to synchronize version number: %w", err) } - if err := game.SetDate(gameID, remoteMetadata.Date); err != nil { + if err := repository.SetDate(gameID, remoteMetadata.Date); err != nil { return fmt.Errorf("failed to synchronize date: %w", err) } } @@ -185,14 +185,14 @@ func conflict(gameID string, m, remoteMetadata game.Metadata, cli *client.Client return nil } -func push(gameID string, m game.Metadata, cli *client.Client) error { - archivePath := filepath.Join(game.DatastorePath(), gameID, "data.tar.gz") +func push(gameID string, m repository.Metadata, cli *client.Client) error { + archivePath := filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz") return cli.Push(gameID, archivePath, m) } func pull(gameID string, cli *client.Client) error { - archivePath := filepath.Join(game.DatastorePath(), gameID, "data.tar.gz") + archivePath := filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz") return cli.Pull(gameID, archivePath) } diff --git a/cmd/cli/commands/version/version.go b/cmd/cli/commands/version/version.go index 79c5bf5..98d6d20 100644 --- a/cmd/cli/commands/version/version.go +++ b/cmd/cli/commands/version/version.go @@ -3,7 +3,7 @@ package version import ( "cloudsave/pkg/constants" "cloudsave/pkg/remote/client" - "cloudsave/pkg/tools/prompt/credentials" + "cloudsave/cmd/cli/tools/prompt/credentials" "context" "flag" "fmt" diff --git a/pkg/tools/prompt/credentials/credentials.go b/cmd/cli/tools/prompt/credentials/credentials.go similarity index 100% rename from pkg/tools/prompt/credentials/credentials.go rename to cmd/cli/tools/prompt/credentials/credentials.go diff --git a/cmd/server/api/api.go b/cmd/server/api/api.go index aaa5e51..a2eb870 100644 --- a/cmd/server/api/api.go +++ b/cmd/server/api/api.go @@ -2,7 +2,7 @@ package api import ( "cloudsave/cmd/server/data" - "cloudsave/pkg/game" + "cloudsave/pkg/repository" "crypto/md5" "encoding/hex" "encoding/json" @@ -61,6 +61,7 @@ func NewServer(documentRoot string, creds map[string]string, port int) *HTTPServ // Data routes gamesRouter.Group(func(saveRouter chi.Router) { saveRouter.Post("/{id}/data", s.upload) + saveRouter.Post("/{id}/hist/data", s.histUpload) saveRouter.Get("/{id}/data", s.download) saveRouter.Get("/{id}/hash", s.hash) saveRouter.Get("/{id}/metadata", s.metadata) @@ -78,7 +79,7 @@ func NewServer(documentRoot string, creds map[string]string, port int) *HTTPServ func (s HTTPServer) all(w http.ResponseWriter, r *http.Request) { path := filepath.Join(s.documentRoot, "data") - datastore := make([]game.Metadata, 0) + datastore := make([]repository.Metadata, 0) if _, err := os.Stat(path); err != nil { if errors.Is(err, os.ErrNotExist) { @@ -104,7 +105,7 @@ func (s HTTPServer) all(w http.ResponseWriter, r *http.Request) { continue } - var m game.Metadata + var m repository.Metadata err = json.Unmarshal(content, &m) if err != nil { fmt.Fprintf(os.Stderr, "corrupted datastore: failed to parse %s/metadata.json: %s", d.Name(), err) @@ -209,6 +210,49 @@ func (s HTTPServer) upload(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) } +func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) { + const ( + sizeLimit int64 = 500 << 20 // 500 MB + ) + + id := chi.URLParam(r, "id") + dt, err := formParseDate("date", r.MultipartForm.Value) + if err != nil { + fmt.Fprintln(os.Stderr, "error: failed to load payload:", err) + badRequest("bad payload", w, r) + return + } + + // Limit max upload size + r.Body = http.MaxBytesReader(w, r.Body, sizeLimit) + + // Parse multipart form + err = r.ParseMultipartForm(sizeLimit) + if err != nil { + fmt.Fprintln(os.Stderr, "error: failed to load payload:", err) + badRequest("bad payload", w, r) + return + } + + // Retrieve file + file, _, err := r.FormFile("payload") + if err != nil { + fmt.Fprintln(os.Stderr, "error: cannot find payload in the form:", err) + badRequest("payload not found", w, r) + return + } + defer file.Close() + + if err := data.WriteHist(id, s.documentRoot, dt, file); err != nil { + fmt.Fprintln(os.Stderr, "error: failed to write file to disk:", err) + internalServerError(w, r) + return + } + + // Respond success + w.WriteHeader(http.StatusCreated) +} + func (s HTTPServer) hash(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "id") path := filepath.Clean(filepath.Join(s.documentRoot, "data", id)) @@ -272,7 +316,7 @@ func (s HTTPServer) metadata(w http.ResponseWriter, r *http.Request) { } defer f.Close() - var metadata game.Metadata + var metadata repository.Metadata d := json.NewDecoder(f) err = d.Decode(&metadata) if err != nil { @@ -284,49 +328,67 @@ func (s HTTPServer) metadata(w http.ResponseWriter, r *http.Request) { ok(metadata, w, r) } -func parseFormMetadata(gameID string, values map[string][]string) (game.Metadata, error) { +func parseFormMetadata(gameID string, values map[string][]string) (repository.Metadata, error) { var name string if v, ok := values["name"]; ok { if len(v) == 0 { - return game.Metadata{}, fmt.Errorf("error: corrupted metadata") + return repository.Metadata{}, fmt.Errorf("error: corrupted metadata") } name = v[0] } else { - return game.Metadata{}, fmt.Errorf("error: cannot find metadata in the form") + return repository.Metadata{}, fmt.Errorf("error: cannot find metadata in the form") } var version int if v, ok := values["version"]; ok { if len(v) == 0 { - return game.Metadata{}, fmt.Errorf("error: corrupted metadata") + return repository.Metadata{}, fmt.Errorf("error: corrupted metadata") } if v, err := strconv.Atoi(v[0]); err == nil { version = v } else { - return game.Metadata{}, err + return repository.Metadata{}, err } } else { - return game.Metadata{}, fmt.Errorf("error: cannot find metadata in the form") + return repository.Metadata{}, fmt.Errorf("error: cannot find metadata in the form") } var date time.Time if v, ok := values["date"]; ok { if len(v) == 0 { - return game.Metadata{}, fmt.Errorf("error: corrupted metadata") + return repository.Metadata{}, fmt.Errorf("error: corrupted metadata") } if v, err := time.Parse(time.RFC3339, v[0]); err == nil { date = v } else { - return game.Metadata{}, err + return repository.Metadata{}, err } } else { - return game.Metadata{}, fmt.Errorf("error: cannot find metadata in the form") + return repository.Metadata{}, fmt.Errorf("error: cannot find metadata in the form") } - return game.Metadata{ + return repository.Metadata{ ID: gameID, Version: version, Name: name, Date: date, }, nil } + +func formParseDate(key string, values map[string][]string) (time.Time, error) { + var date time.Time + if v, ok := values[key]; ok { + if len(v) == 0 { + return time.Time{}, fmt.Errorf("error: corrupted metadata") + } + if v, err := time.Parse(time.RFC3339, v[0]); err == nil { + date = v + } else { + return time.Time{}, err + } + } else { + return time.Time{}, fmt.Errorf("error: cannot find metadata in the form") + } + + return date, nil +} diff --git a/cmd/server/data/data.go b/cmd/server/data/data.go index cddfe7f..58213c2 100644 --- a/cmd/server/data/data.go +++ b/cmd/server/data/data.go @@ -1,12 +1,13 @@ package data import ( - "cloudsave/pkg/game" + "cloudsave/pkg/repository" "encoding/json" "fmt" "io" "os" "path/filepath" + "time" ) func Write(gameID, documentRoot string, r io.Reader) error { @@ -39,7 +40,37 @@ func Write(gameID, documentRoot string, r io.Reader) error { return nil } -func UpdateMetadata(gameID, documentRoot string, m game.Metadata) error { +func WriteHist(gameID, documentRoot string, dt time.Time, r io.Reader) error { + dataFolderPath := filepath.Join(documentRoot, "data", gameID, "hist") + partPath := filepath.Join(dataFolderPath, dt.Format("2006-01-02T15-04-05Z07-00")+".data.tar.gz.part") + finalFilePath := filepath.Join(dataFolderPath, dt.Format("2006-01-02T15-04-05Z07-00")+".data.tar.gz") + + if err := makeDataFolder(gameID, documentRoot); err != nil { + return err + } + + f, err := os.OpenFile(partPath, os.O_CREATE|os.O_WRONLY, 0740) + if err != nil { + return err + } + + if _, err := io.Copy(f, r); err != nil { + f.Close() + if err := os.Remove(partPath); err != nil { + return fmt.Errorf("failed to write the file and cannot clean the folder: %w", err) + } + return fmt.Errorf("failed to write the file: %w", err) + } + f.Close() + + if err := os.Rename(partPath, finalFilePath); err != nil { + return err + } + + return nil +} + +func UpdateMetadata(gameID, documentRoot string, m repository.Metadata) error { if err := makeDataFolder(gameID, documentRoot); err != nil { return err } @@ -56,5 +87,13 @@ func UpdateMetadata(gameID, documentRoot string, m game.Metadata) error { } func makeDataFolder(gameID, documentRoot string) error { - return os.MkdirAll(filepath.Join(documentRoot, "data", gameID), 0740) + if err := os.MkdirAll(filepath.Join(documentRoot, "data", gameID), 0740); err != nil { + return err + } + + if err := os.MkdirAll(filepath.Join(documentRoot, "data", gameID, "hist"), 0740); err != nil { + return err + } + + return nil } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 4e14d23..1dc2fa6 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -1,5 +1,5 @@ package constants -const Version = "0.0.1" +const Version = "0.0.2" const ApiVersion = 1 diff --git a/pkg/remote/client/client.go b/pkg/remote/client/client.go index cafca98..c649a1a 100644 --- a/pkg/remote/client/client.go +++ b/pkg/remote/client/client.go @@ -2,8 +2,8 @@ package client import ( "bytes" - "cloudsave/pkg/game" "cloudsave/pkg/remote/obj" + "cloudsave/pkg/repository" customtime "cloudsave/pkg/tools/time" "encoding/json" "errors" @@ -117,19 +117,19 @@ func (c *Client) Hash(gameID string) (string, error) { return "", errors.New("invalid payload sent by the server") } -func (c *Client) Metadata(gameID string) (game.Metadata, error) { +func (c *Client) Metadata(gameID string) (repository.Metadata, error) { u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "metadata") if err != nil { - return game.Metadata{}, err + return repository.Metadata{}, err } o, err := c.get(u) if err != nil { - return game.Metadata{}, err + return repository.Metadata{}, err } if m, ok := (o.Data).(map[string]any); ok { - gm := game.Metadata{ + gm := repository.Metadata{ ID: m["id"].(string), Name: m["name"].(string), Version: int(m["version"].(float64)), @@ -138,10 +138,10 @@ func (c *Client) Metadata(gameID string) (game.Metadata, error) { return gm, nil } - return game.Metadata{}, errors.New("invalid payload sent by the server") + return repository.Metadata{}, errors.New("invalid payload sent by the server") } -func (c *Client) Push(gameID, archivePath string, m game.Metadata) error { +func (c *Client) Push(gameID, archivePath string, m repository.Metadata) error { u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "data") if err != nil { return err @@ -271,7 +271,7 @@ func (c *Client) Ping() error { return nil } -func (c *Client) All() ([]game.Metadata, error) { +func (c *Client) All() ([]repository.Metadata, error) { u, err := url.JoinPath(c.baseURL, "api", "v1", "games") if err != nil { return nil, err @@ -283,10 +283,10 @@ func (c *Client) All() ([]game.Metadata, error) { } if games, ok := (o.Data).([]any); ok { - var res []game.Metadata + var res []repository.Metadata for _, g := range games { if v, ok := g.(map[string]any); ok { - gm := game.Metadata{ + gm := repository.Metadata{ ID: v["id"].(string), Name: v["name"].(string), Version: int(v["version"].(float64)), diff --git a/pkg/game/game.go b/pkg/repository/repository.go similarity index 77% rename from pkg/game/game.go rename to pkg/repository/repository.go index f3ca35d..bd2a102 100644 --- a/pkg/game/game.go +++ b/pkg/repository/repository.go @@ -1,4 +1,4 @@ -package game +package repository import ( "cloudsave/pkg/tools/id" @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "os" "path/filepath" "time" @@ -135,6 +136,65 @@ func One(gameID string) (Metadata, error) { return m, nil } +func Archive(gameID string) error { + path := filepath.Join(datastorepath, gameID, "data.tar.gz") + + // open old + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return fmt.Errorf("failed to open old file: %w", err) + } + defer f.Close() + + histDirPath := filepath.Join(datastorepath, gameID, "hist") + if err := os.MkdirAll(histDirPath, 0740); err != nil { + return fmt.Errorf("failed to make 'hist' directory") + } + + d, err := os.ReadDir(histDirPath) + if err != nil { + return fmt.Errorf("failed to open 'hist' directory") + } + + // keep the dir under 6 files + if len(d) > 5 { + var oldest *fs.FileInfo + for _, hfile := range d { + finfo, err := hfile.Info() + if err != nil { + return fmt.Errorf("failed to read backup file: %w", err) + } + + if oldest == nil { + oldest = &finfo + continue + } + + if finfo.ModTime().Before((*oldest).ModTime()) { + oldest = &finfo + } + } + + if err := os.Remove((*oldest).Name()); err != nil { + return fmt.Errorf("failed to remove the oldest backup file: %w", err) + } + } + + // open new + nf, err := os.OpenFile(filepath.Join(datastorepath, gameID, "hist", time.Now().Format("2006-01-02T15-04-05Z07-00")+".data.tar.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) + if err != nil { + return fmt.Errorf("failed to open new file: %w", err) + } + defer nf.Close() + + // copy + if _, err := io.Copy(nf, f); err != nil { + return fmt.Errorf("failed to copy data: %w", err) + } + + return nil +} + func DatastorePath() string { return datastorepath } diff --git a/pkg/tools/archive/archive.go b/pkg/tools/archive/archive.go index 63b1688..97f40bb 100644 --- a/pkg/tools/archive/archive.go +++ b/pkg/tools/archive/archive.go @@ -3,6 +3,7 @@ package archive import ( "archive/tar" "compress/gzip" + "fmt" "io" "os" "path/filepath" @@ -71,3 +72,49 @@ func Untar(file io.Reader, path string) error { } } } + +func Tar(file io.Writer, path string) error { + gw := gzip.NewWriter(file) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + // Walk again to add files + err := filepath.Walk(path, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + // Create tar header + header, err := tar.FileInfoHeader(info, path) + if err != nil { + return err + } + // Preserve directory structure relative to srcDir + relPath, err := filepath.Rel(filepath.Dir(path), path) + if err != nil { + return err + } + header.Name = relPath + + if err := tw.WriteHeader(header); err != nil { + return err + } + if info.Mode().IsRegular() { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + if _, err := io.Copy(tw, file); err != nil { + return err + } + } + return nil + }) + if err != nil { + return fmt.Errorf("writing tar entries: %w", err) + } + + return nil +} From 30b76e18871ca042d8258d7ec3bea6e8a4b902f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Wed, 30 Jul 2025 15:14:01 +0200 Subject: [PATCH 2/5] push backup --- .vscode/launch.json | 2 +- cmd/cli/commands/run/run.go | 15 +++-- cmd/cli/commands/sync/sync.go | 43 +++++++++++-- cmd/server/api/api.go | 52 +++++++-------- cmd/server/data/data.go | 42 ++++++++++-- go.mod | 1 + go.sum | 2 + pkg/remote/client/client.go | 118 +++++++++++++++++++++++----------- pkg/repository/repository.go | 102 +++++++++++++++-------------- pkg/tools/archive/archive.go | 38 ++++++----- pkg/tools/hash/hash.go | 23 +++++++ 11 files changed, 290 insertions(+), 148 deletions(-) create mode 100644 pkg/tools/hash/hash.go diff --git a/.vscode/launch.json b/.vscode/launch.json index 3d7adf9..8e7ace7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,7 @@ "type": "go", "request": "launch", "mode": "auto", - "args": ["list", "-a", "http://localhost:8080"], + "args": ["run"], "console": "integratedTerminal", "program": "${workspaceFolder}/cmd/cli" } diff --git a/cmd/cli/commands/run/run.go b/cmd/cli/commands/run/run.go index 42aec35..43bd28e 100644 --- a/cmd/cli/commands/run/run.go +++ b/cmd/cli/commands/run/run.go @@ -37,11 +37,7 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s return subcommands.ExitFailure } - pg := progressbar.New(len(datastore)) - defer pg.Close() - for _, metadata := range datastore { - pg.Describe("Scanning " + metadata.Name + "...") metadataPath := filepath.Join(repository.DatastorePath(), metadata.ID) //todo transaction err := archiveIfChanged(metadata.ID, metadata.Path, filepath.Join(metadataPath, "data.tar.gz"), filepath.Join(metadataPath, ".last_run")) @@ -57,11 +53,8 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s fmt.Fprintf(os.Stderr, "error: cannot process the data of %s: %s\n", metadata.ID, err) return subcommands.ExitFailure } - pg.Add(1) } - pg.Finish() - return subcommands.ExitSuccess } @@ -69,6 +62,11 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s // in srcDir has a modification time > the last run time stored in stateFile. // After archiving, it updates stateFile to the current time. func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error { + pg := progressbar.New(-1) + defer pg.Close() + + pg.Describe("Scanning " + gameID + "...") + // load last run time var lastRun time.Time data, err := os.ReadFile(stateFile) @@ -99,15 +97,18 @@ func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error { } if !changed { + pg.Finish() return nil } // make a backup + pg.Describe("Backup current data...") if err := repository.Archive(gameID); err != nil { return fmt.Errorf("failed to archive data: %w", err) } // create archive + pg.Describe("Archiving new data...") f, err := os.Create(destTarGz) if err != nil { return fmt.Errorf("failed to creating archive file: %w", err) diff --git a/cmd/cli/commands/sync/sync.go b/cmd/cli/commands/sync/sync.go index 146a2a5..a73ce81 100644 --- a/cmd/cli/commands/sync/sync.go +++ b/cmd/cli/commands/sync/sync.go @@ -65,10 +65,13 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } if !exists { - if err := push(r.GameID, g, cli); err != nil { + if err := push(g, cli); err != nil { fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } + if err := pushBackup(g, cli); err != nil { + slog.Warn("failed to push backup files", "err", err) + } continue } @@ -96,6 +99,10 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) continue } + if err := pushBackup(g, cli); err != nil { + slog.Warn("failed to push backup files", "err", err) + } + if hlocal == hremote { if vlocal != remoteMetadata.Version { slog.Debug("version is not the same, but the hash is equal. Updating local database") @@ -109,7 +116,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } if vlocal > remoteMetadata.Version { - if err := push(r.GameID, g, cli); err != nil { + if err := push(g, cli); err != nil { fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } @@ -164,7 +171,7 @@ func conflict(gameID string, m, remoteMetadata repository.Metadata, cli *client. switch res { case prompt.My: { - if err := push(gameID, m, cli); err != nil { + if err := push(m, cli); err != nil { return fmt.Errorf("failed to push: %w", err) } } @@ -185,10 +192,34 @@ func conflict(gameID string, m, remoteMetadata repository.Metadata, cli *client. return nil } -func push(gameID string, m repository.Metadata, cli *client.Client) error { - archivePath := filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz") +func push(m repository.Metadata, cli *client.Client) error { + archivePath := filepath.Join(repository.DatastorePath(), m.ID, "data.tar.gz") - return cli.Push(gameID, archivePath, m) + return cli.PushSave(archivePath, m) +} + +func pushBackup(m repository.Metadata, cli *client.Client) error { + bs, err := repository.Archives(m.ID) + if err != nil { + return err + } + + for _, b := range bs { + binfo, err := cli.ArchiveInfo(m.ID, b.UUID) + if err != nil { + if !errors.Is(err, client.ErrNotFound) { + return fmt.Errorf("failed to get remote information about the backup file: %w", err) + } + } + + if binfo.MD5 != b.MD5 { + if err := cli.PushBackup(b, m); err != nil { + return fmt.Errorf("failed to push backup: %w", err) + } + } + + } + return nil } func pull(gameID string, cli *client.Client) error { diff --git a/cmd/server/api/api.go b/cmd/server/api/api.go index a2eb870..50121dd 100644 --- a/cmd/server/api/api.go +++ b/cmd/server/api/api.go @@ -61,7 +61,8 @@ func NewServer(documentRoot string, creds map[string]string, port int) *HTTPServ // Data routes gamesRouter.Group(func(saveRouter chi.Router) { saveRouter.Post("/{id}/data", s.upload) - saveRouter.Post("/{id}/hist/data", s.histUpload) + saveRouter.Post("/{id}/hist/{uuid}/data", s.histUpload) + saveRouter.Get("/{id}/hist/{uuid}/info", s.histExists) saveRouter.Get("/{id}/data", s.download) saveRouter.Get("/{id}/hash", s.hash) saveRouter.Get("/{id}/metadata", s.metadata) @@ -215,19 +216,14 @@ func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) { sizeLimit int64 = 500 << 20 // 500 MB ) - id := chi.URLParam(r, "id") - dt, err := formParseDate("date", r.MultipartForm.Value) - if err != nil { - fmt.Fprintln(os.Stderr, "error: failed to load payload:", err) - badRequest("bad payload", w, r) - return - } + gameID := chi.URLParam(r, "id") + uuid := chi.URLParam(r, "uuid") // Limit max upload size r.Body = http.MaxBytesReader(w, r.Body, sizeLimit) // Parse multipart form - err = r.ParseMultipartForm(sizeLimit) + err := r.ParseMultipartForm(sizeLimit) if err != nil { fmt.Fprintln(os.Stderr, "error: failed to load payload:", err) badRequest("bad payload", w, r) @@ -243,7 +239,7 @@ func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) { } defer file.Close() - if err := data.WriteHist(id, s.documentRoot, dt, file); err != nil { + if err := data.WriteHist(gameID, s.documentRoot, uuid, file); err != nil { fmt.Fprintln(os.Stderr, "error: failed to write file to disk:", err) internalServerError(w, r) return @@ -253,6 +249,24 @@ func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) } +func (s HTTPServer) histExists(w http.ResponseWriter, r *http.Request) { + gameID := chi.URLParam(r, "id") + uuid := chi.URLParam(r, "uuid") + + finfo, err := data.ArchiveInfo(gameID, s.documentRoot, uuid) + if err != nil { + if errors.Is(err, data.ErrBackupNotExists) { + notFound("backup not found", w, r) + return + } + fmt.Fprintln(os.Stderr, "error: failed to read data:", err) + internalServerError(w, r) + return + } + + ok(finfo, w, r) +} + func (s HTTPServer) hash(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "id") path := filepath.Clean(filepath.Join(s.documentRoot, "data", id)) @@ -374,21 +388,3 @@ func parseFormMetadata(gameID string, values map[string][]string) (repository.Me Date: date, }, nil } - -func formParseDate(key string, values map[string][]string) (time.Time, error) { - var date time.Time - if v, ok := values[key]; ok { - if len(v) == 0 { - return time.Time{}, fmt.Errorf("error: corrupted metadata") - } - if v, err := time.Parse(time.RFC3339, v[0]); err == nil { - date = v - } else { - return time.Time{}, err - } - } else { - return time.Time{}, fmt.Errorf("error: cannot find metadata in the form") - } - - return date, nil -} diff --git a/cmd/server/data/data.go b/cmd/server/data/data.go index 58213c2..6b1c553 100644 --- a/cmd/server/data/data.go +++ b/cmd/server/data/data.go @@ -2,12 +2,17 @@ package data import ( "cloudsave/pkg/repository" + "cloudsave/pkg/tools/hash" "encoding/json" + "errors" "fmt" "io" "os" "path/filepath" - "time" +) + +var ( + ErrBackupNotExists error = errors.New("backup not found") ) func Write(gameID, documentRoot string, r io.Reader) error { @@ -40,15 +45,19 @@ func Write(gameID, documentRoot string, r io.Reader) error { return nil } -func WriteHist(gameID, documentRoot string, dt time.Time, r io.Reader) error { - dataFolderPath := filepath.Join(documentRoot, "data", gameID, "hist") - partPath := filepath.Join(dataFolderPath, dt.Format("2006-01-02T15-04-05Z07-00")+".data.tar.gz.part") - finalFilePath := filepath.Join(dataFolderPath, dt.Format("2006-01-02T15-04-05Z07-00")+".data.tar.gz") +func WriteHist(gameID, documentRoot, uuid string, r io.Reader) error { + dataFolderPath := filepath.Join(documentRoot, "data", gameID, "hist", uuid) + partPath := filepath.Join(dataFolderPath, "data.tar.gz.part") + finalFilePath := filepath.Join(dataFolderPath, "data.tar.gz") if err := makeDataFolder(gameID, documentRoot); err != nil { return err } + if err := os.MkdirAll(dataFolderPath, 0740); err != nil { + return err + } + f, err := os.OpenFile(partPath, os.O_CREATE|os.O_WRONLY, 0740) if err != nil { return err @@ -86,6 +95,29 @@ func UpdateMetadata(gameID, documentRoot string, m repository.Metadata) error { return e.Encode(m) } +func ArchiveInfo(gameID, documentRoot, uuid string) (repository.Backup, error) { + dataFolderPath := filepath.Join(documentRoot, "data", gameID, "hist", uuid, "data.tar.gz") + + finfo, err := os.Stat(dataFolderPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return repository.Backup{}, ErrBackupNotExists + } + return repository.Backup{}, err + } + + h, err := hash.FileMD5(dataFolderPath) + if err != nil { + return repository.Backup{}, fmt.Errorf("failed to calculate file md5: %w", err) + } + + return repository.Backup{ + CreatedAt: finfo.ModTime(), + UUID: uuid, + MD5: h, + }, nil +} + func makeDataFolder(gameID, documentRoot string) error { if err := os.MkdirAll(filepath.Join(documentRoot, "data", gameID), 0740); err != nil { return err diff --git a/go.mod b/go.mod index 9d82114..851103c 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.24 require ( github.com/go-chi/chi/v5 v5.2.1 github.com/google/subcommands v1.2.0 + github.com/google/uuid v1.6.0 github.com/schollz/progressbar/v3 v3.18.0 golang.org/x/crypto v0.38.0 golang.org/x/term v0.32.0 diff --git a/go.sum b/go.sum index aeaaa97..29e363d 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8= github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= diff --git a/pkg/remote/client/client.go b/pkg/remote/client/client.go index c649a1a..72df438 100644 --- a/pkg/remote/client/client.go +++ b/pkg/remote/client/client.go @@ -35,6 +35,10 @@ type ( } ) +var ( + ErrNotFound error = errors.New("not found") +) + func New(baseURL, username, password string) *Client { return &Client{ baseURL: baseURL, @@ -141,59 +145,45 @@ func (c *Client) Metadata(gameID string) (repository.Metadata, error) { return repository.Metadata{}, errors.New("invalid payload sent by the server") } -func (c *Client) Push(gameID, archivePath string, m repository.Metadata) error { - u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "data") +func (c *Client) PushSave(archivePath string, m repository.Metadata) error { + u, err := url.JoinPath(c.baseURL, "api", "v1", "games", m.ID, "data") if err != nil { return err } - f, err := os.OpenFile(archivePath, os.O_RDONLY, 0) - if err != nil { - return fmt.Errorf("failed to open file: %w", err) - } - defer f.Close() + return c.push(u, archivePath, m) +} - buf := new(bytes.Buffer) - writer := multipart.NewWriter(buf) - - part, err := writer.CreateFormFile("payload", "data.tar.gz") +func (c *Client) PushBackup(archiveMetadata repository.Backup, m repository.Metadata) error { + u, err := url.JoinPath(c.baseURL, "api", "v1", "games", m.ID, "hist", archiveMetadata.UUID, "data") if err != nil { return err } - if _, err := io.Copy(part, f); err != nil { - return fmt.Errorf("failed to copy data: %w", err) - } + return c.push(u, archiveMetadata.ArchivePath, m) +} - writer.WriteField("name", m.Name) - writer.WriteField("version", strconv.Itoa(m.Version)) - writer.WriteField("date", m.Date.Format(time.RFC3339)) - - if err := writer.Close(); err != nil { - return err - } - - cli := http.Client{} - - req, err := http.NewRequest("POST", u, buf) +func (c *Client) ArchiveInfo(gameID, uuid string) (repository.Backup, error) { + u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "hist", uuid, "info") if err != nil { - return err + return repository.Backup{}, err } - req.SetBasicAuth(c.username, c.password) - req.Header.Set("Content-Type", writer.FormDataContentType()) - - res, err := cli.Do(req) + o, err := c.get(u) if err != nil { - return err - } - defer res.Body.Close() - - if res.StatusCode != 201 { - return fmt.Errorf("server returns an unexpected status code: %s (expected 201)", res.Status) + return repository.Backup{}, err } - return nil + if m, ok := (o.Data).(map[string]any); ok { + b := repository.Backup{ + UUID: m["uuid"].(string), + CreatedAt: customtime.MustParse(time.RFC3339, m["created_at"].(string)), + MD5: m["md5"].(string), + } + return b, nil + } + + return repository.Backup{}, errors.New("invalid payload sent by the server") } func (c *Client) Pull(gameID, archivePath string) error { @@ -318,6 +308,10 @@ func (c *Client) get(url string) (obj.HTTPObject, error) { } defer res.Body.Close() + if res.StatusCode == 404 { + return obj.HTTPObject{}, ErrNotFound + } + if res.StatusCode != 200 { return obj.HTTPObject{}, fmt.Errorf("server returns an unexpected status code: %d %s (expected 200)", res.StatusCode, res.Status) } @@ -331,3 +325,53 @@ func (c *Client) get(url string) (obj.HTTPObject, error) { return httpObject, nil } + +func (c *Client) push(u, archivePath string, m repository.Metadata) error { + f, err := os.OpenFile(archivePath, os.O_RDONLY, 0) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer f.Close() + + buf := new(bytes.Buffer) + writer := multipart.NewWriter(buf) + + part, err := writer.CreateFormFile("payload", "data.tar.gz") + if err != nil { + return err + } + + if _, err := io.Copy(part, f); err != nil { + return fmt.Errorf("failed to copy data: %w", err) + } + + writer.WriteField("name", m.Name) + writer.WriteField("version", strconv.Itoa(m.Version)) + writer.WriteField("date", m.Date.Format(time.RFC3339)) + + if err := writer.Close(); err != nil { + return err + } + + cli := http.Client{} + + req, err := http.NewRequest("POST", u, buf) + if err != nil { + return err + } + + req.SetBasicAuth(c.username, c.password) + req.Header.Set("Content-Type", writer.FormDataContentType()) + + res, err := cli.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + + if res.StatusCode != 201 { + return fmt.Errorf("server returns an unexpected status code: %s (expected 201)", res.Status) + } + + return nil +} diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index bd2a102..458f5f1 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -1,16 +1,17 @@ package repository import ( + "cloudsave/pkg/tools/hash" "cloudsave/pkg/tools/id" - "crypto/md5" - "encoding/hex" "encoding/json" + "errors" "fmt" "io" - "io/fs" "os" "path/filepath" "time" + + "github.com/google/uuid" ) type ( @@ -21,6 +22,13 @@ type ( Version int `json:"version"` Date time.Time `json:"date"` } + + Backup struct { + CreatedAt time.Time `json:"created_at"` + MD5 string `json:"md5"` + UUID string `json:"uuid"` + ArchivePath string `json:"-"` + } ) var ( @@ -142,46 +150,20 @@ func Archive(gameID string) error { // open old f, err := os.OpenFile(path, os.O_RDONLY, 0) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } return fmt.Errorf("failed to open old file: %w", err) } defer f.Close() - histDirPath := filepath.Join(datastorepath, gameID, "hist") + histDirPath := filepath.Join(datastorepath, gameID, "hist", uuid.NewString()) if err := os.MkdirAll(histDirPath, 0740); err != nil { - return fmt.Errorf("failed to make 'hist' directory") - } - - d, err := os.ReadDir(histDirPath) - if err != nil { - return fmt.Errorf("failed to open 'hist' directory") - } - - // keep the dir under 6 files - if len(d) > 5 { - var oldest *fs.FileInfo - for _, hfile := range d { - finfo, err := hfile.Info() - if err != nil { - return fmt.Errorf("failed to read backup file: %w", err) - } - - if oldest == nil { - oldest = &finfo - continue - } - - if finfo.ModTime().Before((*oldest).ModTime()) { - oldest = &finfo - } - } - - if err := os.Remove((*oldest).Name()); err != nil { - return fmt.Errorf("failed to remove the oldest backup file: %w", err) - } + return fmt.Errorf("failed to make directory: %w", err) } // open new - nf, err := os.OpenFile(filepath.Join(datastorepath, gameID, "hist", time.Now().Format("2006-01-02T15-04-05Z07-00")+".data.tar.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) + nf, err := os.OpenFile(filepath.Join(histDirPath, "data.tar.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) if err != nil { return fmt.Errorf("failed to open new file: %w", err) } @@ -195,6 +177,43 @@ func Archive(gameID string) error { return nil } +func Archives(gameID string) ([]Backup, error) { + histDirPath := filepath.Join(datastorepath, gameID, "hist") + if err := os.MkdirAll(histDirPath, 0740); err != nil { + return nil, fmt.Errorf("failed to make 'hist' directory") + } + + d, err := os.ReadDir(histDirPath) + if err != nil { + return nil, fmt.Errorf("failed to open 'hist' directory") + } + + var res []Backup + for _, f := range d { + finfo, err := f.Info() + if err != nil { + return nil, fmt.Errorf("corrupted datastore: %w", err) + } + path := filepath.Join(histDirPath, finfo.Name()) + archivePath := filepath.Join(path, "data.tar.gz") + + h, err := hash.FileMD5(archivePath) + if err != nil { + return nil, fmt.Errorf("failed to calculate md5 hash: %w", err) + } + + b := Backup{ + CreatedAt: finfo.ModTime(), + UUID: filepath.Base(finfo.Name()), + MD5: h, + ArchivePath: archivePath, + } + + res = append(res, b) + } + return res, nil +} + func DatastorePath() string { return datastorepath } @@ -210,18 +229,7 @@ func Remove(gameID string) error { func Hash(gameID string) (string, error) { path := filepath.Join(datastorepath, gameID, "data.tar.gz") - f, err := os.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return "", err - } - defer f.Close() - - hasher := md5.New() - if _, err := io.Copy(hasher, f); err != nil { - return "", err - } - sum := hasher.Sum(nil) - return hex.EncodeToString(sum), nil + return hash.FileMD5(path) } func Version(gameID string) (int, error) { diff --git a/pkg/tools/archive/archive.go b/pkg/tools/archive/archive.go index 97f40bb..b3a6b7d 100644 --- a/pkg/tools/archive/archive.go +++ b/pkg/tools/archive/archive.go @@ -73,7 +73,7 @@ func Untar(file io.Reader, path string) error { } } -func Tar(file io.Writer, path string) error { +func Tar(file io.Writer, root string) error { gw := gzip.NewWriter(file) defer gw.Close() @@ -81,34 +81,38 @@ func Tar(file io.Writer, path string) error { defer tw.Close() // Walk again to add files - err := filepath.Walk(path, func(path string, info os.FileInfo, walkErr error) error { + err := filepath.Walk(root, func(path string, info os.FileInfo, walkErr error) error { if walkErr != nil { return walkErr } + + path, err := filepath.Rel(root, path) + if err != nil { + return err + } + // Create tar header header, err := tar.FileInfoHeader(info, path) if err != nil { return err } - // Preserve directory structure relative to srcDir - relPath, err := filepath.Rel(filepath.Dir(path), path) - if err != nil { - return err - } - header.Name = relPath + header.Name = path if err := tw.WriteHeader(header); err != nil { return err } - if info.Mode().IsRegular() { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - if _, err := io.Copy(tw, file); err != nil { - return err - } + + if !info.Mode().IsRegular() { + return nil + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + if _, err := io.Copy(tw, file); err != nil { + return err } return nil }) diff --git a/pkg/tools/hash/hash.go b/pkg/tools/hash/hash.go new file mode 100644 index 0000000..a0a93e8 --- /dev/null +++ b/pkg/tools/hash/hash.go @@ -0,0 +1,23 @@ +package hash + +import ( + "crypto/md5" + "encoding/hex" + "io" + "os" +) + +func FileMD5(fp string) (string, error) { + f, err := os.OpenFile(fp, os.O_RDONLY, 0) + if err != nil { + return "", err + } + defer f.Close() + + hasher := md5.New() + if _, err := io.Copy(hasher, f); err != nil { + return "", err + } + sum := hasher.Sum(nil) + return hex.EncodeToString(sum), nil +} From 58c6bc56cfea191d58193384da3b49ad854f5d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Wed, 30 Jul 2025 17:20:43 +0200 Subject: [PATCH 3/5] apply backup --- cmd/cli/commands/apply/apply.go | 69 +++++++++++++++++++++++++++++ cmd/cli/commands/list/list.go | 40 ++++++++++++++--- cmd/cli/commands/run/run.go | 12 ++++- cmd/cli/commands/sync/sync.go | 70 ++++++++++++++++++++++++++++- cmd/server/api/api.go | 78 ++++++++++++++++++++++++++++++++- pkg/remote/client/client.go | 70 +++++++++++++++++++++++++++++ pkg/repository/repository.go | 62 +++++++++++++++++++++++++- 7 files changed, 390 insertions(+), 11 deletions(-) create mode 100644 cmd/cli/commands/apply/apply.go diff --git a/cmd/cli/commands/apply/apply.go b/cmd/cli/commands/apply/apply.go new file mode 100644 index 0000000..0e3e389 --- /dev/null +++ b/cmd/cli/commands/apply/apply.go @@ -0,0 +1,69 @@ +package apply + +import ( + "cloudsave/pkg/repository" + "cloudsave/pkg/tools/archive" + "context" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/google/subcommands" +) + +type ( + ListCmd struct { + } +) + +func (*ListCmd) Name() string { return "apply" } +func (*ListCmd) Synopsis() string { return "apply a backup" } +func (*ListCmd) Usage() string { + return `apply: + Apply a backup +` +} + +func (p *ListCmd) SetFlags(f *flag.FlagSet) { +} + +func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { + if f.NArg() != 2 { + fmt.Fprintln(os.Stderr, "error: missing game ID and/or backup uuid") + return subcommands.ExitUsageError + } + + gameID := f.Arg(0) + uuid := f.Arg(1) + + g, err := repository.One(gameID) + if err != nil { + fmt.Fprintf(os.Stderr, "error: failed to open game metadata: %s\n", err) + return subcommands.ExitFailure + } + + if err := repository.RestoreArchive(gameID, uuid); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to restore backup: %s\n", err) + return subcommands.ExitFailure + } + + if err := os.RemoveAll(g.Path); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to remove old data: %s\n", err) + return subcommands.ExitFailure + } + + file, err := os.OpenFile(filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz"), os.O_RDONLY, 0) + if err != nil { + fmt.Fprintf(os.Stderr, "error: failed to open archive: %s\n", err) + return subcommands.ExitFailure + } + defer file.Close() + + if err := archive.Untar(file, g.Path); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to extract archive: %s\n", err) + return subcommands.ExitFailure + } + + return subcommands.ExitSuccess +} diff --git a/cmd/cli/commands/list/list.go b/cmd/cli/commands/list/list.go index 805c44c..8987ae3 100644 --- a/cmd/cli/commands/list/list.go +++ b/cmd/cli/commands/list/list.go @@ -1,9 +1,9 @@ package list import ( + "cloudsave/cmd/cli/tools/prompt/credentials" "cloudsave/pkg/remote/client" "cloudsave/pkg/repository" - "cloudsave/cmd/cli/tools/prompt/credentials" "context" "flag" "fmt" @@ -15,6 +15,7 @@ import ( type ( ListCmd struct { remote bool + backup bool } ) @@ -28,6 +29,7 @@ func (*ListCmd) Usage() string { func (p *ListCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.remote, "a", false, "list all including remote data") + f.BoolVar(&p.backup, "include-backup", false, "include backup uuids in the output") } func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { @@ -43,20 +45,20 @@ func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitFailure } - if err := remote(f.Arg(0), username, password); err != nil { + if err := remote(f.Arg(0), username, password, p.backup); err != nil { fmt.Fprintln(os.Stderr, "error:", err) return subcommands.ExitFailure } return subcommands.ExitSuccess } - if err := local(); err != nil { + if err := local(p.backup); err != nil { fmt.Fprintln(os.Stderr, "error:", err) return subcommands.ExitFailure } return subcommands.ExitSuccess } -func local() error { +func local(includeBackup bool) error { games, err := repository.All() if err != nil { return fmt.Errorf("failed to load datastore: %w", err) @@ -66,13 +68,25 @@ func local() error { fmt.Println("ID:", g.ID) fmt.Println("Name:", g.Name) fmt.Println("Last Version:", g.Date, "( Version Number", g.Version, ")") + if includeBackup { + bk, err := repository.Archives(g.ID) + if err != nil { + return fmt.Errorf("failed to list backup files: %w", err) + } + if len(bk) > 0 { + fmt.Println("Backup:") + for _, b := range bk { + fmt.Printf(" - %s (%s)\n", b.UUID, b.CreatedAt) + } + } + } fmt.Println("---") } return nil } -func remote(url, username, password string) error { +func remote(url, username, password string, includeBackup bool) error { cli := client.New(url, username, password) if err := cli.Ping(); err != nil { @@ -91,6 +105,22 @@ func remote(url, username, password string) error { fmt.Println("ID:", g.ID) fmt.Println("Name:", g.Name) fmt.Println("Last Version:", g.Date, "( Version Number", g.Version, ")") + if includeBackup { + bk, err := cli.ListArchives(g.ID) + if err != nil { + return fmt.Errorf("failed to list backup files: %w", err) + } + if len(bk) > 0 { + fmt.Println("Backup:") + for _, uuid := range bk { + b, err := cli.ArchiveInfo(g.ID, uuid) + if err != nil { + return fmt.Errorf("failed to list backup files: %w", err) + } + fmt.Printf(" - %s (%s)\n", b.UUID, b.CreatedAt) + } + } + } fmt.Println("---") } diff --git a/cmd/cli/commands/run/run.go b/cmd/cli/commands/run/run.go index 43bd28e..5211bf8 100644 --- a/cmd/cli/commands/run/run.go +++ b/cmd/cli/commands/run/run.go @@ -53,8 +53,10 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s fmt.Fprintf(os.Stderr, "error: cannot process the data of %s: %s\n", metadata.ID, err) return subcommands.ExitFailure } + fmt.Println("✅", metadata.Name) } + fmt.Println("done.") return subcommands.ExitSuccess } @@ -63,7 +65,13 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s // After archiving, it updates stateFile to the current time. func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error { pg := progressbar.New(-1) - defer pg.Close() + destroyPg := func() { + pg.Finish() + pg.Clear() + pg.Close() + + } + defer destroyPg() pg.Describe("Scanning " + gameID + "...") @@ -103,7 +111,7 @@ func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error { // make a backup pg.Describe("Backup current data...") - if err := repository.Archive(gameID); err != nil { + if err := repository.MakeArchive(gameID); err != nil { return fmt.Errorf("failed to archive data: %w", err) } diff --git a/cmd/cli/commands/sync/sync.go b/cmd/cli/commands/sync/sync.go index a73ce81..9fe9e5d 100644 --- a/cmd/cli/commands/sync/sync.go +++ b/cmd/cli/commands/sync/sync.go @@ -16,6 +16,7 @@ import ( "time" "github.com/google/subcommands" + "github.com/schollz/progressbar/v3" ) type ( @@ -51,13 +52,21 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) fmt.Fprintln(os.Stderr, "error: failed to load datastore:", err) return subcommands.ExitFailure } - cli, err := connect(remoteCred, r) if err != nil { fmt.Fprintln(os.Stderr, "error: failed to connect to the remote:", err) return subcommands.ExitFailure } + pg := progressbar.New(-1) + destroyPg := func() { + pg.Finish() + pg.Clear() + pg.Close() + + } + + pg.Describe(fmt.Sprintf("[%s] Checking status...", g.Name)) exists, err := cli.Exists(r.GameID) if err != nil { slog.Error(err.Error()) @@ -65,45 +74,61 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } if !exists { + pg.Describe(fmt.Sprintf("[%s] Pushing data...", g.Name)) if err := push(g, cli); err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } + pg.Describe(fmt.Sprintf("[%s] Pushing backup...", g.Name)) if err := pushBackup(g, cli); err != nil { + destroyPg() slog.Warn("failed to push backup files", "err", err) } continue } + pg.Describe(fmt.Sprintf("[%s] Fetching metadata...", g.Name)) hlocal, err := repository.Hash(r.GameID) if err != nil { + destroyPg() slog.Error(err.Error()) continue } hremote, err := cli.Hash(r.GameID) if err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "error: failed to get the file hash from the remote:", err) continue } vlocal, err := repository.Version(r.GameID) if err != nil { + destroyPg() slog.Error(err.Error()) continue } remoteMetadata, err := cli.Metadata(r.GameID) if err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "error: failed to get the game metadata from the remote:", err) continue } + pg.Describe(fmt.Sprintf("[%s] Pulling backup...", g.Name)) + if err := pullBackup(g, cli); err != nil { + slog.Warn("failed to pull backup files", "err", err) + } + + pg.Describe(fmt.Sprintf("[%s] Pushing backup...", g.Name)) if err := pushBackup(g, cli); err != nil { slog.Warn("failed to push backup files", "err", err) } if hlocal == hremote { + destroyPg() if vlocal != remoteMetadata.Version { slog.Debug("version is not the same, but the hash is equal. Updating local database") if err := repository.SetVersion(r.GameID, remoteMetadata.Version); err != nil { @@ -116,29 +141,38 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } if vlocal > remoteMetadata.Version { + pg.Describe(fmt.Sprintf("[%s] Pushing data...", g.Name)) if err := push(g, cli); err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } + destroyPg() continue } if vlocal < remoteMetadata.Version { + destroyPg() if err := pull(r.GameID, cli); err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } if err := repository.SetVersion(r.GameID, remoteMetadata.Version); err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "error: failed to synchronize version number:", err) continue } if err := repository.SetDate(r.GameID, remoteMetadata.Date); err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "error: failed to synchronize date:", err) continue } continue } + destroyPg() + if vlocal == remoteMetadata.Version { if err := conflict(r.GameID, g, remoteMetadata, cli); err != nil { fmt.Fprintln(os.Stderr, "error: failed to resolve conflict:", err) @@ -148,6 +182,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } } + fmt.Println("done.") return subcommands.ExitSuccess } @@ -222,6 +257,39 @@ func pushBackup(m repository.Metadata, cli *client.Client) error { return nil } +func pullBackup(m repository.Metadata, cli *client.Client) error { + bs, err := cli.ListArchives(m.ID) + if err != nil { + return err + } + + for _, uuid := range bs { + rinfo, err := cli.ArchiveInfo(m.ID, uuid) + if err != nil { + return err + } + + linfo, err := repository.Archive(m.ID, uuid) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + } + + path := filepath.Join(repository.DatastorePath(), m.ID, "hist", uuid) + if err := os.MkdirAll(path, 0740); err != nil { + return err + } + + if rinfo.MD5 != linfo.MD5 { + if err := cli.PullBackup(m.ID, uuid, filepath.Join(path, "data.tar.gz")); err != nil { + return err + } + } + } + return nil +} + func pull(gameID string, cli *client.Client) error { archivePath := filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz") diff --git a/cmd/server/api/api.go b/cmd/server/api/api.go index 50121dd..a08a068 100644 --- a/cmd/server/api/api.go +++ b/cmd/server/api/api.go @@ -61,11 +61,14 @@ func NewServer(documentRoot string, creds map[string]string, port int) *HTTPServ // Data routes gamesRouter.Group(func(saveRouter chi.Router) { saveRouter.Post("/{id}/data", s.upload) - saveRouter.Post("/{id}/hist/{uuid}/data", s.histUpload) - saveRouter.Get("/{id}/hist/{uuid}/info", s.histExists) saveRouter.Get("/{id}/data", s.download) saveRouter.Get("/{id}/hash", s.hash) saveRouter.Get("/{id}/metadata", s.metadata) + + saveRouter.Get("/{id}/hist", s.allHist) + saveRouter.Post("/{id}/hist/{uuid}/data", s.histUpload) + saveRouter.Get("/{id}/hist/{uuid}/data", s.histDownload) + saveRouter.Get("/{id}/hist/{uuid}/info", s.histExists) }) }) }) @@ -211,6 +214,35 @@ func (s HTTPServer) upload(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) } +func (s HTTPServer) allHist(w http.ResponseWriter, r *http.Request) { + gameID := chi.URLParam(r, "id") + path := filepath.Join(s.documentRoot, "data", gameID, "hist") + datastore := make([]string, 0) + + if _, err := os.Stat(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + ok(datastore, w, r) + return + } + fmt.Fprintln(os.Stderr, "failed to open datastore (", s.documentRoot, "):", err) + internalServerError(w, r) + return + } + + ds, err := os.ReadDir(path) + if err != nil { + fmt.Fprintln(os.Stderr, "failed to open datastore (", s.documentRoot, "):", err) + internalServerError(w, r) + return + } + + for _, d := range ds { + datastore = append(datastore, d.Name()) + } + + ok(datastore, w, r) +} + func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) { const ( sizeLimit int64 = 500 << 20 // 500 MB @@ -249,6 +281,48 @@ func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) } +func (s HTTPServer) histDownload(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + uuid := chi.URLParam(r, "uuid") + path := filepath.Clean(filepath.Join(s.documentRoot, "data", id, "hist", uuid)) + + sdir, err := os.Stat(path) + if err != nil { + notFound("id not found", w, r) + return + } + + if !sdir.IsDir() { + notFound("id not found", w, r) + return + } + + path = filepath.Join(path, "data.tar.gz") + + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + notFound("id not found", w, r) + return + } + defer f.Close() + + // Get file info to set headers + fi, err := f.Stat() + if err != nil || fi.IsDir() { + internalServerError(w, r) + return + } + + // Set headers + w.Header().Set("Content-Disposition", "attachment; filename=\"data.tar.gz\"") + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) + w.WriteHeader(200) + + // Stream the file content + http.ServeContent(w, r, "data.tar.gz", fi.ModTime(), f) +} + func (s HTTPServer) histExists(w http.ResponseWriter, r *http.Request) { gameID := chi.URLParam(r, "id") uuid := chi.URLParam(r, "uuid") diff --git a/pkg/remote/client/client.go b/pkg/remote/client/client.go index 72df438..76633c2 100644 --- a/pkg/remote/client/client.go +++ b/pkg/remote/client/client.go @@ -163,6 +163,28 @@ func (c *Client) PushBackup(archiveMetadata repository.Backup, m repository.Meta return c.push(u, archiveMetadata.ArchivePath, m) } +func (c *Client) ListArchives(gameID string) ([]string, error) { + u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "hist") + if err != nil { + return nil, err + } + + o, err := c.get(u) + if err != nil { + return nil, err + } + + if m, ok := (o.Data).([]any); ok { + var res []string + for _, uuid := range m { + res = append(res, uuid.(string)) + } + return res, nil + } + + return nil, errors.New("invalid payload sent by the server") +} + func (c *Client) ArchiveInfo(gameID, uuid string) (repository.Backup, error) { u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "hist", uuid, "info") if err != nil { @@ -234,6 +256,54 @@ func (c *Client) Pull(gameID, archivePath string) error { return nil } +func (c *Client) PullBackup(gameID, uuid, archivePath string) error { + u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "hist", uuid, "data") + if err != nil { + return err + } + + cli := http.Client{} + + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return err + } + + req.SetBasicAuth(c.username, c.password) + + f, err := os.OpenFile(archivePath+".part", os.O_CREATE|os.O_WRONLY, 0740) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer f.Close() + + res, err := cli.Do(req) + if err != nil { + return fmt.Errorf("cannot connect to remote: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("cannot connect to remote: server return code: %s", res.Status) + } + + bar := progressbar.DefaultBytes( + res.ContentLength, + "Pulling...", + ) + defer bar.Close() + + if _, err := io.Copy(io.MultiWriter(f, bar), res.Body); err != nil { + return fmt.Errorf("an error occured while copying the file from the remote: %w", err) + } + + if err := os.Rename(archivePath+".part", archivePath); err != nil { + return fmt.Errorf("failed to move temporary data: %w", err) + } + + return nil +} + func (c *Client) Ping() error { cli := http.Client{} diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index 458f5f1..e69290b 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -144,7 +144,7 @@ func One(gameID string) (Metadata, error) { return m, nil } -func Archive(gameID string) error { +func MakeArchive(gameID string) error { path := filepath.Join(datastorepath, gameID, "data.tar.gz") // open old @@ -177,6 +177,66 @@ func Archive(gameID string) error { return nil } +func RestoreArchive(gameID, uuid string) error { + histDirPath := filepath.Join(datastorepath, gameID, "hist", uuid) + if err := os.MkdirAll(histDirPath, 0740); err != nil { + return fmt.Errorf("failed to make directory: %w", err) + } + + // open old + nf, err := os.OpenFile(filepath.Join(histDirPath, "data.tar.gz"), os.O_RDONLY, 0) + if err != nil { + return fmt.Errorf("failed to open new file: %w", err) + } + defer nf.Close() + + path := filepath.Join(datastorepath, gameID, "data.tar.gz") + + // open new + f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("failed to open old file: %w", err) + } + defer f.Close() + + // copy + if _, err := io.Copy(f, nf); err != nil { + return fmt.Errorf("failed to copy data: %w", err) + } + + return nil +} + +func Archive(gameID, uuid string) (Backup, error) { + histDirPath := filepath.Join(datastorepath, gameID, "hist", uuid) + if err := os.MkdirAll(histDirPath, 0740); err != nil { + return Backup{}, fmt.Errorf("failed to make 'hist' directory") + } + + finfo, err := os.Stat(histDirPath) + if err != nil { + return Backup{}, fmt.Errorf("corrupted datastore: %w", err) + } + archivePath := filepath.Join(histDirPath, "data.tar.gz") + + h, err := hash.FileMD5(archivePath) + if err != nil { + return Backup{}, fmt.Errorf("failed to calculate md5 hash: %w", err) + } + + b := Backup{ + CreatedAt: finfo.ModTime(), + UUID: filepath.Base(finfo.Name()), + MD5: h, + ArchivePath: archivePath, + } + + return b, nil +} + func Archives(gameID string) ([]Backup, error) { histDirPath := filepath.Join(datastorepath, gameID, "hist") if err := os.MkdirAll(histDirPath, 0740); err != nil { From 95857356abb9ec6e9b504816e562d6a02d418e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Thu, 31 Jul 2025 15:44:40 +0200 Subject: [PATCH 4/5] fix build script --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 5eea3c2..386ea6c 100755 --- a/build.sh +++ b/build.sh @@ -51,7 +51,7 @@ for platform in "${platforms[@]}"; do tar -czf build/server_${platform_split[0]}_${platform_split[1]}.tar.gz build/cloudsave_server$EXT rm build/cloudsave_server$EXT else - CGO_ENABLED=0 GOOS=${platform_split[0]} GOARCH=${platform_split[1]} go build -o build/cloudsave_server_${platform_split[0]}_${platform_split[1]}$EXT -a ./cmd/cli + CGO_ENABLED=0 GOOS=${platform_split[0]} GOARCH=${platform_split[1]} go build -o build/cloudsave_server_${platform_split[0]}_${platform_split[1]}$EXT -a ./cmd/server fi done From f2fee0990b69b1b0c47d18e8f0512f4f6631ab15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Thu, 31 Jul 2025 21:23:44 +0200 Subject: [PATCH 5/5] fix rel path error --- cmd/cli/commands/run/run.go | 2 +- pkg/tools/archive/archive.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/cli/commands/run/run.go b/cmd/cli/commands/run/run.go index 5211bf8..45335ab 100644 --- a/cmd/cli/commands/run/run.go +++ b/cmd/cli/commands/run/run.go @@ -124,7 +124,7 @@ func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error { defer f.Close() if err := archive.Tar(f, srcDir); err != nil { - return fmt.Errorf("failed archiving files") + return fmt.Errorf("failed archiving files: %w", err) } now := time.Now().UTC().Format(time.RFC3339) diff --git a/pkg/tools/archive/archive.go b/pkg/tools/archive/archive.go index b3a6b7d..0ebef41 100644 --- a/pkg/tools/archive/archive.go +++ b/pkg/tools/archive/archive.go @@ -83,23 +83,23 @@ func Tar(file io.Writer, root string) error { // Walk again to add files err := filepath.Walk(root, func(path string, info os.FileInfo, walkErr error) error { if walkErr != nil { - return walkErr + return fmt.Errorf("failed to walk through the directory: %w", walkErr) } - path, err := filepath.Rel(root, path) + relpath, err := filepath.Rel(root, path) if err != nil { - return err + return fmt.Errorf("failed to make relative path: %w", err) } // Create tar header header, err := tar.FileInfoHeader(info, path) if err != nil { - return err + return fmt.Errorf("failed to make file info header: %w", err) } - header.Name = path + header.Name = relpath if err := tw.WriteHeader(header); err != nil { - return err + return fmt.Errorf("failed to write header: %w", err) } if !info.Mode().IsRegular() { @@ -108,11 +108,11 @@ func Tar(file io.Writer, root string) error { file, err := os.Open(path) if err != nil { - return err + return fmt.Errorf("failed to open file: %w", err) } defer file.Close() if _, err := io.Copy(tw, file); err != nil { - return err + return fmt.Errorf("failed to copy file: %w", err) } return nil })