From 58c6bc56cfea191d58193384da3b49ad854f5d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Wed, 30 Jul 2025 17:20:43 +0200 Subject: [PATCH] apply backup --- cmd/cli/commands/apply/apply.go | 69 +++++++++++++++++++++++++++++ cmd/cli/commands/list/list.go | 40 ++++++++++++++--- cmd/cli/commands/run/run.go | 12 ++++- cmd/cli/commands/sync/sync.go | 70 ++++++++++++++++++++++++++++- cmd/server/api/api.go | 78 ++++++++++++++++++++++++++++++++- pkg/remote/client/client.go | 70 +++++++++++++++++++++++++++++ pkg/repository/repository.go | 62 +++++++++++++++++++++++++- 7 files changed, 390 insertions(+), 11 deletions(-) create mode 100644 cmd/cli/commands/apply/apply.go diff --git a/cmd/cli/commands/apply/apply.go b/cmd/cli/commands/apply/apply.go new file mode 100644 index 0000000..0e3e389 --- /dev/null +++ b/cmd/cli/commands/apply/apply.go @@ -0,0 +1,69 @@ +package apply + +import ( + "cloudsave/pkg/repository" + "cloudsave/pkg/tools/archive" + "context" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/google/subcommands" +) + +type ( + ListCmd struct { + } +) + +func (*ListCmd) Name() string { return "apply" } +func (*ListCmd) Synopsis() string { return "apply a backup" } +func (*ListCmd) Usage() string { + return `apply: + Apply a backup +` +} + +func (p *ListCmd) SetFlags(f *flag.FlagSet) { +} + +func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { + if f.NArg() != 2 { + fmt.Fprintln(os.Stderr, "error: missing game ID and/or backup uuid") + return subcommands.ExitUsageError + } + + gameID := f.Arg(0) + uuid := f.Arg(1) + + g, err := repository.One(gameID) + if err != nil { + fmt.Fprintf(os.Stderr, "error: failed to open game metadata: %s\n", err) + return subcommands.ExitFailure + } + + if err := repository.RestoreArchive(gameID, uuid); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to restore backup: %s\n", err) + return subcommands.ExitFailure + } + + if err := os.RemoveAll(g.Path); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to remove old data: %s\n", err) + return subcommands.ExitFailure + } + + file, err := os.OpenFile(filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz"), os.O_RDONLY, 0) + if err != nil { + fmt.Fprintf(os.Stderr, "error: failed to open archive: %s\n", err) + return subcommands.ExitFailure + } + defer file.Close() + + if err := archive.Untar(file, g.Path); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to extract archive: %s\n", err) + return subcommands.ExitFailure + } + + return subcommands.ExitSuccess +} diff --git a/cmd/cli/commands/list/list.go b/cmd/cli/commands/list/list.go index 805c44c..8987ae3 100644 --- a/cmd/cli/commands/list/list.go +++ b/cmd/cli/commands/list/list.go @@ -1,9 +1,9 @@ package list import ( + "cloudsave/cmd/cli/tools/prompt/credentials" "cloudsave/pkg/remote/client" "cloudsave/pkg/repository" - "cloudsave/cmd/cli/tools/prompt/credentials" "context" "flag" "fmt" @@ -15,6 +15,7 @@ import ( type ( ListCmd struct { remote bool + backup bool } ) @@ -28,6 +29,7 @@ func (*ListCmd) Usage() string { func (p *ListCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.remote, "a", false, "list all including remote data") + f.BoolVar(&p.backup, "include-backup", false, "include backup uuids in the output") } func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { @@ -43,20 +45,20 @@ func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitFailure } - if err := remote(f.Arg(0), username, password); err != nil { + if err := remote(f.Arg(0), username, password, p.backup); err != nil { fmt.Fprintln(os.Stderr, "error:", err) return subcommands.ExitFailure } return subcommands.ExitSuccess } - if err := local(); err != nil { + if err := local(p.backup); err != nil { fmt.Fprintln(os.Stderr, "error:", err) return subcommands.ExitFailure } return subcommands.ExitSuccess } -func local() error { +func local(includeBackup bool) error { games, err := repository.All() if err != nil { return fmt.Errorf("failed to load datastore: %w", err) @@ -66,13 +68,25 @@ func local() error { fmt.Println("ID:", g.ID) fmt.Println("Name:", g.Name) fmt.Println("Last Version:", g.Date, "( Version Number", g.Version, ")") + if includeBackup { + bk, err := repository.Archives(g.ID) + if err != nil { + return fmt.Errorf("failed to list backup files: %w", err) + } + if len(bk) > 0 { + fmt.Println("Backup:") + for _, b := range bk { + fmt.Printf(" - %s (%s)\n", b.UUID, b.CreatedAt) + } + } + } fmt.Println("---") } return nil } -func remote(url, username, password string) error { +func remote(url, username, password string, includeBackup bool) error { cli := client.New(url, username, password) if err := cli.Ping(); err != nil { @@ -91,6 +105,22 @@ func remote(url, username, password string) error { fmt.Println("ID:", g.ID) fmt.Println("Name:", g.Name) fmt.Println("Last Version:", g.Date, "( Version Number", g.Version, ")") + if includeBackup { + bk, err := cli.ListArchives(g.ID) + if err != nil { + return fmt.Errorf("failed to list backup files: %w", err) + } + if len(bk) > 0 { + fmt.Println("Backup:") + for _, uuid := range bk { + b, err := cli.ArchiveInfo(g.ID, uuid) + if err != nil { + return fmt.Errorf("failed to list backup files: %w", err) + } + fmt.Printf(" - %s (%s)\n", b.UUID, b.CreatedAt) + } + } + } fmt.Println("---") } diff --git a/cmd/cli/commands/run/run.go b/cmd/cli/commands/run/run.go index 43bd28e..5211bf8 100644 --- a/cmd/cli/commands/run/run.go +++ b/cmd/cli/commands/run/run.go @@ -53,8 +53,10 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s fmt.Fprintf(os.Stderr, "error: cannot process the data of %s: %s\n", metadata.ID, err) return subcommands.ExitFailure } + fmt.Println("✅", metadata.Name) } + fmt.Println("done.") return subcommands.ExitSuccess } @@ -63,7 +65,13 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s // After archiving, it updates stateFile to the current time. func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error { pg := progressbar.New(-1) - defer pg.Close() + destroyPg := func() { + pg.Finish() + pg.Clear() + pg.Close() + + } + defer destroyPg() pg.Describe("Scanning " + gameID + "...") @@ -103,7 +111,7 @@ func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error { // make a backup pg.Describe("Backup current data...") - if err := repository.Archive(gameID); err != nil { + if err := repository.MakeArchive(gameID); err != nil { return fmt.Errorf("failed to archive data: %w", err) } diff --git a/cmd/cli/commands/sync/sync.go b/cmd/cli/commands/sync/sync.go index a73ce81..9fe9e5d 100644 --- a/cmd/cli/commands/sync/sync.go +++ b/cmd/cli/commands/sync/sync.go @@ -16,6 +16,7 @@ import ( "time" "github.com/google/subcommands" + "github.com/schollz/progressbar/v3" ) type ( @@ -51,13 +52,21 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) fmt.Fprintln(os.Stderr, "error: failed to load datastore:", err) return subcommands.ExitFailure } - cli, err := connect(remoteCred, r) if err != nil { fmt.Fprintln(os.Stderr, "error: failed to connect to the remote:", err) return subcommands.ExitFailure } + pg := progressbar.New(-1) + destroyPg := func() { + pg.Finish() + pg.Clear() + pg.Close() + + } + + pg.Describe(fmt.Sprintf("[%s] Checking status...", g.Name)) exists, err := cli.Exists(r.GameID) if err != nil { slog.Error(err.Error()) @@ -65,45 +74,61 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } if !exists { + pg.Describe(fmt.Sprintf("[%s] Pushing data...", g.Name)) if err := push(g, cli); err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } + pg.Describe(fmt.Sprintf("[%s] Pushing backup...", g.Name)) if err := pushBackup(g, cli); err != nil { + destroyPg() slog.Warn("failed to push backup files", "err", err) } continue } + pg.Describe(fmt.Sprintf("[%s] Fetching metadata...", g.Name)) hlocal, err := repository.Hash(r.GameID) if err != nil { + destroyPg() slog.Error(err.Error()) continue } hremote, err := cli.Hash(r.GameID) if err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "error: failed to get the file hash from the remote:", err) continue } vlocal, err := repository.Version(r.GameID) if err != nil { + destroyPg() slog.Error(err.Error()) continue } remoteMetadata, err := cli.Metadata(r.GameID) if err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "error: failed to get the game metadata from the remote:", err) continue } + pg.Describe(fmt.Sprintf("[%s] Pulling backup...", g.Name)) + if err := pullBackup(g, cli); err != nil { + slog.Warn("failed to pull backup files", "err", err) + } + + pg.Describe(fmt.Sprintf("[%s] Pushing backup...", g.Name)) if err := pushBackup(g, cli); err != nil { slog.Warn("failed to push backup files", "err", err) } if hlocal == hremote { + destroyPg() if vlocal != remoteMetadata.Version { slog.Debug("version is not the same, but the hash is equal. Updating local database") if err := repository.SetVersion(r.GameID, remoteMetadata.Version); err != nil { @@ -116,29 +141,38 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } if vlocal > remoteMetadata.Version { + pg.Describe(fmt.Sprintf("[%s] Pushing data...", g.Name)) if err := push(g, cli); err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } + destroyPg() continue } if vlocal < remoteMetadata.Version { + destroyPg() if err := pull(r.GameID, cli); err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } if err := repository.SetVersion(r.GameID, remoteMetadata.Version); err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "error: failed to synchronize version number:", err) continue } if err := repository.SetDate(r.GameID, remoteMetadata.Date); err != nil { + destroyPg() fmt.Fprintln(os.Stderr, "error: failed to synchronize date:", err) continue } continue } + destroyPg() + if vlocal == remoteMetadata.Version { if err := conflict(r.GameID, g, remoteMetadata, cli); err != nil { fmt.Fprintln(os.Stderr, "error: failed to resolve conflict:", err) @@ -148,6 +182,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } } + fmt.Println("done.") return subcommands.ExitSuccess } @@ -222,6 +257,39 @@ func pushBackup(m repository.Metadata, cli *client.Client) error { return nil } +func pullBackup(m repository.Metadata, cli *client.Client) error { + bs, err := cli.ListArchives(m.ID) + if err != nil { + return err + } + + for _, uuid := range bs { + rinfo, err := cli.ArchiveInfo(m.ID, uuid) + if err != nil { + return err + } + + linfo, err := repository.Archive(m.ID, uuid) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + } + + path := filepath.Join(repository.DatastorePath(), m.ID, "hist", uuid) + if err := os.MkdirAll(path, 0740); err != nil { + return err + } + + if rinfo.MD5 != linfo.MD5 { + if err := cli.PullBackup(m.ID, uuid, filepath.Join(path, "data.tar.gz")); err != nil { + return err + } + } + } + return nil +} + func pull(gameID string, cli *client.Client) error { archivePath := filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz") diff --git a/cmd/server/api/api.go b/cmd/server/api/api.go index 50121dd..a08a068 100644 --- a/cmd/server/api/api.go +++ b/cmd/server/api/api.go @@ -61,11 +61,14 @@ func NewServer(documentRoot string, creds map[string]string, port int) *HTTPServ // Data routes gamesRouter.Group(func(saveRouter chi.Router) { saveRouter.Post("/{id}/data", s.upload) - saveRouter.Post("/{id}/hist/{uuid}/data", s.histUpload) - saveRouter.Get("/{id}/hist/{uuid}/info", s.histExists) saveRouter.Get("/{id}/data", s.download) saveRouter.Get("/{id}/hash", s.hash) saveRouter.Get("/{id}/metadata", s.metadata) + + saveRouter.Get("/{id}/hist", s.allHist) + saveRouter.Post("/{id}/hist/{uuid}/data", s.histUpload) + saveRouter.Get("/{id}/hist/{uuid}/data", s.histDownload) + saveRouter.Get("/{id}/hist/{uuid}/info", s.histExists) }) }) }) @@ -211,6 +214,35 @@ func (s HTTPServer) upload(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) } +func (s HTTPServer) allHist(w http.ResponseWriter, r *http.Request) { + gameID := chi.URLParam(r, "id") + path := filepath.Join(s.documentRoot, "data", gameID, "hist") + datastore := make([]string, 0) + + if _, err := os.Stat(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + ok(datastore, w, r) + return + } + fmt.Fprintln(os.Stderr, "failed to open datastore (", s.documentRoot, "):", err) + internalServerError(w, r) + return + } + + ds, err := os.ReadDir(path) + if err != nil { + fmt.Fprintln(os.Stderr, "failed to open datastore (", s.documentRoot, "):", err) + internalServerError(w, r) + return + } + + for _, d := range ds { + datastore = append(datastore, d.Name()) + } + + ok(datastore, w, r) +} + func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) { const ( sizeLimit int64 = 500 << 20 // 500 MB @@ -249,6 +281,48 @@ func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) } +func (s HTTPServer) histDownload(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + uuid := chi.URLParam(r, "uuid") + path := filepath.Clean(filepath.Join(s.documentRoot, "data", id, "hist", uuid)) + + sdir, err := os.Stat(path) + if err != nil { + notFound("id not found", w, r) + return + } + + if !sdir.IsDir() { + notFound("id not found", w, r) + return + } + + path = filepath.Join(path, "data.tar.gz") + + f, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + notFound("id not found", w, r) + return + } + defer f.Close() + + // Get file info to set headers + fi, err := f.Stat() + if err != nil || fi.IsDir() { + internalServerError(w, r) + return + } + + // Set headers + w.Header().Set("Content-Disposition", "attachment; filename=\"data.tar.gz\"") + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) + w.WriteHeader(200) + + // Stream the file content + http.ServeContent(w, r, "data.tar.gz", fi.ModTime(), f) +} + func (s HTTPServer) histExists(w http.ResponseWriter, r *http.Request) { gameID := chi.URLParam(r, "id") uuid := chi.URLParam(r, "uuid") diff --git a/pkg/remote/client/client.go b/pkg/remote/client/client.go index 72df438..76633c2 100644 --- a/pkg/remote/client/client.go +++ b/pkg/remote/client/client.go @@ -163,6 +163,28 @@ func (c *Client) PushBackup(archiveMetadata repository.Backup, m repository.Meta return c.push(u, archiveMetadata.ArchivePath, m) } +func (c *Client) ListArchives(gameID string) ([]string, error) { + u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "hist") + if err != nil { + return nil, err + } + + o, err := c.get(u) + if err != nil { + return nil, err + } + + if m, ok := (o.Data).([]any); ok { + var res []string + for _, uuid := range m { + res = append(res, uuid.(string)) + } + return res, nil + } + + return nil, errors.New("invalid payload sent by the server") +} + func (c *Client) ArchiveInfo(gameID, uuid string) (repository.Backup, error) { u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "hist", uuid, "info") if err != nil { @@ -234,6 +256,54 @@ func (c *Client) Pull(gameID, archivePath string) error { return nil } +func (c *Client) PullBackup(gameID, uuid, archivePath string) error { + u, err := url.JoinPath(c.baseURL, "api", "v1", "games", gameID, "hist", uuid, "data") + if err != nil { + return err + } + + cli := http.Client{} + + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return err + } + + req.SetBasicAuth(c.username, c.password) + + f, err := os.OpenFile(archivePath+".part", os.O_CREATE|os.O_WRONLY, 0740) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer f.Close() + + res, err := cli.Do(req) + if err != nil { + return fmt.Errorf("cannot connect to remote: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("cannot connect to remote: server return code: %s", res.Status) + } + + bar := progressbar.DefaultBytes( + res.ContentLength, + "Pulling...", + ) + defer bar.Close() + + if _, err := io.Copy(io.MultiWriter(f, bar), res.Body); err != nil { + return fmt.Errorf("an error occured while copying the file from the remote: %w", err) + } + + if err := os.Rename(archivePath+".part", archivePath); err != nil { + return fmt.Errorf("failed to move temporary data: %w", err) + } + + return nil +} + func (c *Client) Ping() error { cli := http.Client{} diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index 458f5f1..e69290b 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -144,7 +144,7 @@ func One(gameID string) (Metadata, error) { return m, nil } -func Archive(gameID string) error { +func MakeArchive(gameID string) error { path := filepath.Join(datastorepath, gameID, "data.tar.gz") // open old @@ -177,6 +177,66 @@ func Archive(gameID string) error { return nil } +func RestoreArchive(gameID, uuid string) error { + histDirPath := filepath.Join(datastorepath, gameID, "hist", uuid) + if err := os.MkdirAll(histDirPath, 0740); err != nil { + return fmt.Errorf("failed to make directory: %w", err) + } + + // open old + nf, err := os.OpenFile(filepath.Join(histDirPath, "data.tar.gz"), os.O_RDONLY, 0) + if err != nil { + return fmt.Errorf("failed to open new file: %w", err) + } + defer nf.Close() + + path := filepath.Join(datastorepath, gameID, "data.tar.gz") + + // open new + f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("failed to open old file: %w", err) + } + defer f.Close() + + // copy + if _, err := io.Copy(f, nf); err != nil { + return fmt.Errorf("failed to copy data: %w", err) + } + + return nil +} + +func Archive(gameID, uuid string) (Backup, error) { + histDirPath := filepath.Join(datastorepath, gameID, "hist", uuid) + if err := os.MkdirAll(histDirPath, 0740); err != nil { + return Backup{}, fmt.Errorf("failed to make 'hist' directory") + } + + finfo, err := os.Stat(histDirPath) + if err != nil { + return Backup{}, fmt.Errorf("corrupted datastore: %w", err) + } + archivePath := filepath.Join(histDirPath, "data.tar.gz") + + h, err := hash.FileMD5(archivePath) + if err != nil { + return Backup{}, fmt.Errorf("failed to calculate md5 hash: %w", err) + } + + b := Backup{ + CreatedAt: finfo.ModTime(), + UUID: filepath.Base(finfo.Name()), + MD5: h, + ArchivePath: archivePath, + } + + return b, nil +} + func Archives(gameID string) ([]Backup, error) { histDirPath := filepath.Join(datastorepath, gameID, "hist") if err := os.MkdirAll(histDirPath, 0740); err != nil {