From 810c5ac627e087742758617021dc49be91943dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Sat, 9 Aug 2025 22:19:57 +0200 Subject: [PATCH 1/7] wip --- cmd/cli/commands/add/add.go | 20 +- cmd/cli/commands/apply/apply.go | 35 +- cmd/cli/commands/list/list.go | 21 +- cmd/cli/commands/pull/pull.go | 43 +-- cmd/cli/commands/remote/remote.go | 13 +- cmd/cli/commands/remove/remove.go | 8 +- cmd/cli/commands/run/run.go | 100 +----- cmd/cli/commands/sync/sync.go | 96 ++---- cmd/cli/main.go | 35 +- cmd/server/api/api.go | 1 - cmd/server/data/data.go | 243 -------------- pkg/data/data.go | 253 ++++++++++++++ pkg/repository/repository.go | 527 +++++++++++++----------------- 13 files changed, 605 insertions(+), 790 deletions(-) delete mode 100644 cmd/server/data/data.go create mode 100644 pkg/data/data.go diff --git a/cmd/cli/commands/add/add.go b/cmd/cli/commands/add/add.go index e4d9974..ba466cd 100644 --- a/cmd/cli/commands/add/add.go +++ b/cmd/cli/commands/add/add.go @@ -1,22 +1,21 @@ package add import ( - "cloudsave/pkg/remote" - "cloudsave/pkg/repository" + "cloudsave/pkg/data" "context" "flag" "fmt" "os" "path/filepath" - "strings" "github.com/google/subcommands" ) type ( AddCmd struct { - name string - remote string + Service *data.Service + name string + remote string } ) @@ -51,17 +50,16 @@ func (p *AddCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s p.name = filepath.Base(path) } - m, err := repository.Add(p.name, path) + gameID, err := p.Service.Add(p.name, path, p.remote) if err != nil { - fmt.Fprintln(os.Stderr, "error: failed to add game reference:", err) + fmt.Fprintln(os.Stderr, "error: failed to add this gamesave to the datastore:", err) return subcommands.ExitFailure } - if len(strings.TrimSpace(p.remote)) > 0 { - remote.Set(m.ID, p.remote) + if err := p.Service.Scan(gameID); err != nil { + fmt.Fprintln(os.Stderr, "error: failed to scan:", err) + return subcommands.ExitFailure } - fmt.Println(m.ID) - return subcommands.ExitSuccess } diff --git a/cmd/cli/commands/apply/apply.go b/cmd/cli/commands/apply/apply.go index b6610d0..041b805 100644 --- a/cmd/cli/commands/apply/apply.go +++ b/cmd/cli/commands/apply/apply.go @@ -1,13 +1,10 @@ package apply import ( - "cloudsave/pkg/repository" - "cloudsave/pkg/tools/archive" "context" "flag" "fmt" "os" - "path/filepath" "github.com/google/subcommands" ) @@ -35,36 +32,10 @@ func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitUsageError } - gameID := f.Arg(0) - uuid := f.Arg(1) + //gameID := f.Arg(0) + //uuid := f.Arg(1) - g, err := repository.One(gameID) - if err != nil { - fmt.Fprintf(os.Stderr, "error: failed to open game metadata: %s\n", err) - return subcommands.ExitFailure - } - - if err := repository.RestoreArchive(gameID, uuid); err != nil { - fmt.Fprintf(os.Stderr, "error: failed to restore backup: %s\n", err) - return subcommands.ExitFailure - } - - if err := os.RemoveAll(g.Path); err != nil { - fmt.Fprintf(os.Stderr, "error: failed to remove old data: %s\n", err) - return subcommands.ExitFailure - } - - file, err := os.OpenFile(filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz"), os.O_RDONLY, 0) - if err != nil { - fmt.Fprintf(os.Stderr, "error: failed to open archive: %s\n", err) - return subcommands.ExitFailure - } - defer file.Close() - - if err := archive.Untar(file, g.Path); err != nil { - fmt.Fprintf(os.Stderr, "error: failed to extract archive: %s\n", err) - return subcommands.ExitFailure - } + panic("not implemented") return subcommands.ExitSuccess } diff --git a/cmd/cli/commands/list/list.go b/cmd/cli/commands/list/list.go index 3b80f24..330127e 100644 --- a/cmd/cli/commands/list/list.go +++ b/cmd/cli/commands/list/list.go @@ -2,8 +2,8 @@ package list import ( "cloudsave/cmd/cli/tools/prompt/credentials" + "cloudsave/pkg/data" "cloudsave/pkg/remote/client" - "cloudsave/pkg/repository" "context" "flag" "fmt" @@ -14,8 +14,9 @@ import ( type ( ListCmd struct { - remote bool - backup bool + Service *data.Service + remote bool + backup bool } ) @@ -44,25 +45,25 @@ func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) username, password, err := credentials.Read() if err != nil { - fmt.Fprintf(os.Stderr, "failed to read std output: %s", err) + fmt.Fprintf(os.Stderr, "error: failed to read std output: %s", err) return subcommands.ExitFailure } - if err := remote(f.Arg(0), username, password, p.backup); err != nil { + if err := p.server(f.Arg(0), username, password, p.backup); err != nil { fmt.Fprintln(os.Stderr, "error:", err) return subcommands.ExitFailure } return subcommands.ExitSuccess } - if err := local(p.backup); err != nil { + if err := p.local(p.backup); err != nil { fmt.Fprintln(os.Stderr, "error:", err) return subcommands.ExitFailure } return subcommands.ExitSuccess } -func local(includeBackup bool) error { - games, err := repository.All() +func (p *ListCmd) local(includeBackup bool) error { + games, err := p.Service.AllGames() if err != nil { return fmt.Errorf("failed to load datastore: %w", err) } @@ -72,7 +73,7 @@ func local(includeBackup bool) error { fmt.Println("Name:", g.Name) fmt.Println("Last Version:", g.Date, "( Version Number", g.Version, ")") if includeBackup { - bk, err := repository.Archives(g.ID) + bk, err := p.Service.AllBackups(g.ID) if err != nil { return fmt.Errorf("failed to list backup files: %w", err) } @@ -89,7 +90,7 @@ func local(includeBackup bool) error { return nil } -func remote(url, username, password string, includeBackup bool) error { +func (p *ListCmd) server(url, username, password string, includeBackup bool) error { cli := client.New(url, username, password) if err := cli.Ping(); err != nil { diff --git a/cmd/cli/commands/pull/pull.go b/cmd/cli/commands/pull/pull.go index 4490d1c..0547bb6 100644 --- a/cmd/cli/commands/pull/pull.go +++ b/cmd/cli/commands/pull/pull.go @@ -2,20 +2,19 @@ package pull import ( "cloudsave/cmd/cli/tools/prompt/credentials" + "cloudsave/pkg/data" "cloudsave/pkg/remote/client" - "cloudsave/pkg/repository" - "cloudsave/pkg/tools/archive" "context" "flag" "fmt" "os" - "path/filepath" "github.com/google/subcommands" ) type ( PullCmd struct { + Service *data.Service } ) @@ -44,45 +43,33 @@ func (p *PullCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) username, password, err := credentials.Read() if err != nil { - fmt.Fprintf(os.Stderr, "failed to read std output: %s", err) + fmt.Fprintf(os.Stderr, "error: failed to read std output: %s", err) return subcommands.ExitFailure } cli := client.New(url, username, password) if err := cli.Ping(); err != nil { - fmt.Fprintf(os.Stderr, "failed to connect to the remote: %s", err) + fmt.Fprintf(os.Stderr, "error: failed to connect to the remote: %s", err) return subcommands.ExitFailure } - archivePath := filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz") + if err := p.Service.PullCurrent(gameID, path, cli); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to pull current archive: %s", err) + return subcommands.ExitFailure + } - m, err := cli.Metadata(gameID) + ids, err := cli.ListArchives(gameID) if err != nil { - fmt.Fprintf(os.Stderr, "failed to get metadata: %s", err) + fmt.Fprintf(os.Stderr, "error: failed to list backup archive: %s", err) return subcommands.ExitFailure } - err = repository.Register(m, path) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to register local metadata: %s", err) - return subcommands.ExitFailure - } - - if err := cli.Pull(gameID, archivePath); err != nil { - fmt.Fprintf(os.Stderr, "failed to pull from the remote: %s", err) - return subcommands.ExitFailure - } - - fi, err := os.OpenFile(archivePath, os.O_RDONLY, 0) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to open archive: %s", err) - return subcommands.ExitFailure - } - - if err := archive.Untar(fi, path); err != nil { - fmt.Fprintf(os.Stderr, "failed to unarchive file: %s", err) - return subcommands.ExitFailure + for _, id := range ids { + if err := p.Service.PullBackup(gameID, id, cli); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to pull backup archive %s: %s", id, err) + return subcommands.ExitFailure + } } return subcommands.ExitSuccess diff --git a/cmd/cli/commands/remote/remote.go b/cmd/cli/commands/remote/remote.go index d35a1c5..8bc8186 100644 --- a/cmd/cli/commands/remote/remote.go +++ b/cmd/cli/commands/remote/remote.go @@ -1,9 +1,9 @@ package remote import ( + "cloudsave/pkg/data" "cloudsave/pkg/remote" "cloudsave/pkg/remote/client" - "cloudsave/pkg/repository" "context" "flag" "fmt" @@ -14,8 +14,9 @@ import ( type ( RemoteCmd struct { - set bool - list bool + Service *data.Service + set bool + list bool } ) @@ -43,7 +44,7 @@ func (p *RemoteCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface switch { case p.list: { - if err := list(); err != nil { + if err := p.print(); err != nil { fmt.Fprintln(os.Stderr, "error:", err) return subcommands.ExitFailure } @@ -68,8 +69,8 @@ func (p *RemoteCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface return subcommands.ExitSuccess } -func list() error { - games, err := repository.All() +func (p *RemoteCmd) print() error { + games, err := p.Service.AllGames() if err != nil { return fmt.Errorf("failed to load datastore: %w", err) } diff --git a/cmd/cli/commands/remove/remove.go b/cmd/cli/commands/remove/remove.go index bdfc1e1..1b424c1 100644 --- a/cmd/cli/commands/remove/remove.go +++ b/cmd/cli/commands/remove/remove.go @@ -1,7 +1,7 @@ package remove import ( - "cloudsave/pkg/repository" + "cloudsave/pkg/data" "context" "flag" "fmt" @@ -11,7 +11,9 @@ import ( ) type ( - RemoveCmd struct{} + RemoveCmd struct { + Service *data.Service + } ) func (*RemoveCmd) Name() string { return "remove" } @@ -33,7 +35,7 @@ func (p *RemoveCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} return subcommands.ExitUsageError } - err := repository.Remove(f.Arg(0)) + err := p.Service.RemoveGame(f.Arg(0)) if err != nil { fmt.Fprintln(os.Stderr, "error: failed to unregister the game:", err) return subcommands.ExitFailure diff --git a/cmd/cli/commands/run/run.go b/cmd/cli/commands/run/run.go index 8c260fb..e381a4f 100644 --- a/cmd/cli/commands/run/run.go +++ b/cmd/cli/commands/run/run.go @@ -1,22 +1,18 @@ package run import ( - "cloudsave/pkg/repository" - "cloudsave/pkg/tools/archive" + "cloudsave/pkg/data" "context" "flag" "fmt" - "io" "os" - "path/filepath" - "time" "github.com/google/subcommands" - "github.com/schollz/progressbar/v3" ) type ( RunCmd struct { + Service *data.Service } ) @@ -34,26 +30,15 @@ and a new archive is created with a new version number. func (p *RunCmd) SetFlags(f *flag.FlagSet) {} func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { - datastore, err := repository.All() + datastore, err := p.Service.AllGames() if err != nil { fmt.Fprintln(os.Stderr, "error: failed to load datastore:", err) return subcommands.ExitFailure } for _, metadata := range datastore { - metadataPath := filepath.Join(repository.DatastorePath(), metadata.ID) - //todo transaction - err := archiveIfChanged(metadata.ID, metadata.Path, filepath.Join(metadataPath, "data.tar.gz"), filepath.Join(metadataPath, ".last_run")) - if err != nil { - fmt.Fprintf(os.Stderr, "error: cannot process the data of %s: %s\n", metadata.ID, err) - return subcommands.ExitFailure - } - if err := repository.SetVersion(metadata.ID, metadata.Version+1); err != nil { - fmt.Fprintf(os.Stderr, "error: cannot process the data of %s: %s\n", metadata.ID, err) - return subcommands.ExitFailure - } - if err := repository.SetDate(metadata.ID, time.Now()); err != nil { - fmt.Fprintf(os.Stderr, "error: cannot process the data of %s: %s\n", metadata.ID, err) + if err := p.Service.Scan(metadata.ID); err != nil { + fmt.Fprintln(os.Stderr, "error: failed to scan:", err) return subcommands.ExitFailure } fmt.Println("✅", metadata.Name) @@ -62,78 +47,3 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s fmt.Println("done.") return subcommands.ExitSuccess } - -// archiveIfChanged will archive srcDir into destTarGz only if any file -// in srcDir has a modification time > the last run time stored in stateFile. -// After archiving, it updates stateFile to the current time. -func archiveIfChanged(gameID, srcDir, destTarGz, stateFile string) error { - pg := progressbar.New(-1) - destroyPg := func() { - pg.Finish() - pg.Clear() - pg.Close() - - } - defer destroyPg() - - pg.Describe("Scanning " + gameID + "...") - - // load last run time - var lastRun time.Time - data, err := os.ReadFile(stateFile) - if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to reading state file: %w", err) - } - if err == nil { - lastRun, err = time.Parse(time.RFC3339, string(data)) - if err != nil { - return fmt.Errorf("parsing state file timestamp: %w", err) - } - } - - // check for changes - changed := false - err = filepath.Walk(srcDir, func(path string, info os.FileInfo, walkErr error) error { - if walkErr != nil { - return walkErr - } - if info.ModTime().After(lastRun) { - changed = true - return io.EOF // early exit - } - return nil - }) - if err != nil && err != io.EOF { - return fmt.Errorf("failed to scanning source directory: %w", err) - } - - if !changed { - pg.Finish() - return nil - } - - // make a backup - pg.Describe("Backup current data...") - if err := repository.MakeArchive(gameID); err != nil { - return fmt.Errorf("failed to archive data: %w", err) - } - - // create archive - pg.Describe("Archiving new data...") - f, err := os.Create(destTarGz) - if err != nil { - return fmt.Errorf("failed to creating archive file: %w", err) - } - defer f.Close() - - if err := archive.Tar(f, srcDir); err != nil { - return fmt.Errorf("failed archiving files: %w", err) - } - - now := time.Now().UTC().Format(time.RFC3339) - if err := os.WriteFile(stateFile, []byte(now), 0644); err != nil { - return fmt.Errorf("updating state file: %w", err) - } - - return nil -} diff --git a/cmd/cli/commands/sync/sync.go b/cmd/cli/commands/sync/sync.go index 9702fbb..09d8b0d 100644 --- a/cmd/cli/commands/sync/sync.go +++ b/cmd/cli/commands/sync/sync.go @@ -3,6 +3,7 @@ package sync import ( "cloudsave/cmd/cli/tools/prompt" "cloudsave/cmd/cli/tools/prompt/credentials" + "cloudsave/pkg/data" "cloudsave/pkg/remote" "cloudsave/pkg/remote/client" "cloudsave/pkg/repository" @@ -12,7 +13,6 @@ import ( "fmt" "log/slog" "os" - "path/filepath" "time" "github.com/google/subcommands" @@ -21,6 +21,7 @@ import ( type ( SyncCmd struct { + Service *data.Service } ) @@ -37,7 +38,7 @@ func (p *SyncCmd) SetFlags(f *flag.FlagSet) { } func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { - games, err := repository.All() + games, err := p.Service.AllGames() if err != nil { fmt.Fprintln(os.Stderr, "error: failed to load datastore:", err) return subcommands.ExitFailure @@ -92,12 +93,6 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } pg.Describe(fmt.Sprintf("[%s] Fetching metadata...", g.Name)) - hlocal, err := repository.Hash(r.GameID) - if err != nil { - destroyPg() - slog.Error(err.Error()) - continue - } hremote, err := cli.Hash(r.GameID) if err != nil { @@ -106,13 +101,6 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) continue } - vlocal, err := repository.Version(r.GameID) - if err != nil { - destroyPg() - slog.Error(err.Error()) - continue - } - remoteMetadata, err := cli.Metadata(r.GameID) if err != nil { destroyPg() @@ -130,11 +118,11 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) slog.Warn("failed to push backup files", "err", err) } - if hlocal == hremote { + if g.MD5 == hremote { destroyPg() - if vlocal != remoteMetadata.Version { + if g.Version != remoteMetadata.Version { slog.Debug("version is not the same, but the hash is equal. Updating local database") - if err := repository.SetVersion(r.GameID, remoteMetadata.Version); err != nil { + if err := p.Service.SetVersion(r.GameID, remoteMetadata.Version); err != nil { fmt.Fprintln(os.Stderr, "error: failed to synchronize version number:", err) continue } @@ -143,7 +131,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) continue } - if vlocal > remoteMetadata.Version { + if g.Version > remoteMetadata.Version { pg.Describe(fmt.Sprintf("[%s] Pushing data...", g.Name)) if err := push(g, cli); err != nil { destroyPg() @@ -155,22 +143,21 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) continue } - if vlocal < remoteMetadata.Version { + if g.Version < remoteMetadata.Version { destroyPg() if err := pull(r.GameID, cli); err != nil { destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } - if err := repository.SetVersion(r.GameID, remoteMetadata.Version); err != nil { + + g.Version = remoteMetadata.Version + g.Date = remoteMetadata.Date + + if err := p.Service.UpdateMetadata(g.ID, g); err != nil { destroyPg() - fmt.Fprintln(os.Stderr, "error: failed to synchronize version number:", err) - continue - } - if err := repository.SetDate(r.GameID, remoteMetadata.Date); err != nil { - destroyPg() - fmt.Fprintln(os.Stderr, "error: failed to synchronize date:", err) - continue + fmt.Fprintln(os.Stderr, "failed to push:", err) + return subcommands.ExitFailure } fmt.Println(g.Name + ": pulled") continue @@ -178,8 +165,8 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) destroyPg() - if vlocal == remoteMetadata.Version { - if err := conflict(r.GameID, g, remoteMetadata, cli); err != nil { + if g.Version == remoteMetadata.Version { + if err := p.conflict(r.GameID, g, remoteMetadata, cli); err != nil { fmt.Fprintln(os.Stderr, "error: failed to resolve conflict:", err) continue } @@ -191,8 +178,8 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) return subcommands.ExitSuccess } -func conflict(gameID string, m, remoteMetadata repository.Metadata, cli *client.Client) error { - g, err := repository.One(gameID) +func (p *SyncCmd) conflict(gameID string, m, remoteMetadata repository.Metadata, cli *client.Client) error { + g, err := p.Service.One(gameID) if err != nil { slog.Warn("a conflict was found but the game is not found in the database") slog.Debug("debug info", "gameID", gameID) @@ -211,35 +198,33 @@ func conflict(gameID string, m, remoteMetadata repository.Metadata, cli *client. switch res { case prompt.My: { - if err := push(m, cli); err != nil { + if err := p.push(m, cli); err != nil { return fmt.Errorf("failed to push: %w", err) } } case prompt.Their: { - if err := pull(gameID, cli); err != nil { + if err := p.pull(gameID, cli); err != nil { return fmt.Errorf("failed to push: %w", err) } - if err := repository.SetVersion(gameID, remoteMetadata.Version); err != nil { - return fmt.Errorf("failed to synchronize version number: %w", err) - } - if err := repository.SetDate(gameID, remoteMetadata.Date); err != nil { - return fmt.Errorf("failed to synchronize date: %w", err) + g.Version = remoteMetadata.Version + g.Date = remoteMetadata.Date + + if err := p.Service.UpdateMetadata(g.ID, g); err != nil { + return fmt.Errorf("failed to push: %w", err) } } } return nil } -func push(m repository.Metadata, cli *client.Client) error { - archivePath := filepath.Join(repository.DatastorePath(), m.ID, "data.tar.gz") - - return cli.PushSave(archivePath, m) +func (p *SyncCmd) push(m repository.Metadata, cli *client.Client) error { + } -func pushBackup(m repository.Metadata, cli *client.Client) error { - bs, err := repository.Archives(m.ID) +func (p *SyncCmd) pushBackup(m repository.Metadata, cli *client.Client) error { + bs, err := p.Service.AllBackups(m.ID) if err != nil { return err } @@ -262,7 +247,7 @@ func pushBackup(m repository.Metadata, cli *client.Client) error { return nil } -func pullBackup(m repository.Metadata, cli *client.Client) error { +func (p *SyncCmd) pullBackup(m repository.Metadata, cli *client.Client) error { bs, err := cli.ListArchives(m.ID) if err != nil { return err @@ -274,20 +259,13 @@ func pullBackup(m repository.Metadata, cli *client.Client) error { return err } - linfo, err := repository.Archive(m.ID, uuid) + linfo, err := p.Service.Backup(m.ID, uuid) if err != nil { - if !errors.Is(err, os.ErrNotExist) { - return err - } - } - - path := filepath.Join(repository.DatastorePath(), m.ID, "hist", uuid) - if err := os.MkdirAll(path, 0740); err != nil { return err } - if rinfo.MD5 != linfo.MD5 { - if err := cli.PullBackup(m.ID, uuid, filepath.Join(path, "data.tar.gz")); err != nil { + if linfo != rinfo { + if err := p.Service.PullBackup(m.ID, uuid, cli); err != nil { return err } } @@ -295,10 +273,8 @@ func pullBackup(m repository.Metadata, cli *client.Client) error { return nil } -func pull(gameID string, cli *client.Client) error { - archivePath := filepath.Join(repository.DatastorePath(), gameID, "data.tar.gz") - - return cli.Pull(gameID, archivePath) +func (p *SyncCmd) pull(gameID string, cli *client.Client) error { + return p.Service.PullArchive(gameID, "", cli) } func connect(remoteCred map[string]map[string]string, r remote.Remote) (*client.Client, error) { diff --git a/cmd/cli/main.go b/cmd/cli/main.go index db4ce2a..2b43b79 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -9,27 +9,48 @@ import ( "cloudsave/cmd/cli/commands/run" "cloudsave/cmd/cli/commands/sync" "cloudsave/cmd/cli/commands/version" + "cloudsave/pkg/data" + "cloudsave/pkg/repository" "context" "flag" "os" + "path/filepath" "github.com/google/subcommands" ) func main() { + roaming, err := os.UserConfigDir() + if err != nil { + panic("failed to get user config path: " + err.Error()) + } + + datastorepath := filepath.Join(roaming, "cloudsave", "data") + err = os.MkdirAll(datastorepath, 0740) + if err != nil { + panic("cannot make the datastore:" + err.Error()) + } + + repo, err := repository.NewLazyRepository(datastorepath) + if err != nil { + panic("cannot make the datastore:" + err.Error()) + } + + s := data.NewService(repo) + subcommands.Register(subcommands.HelpCommand(), "help") subcommands.Register(subcommands.FlagsCommand(), "help") subcommands.Register(subcommands.CommandsCommand(), "help") subcommands.Register(&version.VersionCmd{}, "help") - subcommands.Register(&add.AddCmd{}, "management") - subcommands.Register(&run.RunCmd{}, "management") - subcommands.Register(&list.ListCmd{}, "management") - subcommands.Register(&remove.RemoveCmd{}, "management") + subcommands.Register(&add.AddCmd{Service: s}, "management") + subcommands.Register(&run.RunCmd{Service: s}, "management") + subcommands.Register(&list.ListCmd{Service: s}, "management") + subcommands.Register(&remove.RemoveCmd{Service: s}, "management") - subcommands.Register(&remote.RemoteCmd{}, "remote") - subcommands.Register(&sync.SyncCmd{}, "remote") - subcommands.Register(&pull.PullCmd{}, "remote") + subcommands.Register(&remote.RemoteCmd{Service: s}, "remote") + subcommands.Register(&sync.SyncCmd{Service: s}, "remote") + subcommands.Register(&pull.PullCmd{Service: s}, "remote") flag.Parse() ctx := context.Background() diff --git a/cmd/server/api/api.go b/cmd/server/api/api.go index 74c0c5c..c041293 100644 --- a/cmd/server/api/api.go +++ b/cmd/server/api/api.go @@ -1,7 +1,6 @@ package api import ( - "cloudsave/cmd/server/data" "cloudsave/pkg/repository" "encoding/json" "errors" diff --git a/cmd/server/data/data.go b/cmd/server/data/data.go deleted file mode 100644 index 1a99804..0000000 --- a/cmd/server/data/data.go +++ /dev/null @@ -1,243 +0,0 @@ -package data - -import ( - "cloudsave/pkg/repository" - "cloudsave/pkg/tools/hash" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sync" -) - -type ( - cache map[string]cachedInfo - - cachedInfo struct { - MD5 string - Version int - } -) - -var ( - ErrBackupNotExists error = errors.New("backup not found") - ErrNotExists error = errors.New("not found") - - // singleton - hashCacheMu sync.RWMutex - hashCache cache = make(map[string]cachedInfo) -) - -func (c cache) Get(gameID string) (cachedInfo, bool) { - hashCacheMu.RLock() - defer hashCacheMu.RUnlock() - - if v, ok := c[gameID]; ok { - return v, true - } - return cachedInfo{}, false -} - -func (c cache) Register(gameID string, v cachedInfo) { - hashCacheMu.Lock() - defer hashCacheMu.Unlock() - - c[gameID] = v -} - -func (c cache) Remove(gameID string) { - hashCacheMu.Lock() - defer hashCacheMu.Unlock() - - delete(c, gameID) -} - -func Write(gameID, documentRoot string, r io.Reader) error { - dataFolderPath := filepath.Join(documentRoot, "data", gameID) - partPath := filepath.Join(dataFolderPath, "data.tar.gz.part") - finalFilePath := filepath.Join(dataFolderPath, "data.tar.gz") - - if err := makeDataFolder(gameID, documentRoot); err != nil { - return err - } - - f, err := os.OpenFile(partPath, os.O_CREATE|os.O_WRONLY, 0740) - if err != nil { - return err - } - - if _, err := io.Copy(f, r); err != nil { - f.Close() - if err := os.Remove(partPath); err != nil { - return fmt.Errorf("failed to write the file and cannot clean the folder: %w", err) - } - return fmt.Errorf("failed to write the file: %w", err) - } - f.Close() - - if err := os.Rename(partPath, finalFilePath); err != nil { - return err - } - - hashCache.Remove(gameID) - return nil -} - -func WriteHist(gameID, documentRoot, uuid string, r io.Reader) error { - dataFolderPath := filepath.Join(documentRoot, "data", gameID, "hist", uuid) - partPath := filepath.Join(dataFolderPath, "data.tar.gz.part") - finalFilePath := filepath.Join(dataFolderPath, "data.tar.gz") - - if err := makeDataFolder(gameID, documentRoot); err != nil { - return err - } - - if err := os.MkdirAll(dataFolderPath, 0740); err != nil { - return err - } - - f, err := os.OpenFile(partPath, os.O_CREATE|os.O_WRONLY, 0740) - if err != nil { - return err - } - - if _, err := io.Copy(f, r); err != nil { - f.Close() - if err := os.Remove(partPath); err != nil { - return fmt.Errorf("failed to write the file and cannot clean the folder: %w", err) - } - return fmt.Errorf("failed to write the file: %w", err) - } - f.Close() - - if err := os.Rename(partPath, finalFilePath); err != nil { - return err - } - - return nil -} - -func UpdateMetadata(gameID, documentRoot string, m repository.Metadata) error { - if err := makeDataFolder(gameID, documentRoot); err != nil { - return err - } - path := filepath.Join(documentRoot, "data", gameID, "metadata.json") - - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0740) - if err != nil { - return err - } - defer f.Close() - - e := json.NewEncoder(f) - return e.Encode(m) -} - -func ArchiveInfo(gameID, documentRoot, uuid string) (repository.Backup, error) { - dataFolderPath := filepath.Join(documentRoot, "data", gameID, "hist", uuid, "data.tar.gz") - cacheID := gameID + ":" + uuid - - finfo, err := os.Stat(dataFolderPath) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return repository.Backup{}, ErrBackupNotExists - } - return repository.Backup{}, err - } - - if m, ok := hashCache.Get(cacheID); ok { - return repository.Backup{ - CreatedAt: finfo.ModTime(), - UUID: uuid, - MD5: m.MD5, - }, nil - } - - h, err := hash.FileMD5(dataFolderPath) - if err != nil { - return repository.Backup{}, fmt.Errorf("failed to calculate file md5: %w", err) - } - - hashCache.Register(cacheID, cachedInfo{ - MD5: h, - }) - - return repository.Backup{ - CreatedAt: finfo.ModTime(), - UUID: uuid, - MD5: h, - }, nil -} - -func Hash(gameID, documentRoot string) (string, error) { - path := filepath.Clean(filepath.Join(documentRoot, "data", gameID)) - - sdir, err := os.Stat(path) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return "", ErrNotExists - } - return "", err - } - - if !sdir.IsDir() { - return "", err - } - - v, err := getVersion(gameID, documentRoot) - if err != nil { - return "", fmt.Errorf("failed to read game metadata: %w", err) - } - - if m, ok := hashCache.Get(gameID); ok { - if v == m.Version { - return m.MD5, nil - } - } - - path = filepath.Join(path, "data.tar.gz") - - h, err := hash.FileMD5(path) - if err != nil { - return "", err - } - - hashCache.Register(gameID, cachedInfo{ - Version: v, - MD5: h, - }) - - return h, nil -} - -func getVersion(gameID, documentRoot string) (int, error) { - path := filepath.Join(documentRoot, "data", gameID, "metadata.json") - - f, err := os.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return 0, err - } - defer f.Close() - - d := json.NewDecoder(f) - var m repository.Metadata - if err := d.Decode(&m); err != nil { - return 0, err - } - - return m.Version, nil -} - -func makeDataFolder(gameID, documentRoot string) error { - if err := os.MkdirAll(filepath.Join(documentRoot, "data", gameID), 0740); err != nil { - return err - } - - if err := os.MkdirAll(filepath.Join(documentRoot, "data", gameID, "hist"), 0740); err != nil { - return err - } - - return nil -} diff --git a/pkg/data/data.go b/pkg/data/data.go new file mode 100644 index 0000000..6c5d28d --- /dev/null +++ b/pkg/data/data.go @@ -0,0 +1,253 @@ +package data + +import ( + "cloudsave/pkg/remote/client" + "cloudsave/pkg/repository" + "cloudsave/pkg/tools/archive" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/google/uuid" +) + +type ( + Service struct { + repo repository.Repository + } +) + +func NewService(repo repository.Repository) *Service { + return &Service{ + repo: repo, + } +} + +func (s *Service) Add(name, path, remote string) (string, error) { + gameID := repository.NewGameIdentifier(uuid.NewString()) + + if err := s.repo.Mkdir(gameID); err != nil { + return "", fmt.Errorf("failed to add game reference: %w", err) + } + + m := repository.Metadata{ + ID: gameID.Key(), + Name: name, + Path: path, + Version: 1, + Date: time.Now(), + } + + if err := s.repo.WriteMetadata(gameID, m); err != nil { + return "", fmt.Errorf("failed to add game reference: %w", err) + } + + return gameID.Key(), nil +} + +func (s *Service) One(gameID string) (repository.Metadata, error) { + id := repository.NewGameIdentifier(gameID) + + m, err := s.repo.Metadata(id) + if err != nil { + return repository.Metadata{}, fmt.Errorf("failed to get metadata: %w", err) + } + + return m, nil +} + +func (s *Service) Backup(gameID, backupID string) (repository.Backup, error) { + id := repository.NewBackupIdentifier(gameID, backupID) + + return s.repo.Backup(id) +} + +func (s *Service) UpdateMetadata(gameID string, m repository.Metadata) error { + id := repository.NewGameIdentifier(gameID) + + if err := s.repo.WriteMetadata(id, m); err != nil { + return fmt.Errorf("failed to write metadate: %w", err) + } + + return nil +} + +func (s *Service) Scan(gameID string) error { + id := repository.NewGameIdentifier(gameID) + + lastRun, err := s.repo.LastScan(id) + if err != nil { + return fmt.Errorf("failed to get last scan time: %w", err) + } + + m, err := s.repo.Metadata(id) + if err != nil { + return fmt.Errorf("failed to get game metadata: %w", err) + } + + if !IsDirectoryChanged(m.Path, lastRun) { + return nil + } + + f, err := s.repo.WriteBlob(id) + if err != nil { + return fmt.Errorf("failed to get datastore stream: %w", err) + } + if v, ok := f.(io.Closer); ok { + defer v.Close() + } + + if err := archive.Tar(f, m.Path); err != nil { + return fmt.Errorf("failed to make archive: %w", err) + } + + if err := s.repo.ResetLastScan(id); err != nil { + return fmt.Errorf("failed to reset scan date: %w", err) + } + + m.Date = time.Now() + m.Version += 1 + + if err := s.repo.WriteMetadata(id, m); err != nil { + return fmt.Errorf("failed to update metadata: %w", err) + } + + return nil +} + +func (s *Service) AllGames() ([]repository.Metadata, error) { + ids, err := s.repo.All() + if err != nil { + return nil, fmt.Errorf("failed to get the list of ids: %w", err) + } + + var ms []repository.Metadata + for _, id := range ids { + m, err := s.repo.Metadata(repository.NewGameIdentifier(id)) + if err != nil { + return nil, fmt.Errorf("failed to open metadata: %w", err) + } + ms = append(ms, m) + } + + return ms, nil +} + +func (s *Service) AllBackups(gameID string) ([]repository.Backup, error) { + ids, err := s.repo.AllHist(repository.NewGameIdentifier(gameID)) + if err != nil { + return nil, fmt.Errorf("failed to get the list of ids: %w", err) + } + + var bs []repository.Backup + for _, id := range ids { + b, err := s.repo.Backup(repository.NewBackupIdentifier(gameID, id)) + if err != nil { + return nil, fmt.Errorf("failed to open metadata: %w", err) + } + bs = append(bs, b) + } + + return bs, nil +} + +func (l Service) PullArchive(gameID, backupID string, cli *client.Client) error { + if len(backupID) > 0 { + path := l.repo.DataPath(repository.NewBackupIdentifier(gameID, backupID)) + return cli.PullBackup(gameID, backupID, filepath.Join(path, "data.tar.gz")) + } + + path := l.repo.DataPath(repository.NewGameIdentifier(gameID)) + return cli.Pull(gameID, filepath.Join(path, "data.tar.gz")) +} + +func (l Service) PullCurrent(id, path string, cli *client.Client) error { + gameID := repository.NewGameIdentifier(id) + if err := l.repo.Mkdir(gameID); err != nil { + return err + } + + m, err := cli.Metadata(id) + if err != nil { + return fmt.Errorf("failed to get metadata from the server: %w", err) + } + + if err := l.repo.WriteMetadata(gameID, m); err != nil { + return fmt.Errorf("failed to write metadata: %w", err) + } + + archivePath := filepath.Join(l.repo.DataPath(gameID), "data.tar.gz") + + if err := cli.Pull(id, archivePath); err != nil { + return fmt.Errorf("failed to pull from the server: %w", err) + } + + f, err := l.repo.ReadBlob(gameID) + if err != nil { + return fmt.Errorf("failed to open blob from local repository: %w", err) + } + + if err := os.MkdirAll(path, 0740); err != nil { + return fmt.Errorf("failed to create destination directory: %w", err) + } + + if err := archive.Untar(f, path); err != nil { + return fmt.Errorf("failed to untar archive: %w", err) + } + + if err := l.repo.ResetLastScan(gameID); err != nil { + return fmt.Errorf("failed to create .last_run file: %w", err) + } + + return nil +} + +func (l Service) PullBackup(gameID, backupID string, cli *client.Client) error { + id := repository.NewBackupIdentifier(gameID, backupID) + + archivePath := filepath.Join(l.repo.DataPath(id), "data.tar.gz") + + if err := cli.PullBackup(gameID, backupID, archivePath); err != nil { + return fmt.Errorf("failed to pull backup: %w", err) + } + + return nil +} + +func (l Service) RemoveGame(gameID string) error { + return l.repo.Remove(repository.NewGameIdentifier(gameID)) +} + +func (l Service) SetVersion(gameID string, value int) error { + id := repository.NewGameIdentifier(gameID) + + m, err := l.repo.Metadata(id) + if err != nil { + return fmt.Errorf("failed to get metadata from the server: %w", err) + } + + m.Version = value + + if err := l.repo.WriteMetadata(id, m); err != nil { + return fmt.Errorf("failed to write metadata: %w", err) + } + + return nil +} + +func IsDirectoryChanged(path string, lastRun time.Time) bool { + changed := false + _ = filepath.Walk(path, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return nil + } + if info.ModTime().After(lastRun) { + changed = true + return io.EOF // early exit + } + return nil + }) + return changed +} diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index e69290b..c07882c 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -2,7 +2,6 @@ package repository import ( "cloudsave/pkg/tools/hash" - "cloudsave/pkg/tools/id" "encoding/json" "errors" "fmt" @@ -10,8 +9,6 @@ import ( "os" "path/filepath" "time" - - "github.com/google/uuid" ) type ( @@ -21,6 +18,12 @@ type ( Path string `json:"path"` Version int `json:"version"` Date time.Time `json:"date"` + MD5 string `json:"-"` + } + + Remote struct { + URL string `json:"url"` + GameID string `json:"-"` } Backup struct { @@ -29,6 +32,57 @@ type ( UUID string `json:"uuid"` ArchivePath string `json:"-"` } + + Data struct { + Metadata Metadata + DataPath string + Backup map[string]Data + } + + GameIdentifier struct { + gameID string + } + + BackupIdentifier struct { + gameID string + backupID string + } + + Identifier interface { + Key() string + } + + LazyRepository struct { + dataRoot string + } + + EagerRepository struct { + LazyRepository + + data map[string]Data + } + + Repository interface { + Mkdir(id Identifier) error + + All() ([]string, error) + AllHist(gameID GameIdentifier) ([]string, error) + + WriteBlob(ID Identifier) (io.Writer, error) + WriteMetadata(gameID GameIdentifier, m Metadata) error + + Metadata(gameID GameIdentifier) (Metadata, error) + LastScan(gameID GameIdentifier) (time.Time, error) + ReadBlob(gameID Identifier) (io.Reader, error) + Backup(id BackupIdentifier) (Backup, error) + + SetRemote(gameID GameIdentifier, url string) error + ResetLastScan(id GameIdentifier) error + + DataPath(id Identifier) string + + Remove(gameID GameIdentifier) error + } ) var ( @@ -36,347 +90,232 @@ var ( datastorepath string ) -func init() { - var err error - roaming, err = os.UserConfigDir() - if err != nil { - panic("failed to get user config path: " + err.Error()) +func NewGameIdentifier(gameID string) GameIdentifier { + return GameIdentifier{ + gameID: gameID, } +} +func (bi GameIdentifier) Key() string { + return bi.gameID +} - datastorepath = filepath.Join(roaming, "cloudsave", "data") - err = os.MkdirAll(datastorepath, 0740) - if err != nil { - panic("cannot make the datastore:" + err.Error()) +func NewBackupIdentifier(gameID, backupID string) BackupIdentifier { + return BackupIdentifier{ + gameID: gameID, + backupID: backupID, } } -func Add(name, path string) (Metadata, error) { - m := Metadata{ - ID: id.New(), - Name: name, - Path: path, - } - - err := os.MkdirAll(filepath.Join(datastorepath, m.ID), 0740) - if err != nil { - panic("cannot make directory for the game:" + err.Error()) - } - - f, err := os.OpenFile(filepath.Join(datastorepath, m.ID, "metadata.json"), os.O_CREATE|os.O_WRONLY, 0740) - if err != nil { - return Metadata{}, fmt.Errorf("cannot open the metadata file in the datastore: %w", err) - } - defer f.Close() - - e := json.NewEncoder(f) - err = e.Encode(m) - if err != nil { - return Metadata{}, fmt.Errorf("cannot write into the metadata file in the datastore: %w", err) - } - - return m, nil +func (bi BackupIdentifier) Key() string { + return bi.gameID + ":" + bi.backupID } -func Register(m Metadata, path string) error { - m.Path = path - - err := os.MkdirAll(filepath.Join(datastorepath, m.ID), 0740) +func NewLazyRepository(dataRootPath string) (Repository, error) { + m, err := os.Stat(dataRootPath) if err != nil { - panic("cannot make directory for the game:" + err.Error()) + return nil, fmt.Errorf("failed to open datastore: %w", err) } - f, err := os.OpenFile(filepath.Join(datastorepath, m.ID, "metadata.json"), os.O_CREATE|os.O_WRONLY, 0740) - if err != nil { - return fmt.Errorf("cannot open the metadata file in the datastore: %w", err) - } - defer f.Close() - - e := json.NewEncoder(f) - err = e.Encode(m) - if err != nil { - return fmt.Errorf("cannot write into the metadata file in the datastore: %w", err) + if !m.IsDir() { + return nil, fmt.Errorf("failed to open datastore: not a directory") } - return nil + return &LazyRepository{ + dataRoot: dataRootPath, + }, nil } -func All() ([]Metadata, error) { - ds, err := os.ReadDir(datastorepath) - if err != nil { - return nil, fmt.Errorf("cannot open the datastore: %w", err) - } - - var datastore []Metadata - for _, d := range ds { - content, err := os.ReadFile(filepath.Join(datastorepath, d.Name(), "metadata.json")) - if err != nil { - continue - } - - var m Metadata - err = json.Unmarshal(content, &m) - if err != nil { - return nil, fmt.Errorf("corrupted datastore: failed to parse %s/metadata.json: %w", d.Name(), err) - } - - datastore = append(datastore, m) - } - return datastore, nil +func (l LazyRepository) Mkdir(id Identifier) error { + return os.MkdirAll(l.DataPath(id), 0740) } -func One(gameID string) (Metadata, error) { - _, err := os.ReadDir(datastorepath) +func (l LazyRepository) All() ([]string, error) { + dir, err := os.ReadDir(l.dataRoot) if err != nil { - return Metadata{}, fmt.Errorf("cannot open the datastore: %w", err) + return nil, fmt.Errorf("failed to open directory: %w", err) } - content, err := os.ReadFile(filepath.Join(datastorepath, gameID, "metadata.json")) - if err != nil { - return Metadata{}, fmt.Errorf("game not found: %w", err) + var res []string + for _, d := range dir { + res = append(res, d.Name()) } - var m Metadata - err = json.Unmarshal(content, &m) - if err != nil { - return Metadata{}, fmt.Errorf("corrupted datastore: failed to parse %s/metadata.json: %w", gameID, err) - } - - return m, nil -} - -func MakeArchive(gameID string) error { - path := filepath.Join(datastorepath, gameID, "data.tar.gz") - - // open old - f, err := os.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil - } - return fmt.Errorf("failed to open old file: %w", err) - } - defer f.Close() - - histDirPath := filepath.Join(datastorepath, gameID, "hist", uuid.NewString()) - if err := os.MkdirAll(histDirPath, 0740); err != nil { - return fmt.Errorf("failed to make directory: %w", err) - } - - // open new - nf, err := os.OpenFile(filepath.Join(histDirPath, "data.tar.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) - if err != nil { - return fmt.Errorf("failed to open new file: %w", err) - } - defer nf.Close() - - // copy - if _, err := io.Copy(nf, f); err != nil { - return fmt.Errorf("failed to copy data: %w", err) - } - - return nil -} - -func RestoreArchive(gameID, uuid string) error { - histDirPath := filepath.Join(datastorepath, gameID, "hist", uuid) - if err := os.MkdirAll(histDirPath, 0740); err != nil { - return fmt.Errorf("failed to make directory: %w", err) - } - - // open old - nf, err := os.OpenFile(filepath.Join(histDirPath, "data.tar.gz"), os.O_RDONLY, 0) - if err != nil { - return fmt.Errorf("failed to open new file: %w", err) - } - defer nf.Close() - - path := filepath.Join(datastorepath, gameID, "data.tar.gz") - - // open new - f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil - } - return fmt.Errorf("failed to open old file: %w", err) - } - defer f.Close() - - // copy - if _, err := io.Copy(f, nf); err != nil { - return fmt.Errorf("failed to copy data: %w", err) - } - - return nil -} - -func Archive(gameID, uuid string) (Backup, error) { - histDirPath := filepath.Join(datastorepath, gameID, "hist", uuid) - if err := os.MkdirAll(histDirPath, 0740); err != nil { - return Backup{}, fmt.Errorf("failed to make 'hist' directory") - } - - finfo, err := os.Stat(histDirPath) - if err != nil { - return Backup{}, fmt.Errorf("corrupted datastore: %w", err) - } - archivePath := filepath.Join(histDirPath, "data.tar.gz") - - h, err := hash.FileMD5(archivePath) - if err != nil { - return Backup{}, fmt.Errorf("failed to calculate md5 hash: %w", err) - } - - b := Backup{ - CreatedAt: finfo.ModTime(), - UUID: filepath.Base(finfo.Name()), - MD5: h, - ArchivePath: archivePath, - } - - return b, nil -} - -func Archives(gameID string) ([]Backup, error) { - histDirPath := filepath.Join(datastorepath, gameID, "hist") - if err := os.MkdirAll(histDirPath, 0740); err != nil { - return nil, fmt.Errorf("failed to make 'hist' directory") - } - - d, err := os.ReadDir(histDirPath) - if err != nil { - return nil, fmt.Errorf("failed to open 'hist' directory") - } - - var res []Backup - for _, f := range d { - finfo, err := f.Info() - if err != nil { - return nil, fmt.Errorf("corrupted datastore: %w", err) - } - path := filepath.Join(histDirPath, finfo.Name()) - archivePath := filepath.Join(path, "data.tar.gz") - - h, err := hash.FileMD5(archivePath) - if err != nil { - return nil, fmt.Errorf("failed to calculate md5 hash: %w", err) - } - - b := Backup{ - CreatedAt: finfo.ModTime(), - UUID: filepath.Base(finfo.Name()), - MD5: h, - ArchivePath: archivePath, - } - - res = append(res, b) - } return res, nil } -func DatastorePath() string { - return datastorepath +func (l LazyRepository) AllHist(id GameIdentifier) ([]string, error) { + path := l.DataPath(id) + + dir, err := os.ReadDir(filepath.Join(path, "hist")) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to open directory: %w", err) + } + + var res []string + for _, d := range dir { + res = append(res, d.Name()) + } + + return res, nil } -func Remove(gameID string) error { - err := os.RemoveAll(filepath.Join(datastorepath, gameID)) +func (l LazyRepository) WriteBlob(ID Identifier) (io.Writer, error) { + path := l.DataPath(ID) + + dst, err := os.OpenFile(filepath.Join(path, "data.tar.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) if err != nil { - return err + return nil, fmt.Errorf("failed to open destination file: %w", err) } - return nil + + return dst, nil } -func Hash(gameID string) (string, error) { - path := filepath.Join(datastorepath, gameID, "data.tar.gz") +func (l LazyRepository) WriteMetadata(id GameIdentifier, m Metadata) error { + path := l.DataPath(id) - return hash.FileMD5(path) -} - -func Version(gameID string) (int, error) { - path := filepath.Join(datastorepath, gameID, "metadata.json") - - f, err := os.OpenFile(path, os.O_RDONLY, 0) + dst, err := os.OpenFile(filepath.Join(path, "metadata.json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) if err != nil { - return 0, err + return fmt.Errorf("failed to open destination file: %w", err) } - defer f.Close() + defer dst.Close() - var metadata Metadata - d := json.NewDecoder(f) - err = d.Decode(&metadata) - if err != nil { - return 0, err - } - - return metadata.Version, nil -} - -func SetVersion(gameID string, version int) error { - path := filepath.Join(datastorepath, gameID, "metadata.json") - - f, err := os.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return err - } - - var metadata Metadata - d := json.NewDecoder(f) - err = d.Decode(&metadata) - if err != nil { - f.Close() - return err - } - - f.Close() - - metadata.Version = version - - f, err = os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0740) - if err != nil { - return err - } - defer f.Close() - - e := json.NewEncoder(f) - err = e.Encode(metadata) - if err != nil { - return err + e := json.NewEncoder(dst) + if err := e.Encode(m); err != nil { + return fmt.Errorf("failed to encode data: %w", err) } return nil } -func SetDate(gameID string, dt time.Time) error { - path := filepath.Join(datastorepath, gameID, "metadata.json") +func (l LazyRepository) Metadata(id GameIdentifier) (Metadata, error) { + path := l.DataPath(id) - f, err := os.OpenFile(path, os.O_RDONLY, 0) + src, err := os.OpenFile(filepath.Join(path, "metadata.json"), os.O_RDONLY, 0) if err != nil { - return err + return Metadata{}, fmt.Errorf("corrupted datastore: failed to open metadata: %w", err) } - var metadata Metadata - d := json.NewDecoder(f) - err = d.Decode(&metadata) - if err != nil { - f.Close() - return err + var m Metadata + d := json.NewDecoder(src) + if err := d.Decode(&m); err != nil { + return Metadata{}, fmt.Errorf("corrupted datastore: failed to parse metadata: %w", err) } - f.Close() - - metadata.Date = dt - - f, err = os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0740) + m.MD5, err = hash.FileMD5(filepath.Join(path, "data.tar.gz")) if err != nil { - return err + return Metadata{}, fmt.Errorf("failed to calculate md5: %w", err) + } + + return m, nil +} + +func (l LazyRepository) Backup(id BackupIdentifier) (Backup, error) { + path := l.DataPath(id) + + fs, err := os.Stat(filepath.Join(path, "data.tar.gz")) + if err != nil { + return Backup{}, fmt.Errorf("corrupted datastore: failed to open metadata: %w", err) + } + + h, err := hash.FileMD5(filepath.Join(path, "data.tar.gz")) + if err != nil { + return Backup{}, fmt.Errorf("corrupted datastore: failed to open metadata: %w", err) + } + + return Backup{ + CreatedAt: fs.ModTime(), + MD5: h, + UUID: id.backupID, + ArchivePath: filepath.Join(path, "data.tar.gz"), + }, nil +} + +func (l LazyRepository) LastScan(id GameIdentifier) (time.Time, error) { + path := l.DataPath(id) + + data, err := os.ReadFile(filepath.Join(path, ".last_run")) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return time.Time{}, nil + } + return time.Time{}, fmt.Errorf("failed to reading state file: %w", err) + } + + lastRun, err := time.Parse(time.RFC3339, string(data)) + if err != nil { + return time.Time{}, fmt.Errorf("parsing state file timestamp: %w", err) + } + + return lastRun, nil +} + +func (l LazyRepository) ResetLastScan(id GameIdentifier) error { + path := l.DataPath(id) + + f, err := os.OpenFile(filepath.Join(path, ".last_run"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) } defer f.Close() - e := json.NewEncoder(f) - err = e.Encode(metadata) - if err != nil { - return err + data := time.Now().Format(time.RFC3339) + + if _, err := f.WriteString(data); err != nil { + return fmt.Errorf("failed to write file: %w", err) } return nil } + +func (l LazyRepository) ReadBlob(id Identifier) (io.Reader, error) { + path := l.DataPath(id) + + dst, err := os.OpenFile(filepath.Join(path, "data.tar.gz"), os.O_RDONLY, 0) + if err != nil { + return nil, fmt.Errorf("failed to open blob: %w", err) + } + + return dst, nil +} + +func (l LazyRepository) SetRemote(id GameIdentifier, url string) error { + path := l.DataPath(id) + + src, err := os.OpenFile(filepath.Join(path, "remote.json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) + if err != nil { + return fmt.Errorf("failed to open remote description: %w", err) + } + defer src.Close() + + var r Remote + d := json.NewEncoder(src) + if err := d.Encode(r); err != nil { + return fmt.Errorf("failed to marshall remote description: %w", err) + } + + return nil +} + +func (l LazyRepository) Remove(id GameIdentifier) error { + path := l.DataPath(id) + + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("failed to remove game folder from the datastore: %w", err) + } + + return nil +} + +func (r LazyRepository) DataPath(id Identifier) string { + switch identifier := id.(type) { + case GameIdentifier: + return filepath.Join(r.dataRoot, identifier.gameID) + case BackupIdentifier: + return filepath.Join(r.dataRoot, identifier.gameID, "hist", identifier.backupID) + } + + panic("identifier type not supported") +} From e34dc704ca5e8fa2e49db753f6f8ff77d869480c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Sun, 10 Aug 2025 02:03:27 +0200 Subject: [PATCH 2/7] wip refactoring --- cmd/cli/commands/run/run.go | 4 + cmd/cli/commands/sync/sync.go | 14 +-- cmd/server/api/api.go | 29 +++-- cmd/server/runner.go | 24 +++- pkg/data/data.go | 93 +++++++++++++++ pkg/repository/repository.go | 217 +++++++++++++++++++++++++++++----- 6 files changed, 333 insertions(+), 48 deletions(-) diff --git a/cmd/cli/commands/run/run.go b/cmd/cli/commands/run/run.go index e381a4f..ed645bc 100644 --- a/cmd/cli/commands/run/run.go +++ b/cmd/cli/commands/run/run.go @@ -37,6 +37,10 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s } for _, metadata := range datastore { + if err := p.Service.MakeBackup(metadata.ID); err != nil { + fmt.Fprintln(os.Stderr, "error: failed to make backup:", err) + return subcommands.ExitFailure + } if err := p.Service.Scan(metadata.ID); err != nil { fmt.Fprintln(os.Stderr, "error: failed to scan:", err) return subcommands.ExitFailure diff --git a/cmd/cli/commands/sync/sync.go b/cmd/cli/commands/sync/sync.go index 09d8b0d..ad26bed 100644 --- a/cmd/cli/commands/sync/sync.go +++ b/cmd/cli/commands/sync/sync.go @@ -78,13 +78,13 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) if !exists { pg.Describe(fmt.Sprintf("[%s] Pushing data...", g.Name)) - if err := push(g, cli); err != nil { + if err := p.push(g, cli); err != nil { destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } pg.Describe(fmt.Sprintf("[%s] Pushing backup...", g.Name)) - if err := pushBackup(g, cli); err != nil { + if err := p.pushBackup(g, cli); err != nil { destroyPg() slog.Warn("failed to push backup files", "err", err) } @@ -109,12 +109,12 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } pg.Describe(fmt.Sprintf("[%s] Pulling backup...", g.Name)) - if err := pullBackup(g, cli); err != nil { + if err := p.pullBackup(g, cli); err != nil { slog.Warn("failed to pull backup files", "err", err) } pg.Describe(fmt.Sprintf("[%s] Pushing backup...", g.Name)) - if err := pushBackup(g, cli); err != nil { + if err := p.pushBackup(g, cli); err != nil { slog.Warn("failed to push backup files", "err", err) } @@ -133,7 +133,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) if g.Version > remoteMetadata.Version { pg.Describe(fmt.Sprintf("[%s] Pushing data...", g.Name)) - if err := push(g, cli); err != nil { + if err := p.push(g, cli); err != nil { destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure @@ -145,7 +145,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) if g.Version < remoteMetadata.Version { destroyPg() - if err := pull(r.GameID, cli); err != nil { + if err := p.pull(r.GameID, cli); err != nil { destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure @@ -220,7 +220,7 @@ func (p *SyncCmd) conflict(gameID string, m, remoteMetadata repository.Metadata, } func (p *SyncCmd) push(m repository.Metadata, cli *client.Client) error { - + return p.Service.PushArchive(m.ID, "", cli) } func (p *SyncCmd) pushBackup(m repository.Metadata, cli *client.Client) error { diff --git a/cmd/server/api/api.go b/cmd/server/api/api.go index c041293..d7fb2cf 100644 --- a/cmd/server/api/api.go +++ b/cmd/server/api/api.go @@ -1,6 +1,7 @@ package api import ( + "cloudsave/pkg/data" "cloudsave/pkg/repository" "encoding/json" "errors" @@ -19,16 +20,18 @@ import ( type ( HTTPServer struct { Server *http.Server + Service *data.Service documentRoot string } ) // NewServer start the http server -func NewServer(documentRoot string, creds map[string]string, port int) *HTTPServer { +func NewServer(documentRoot string, srv *data.Service, creds map[string]string, port int) *HTTPServer { if !filepath.IsAbs(documentRoot) { panic("the document root is not an absolute path") } s := &HTTPServer{ + Service: srv, documentRoot: documentRoot, } router := chi.NewRouter() @@ -194,14 +197,14 @@ func (s HTTPServer) upload(w http.ResponseWriter, r *http.Request) { defer file.Close() //TODO make a transaction - if err := data.UpdateMetadata(id, s.documentRoot, m); err != nil { + if err := s.Service.UpdateMetadata(id, m); err != nil { fmt.Fprintln(os.Stderr, "error: failed to write metadata to disk:", err) internalServerError(w, r) return } - if err := data.Write(id, s.documentRoot, file); err != nil { - fmt.Fprintln(os.Stderr, "error: failed to write file to disk:", err) + if err := s.Service.Copy(id, file); err != nil { + fmt.Fprintln(os.Stderr, "error: failed to write data to disk:", err) internalServerError(w, r) return } @@ -267,8 +270,8 @@ func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) { } defer file.Close() - if err := data.WriteHist(gameID, s.documentRoot, uuid, file); err != nil { - fmt.Fprintln(os.Stderr, "error: failed to write file to disk:", err) + if err := s.Service.CopyBackup(gameID, uuid, file); err != nil { + fmt.Fprintln(os.Stderr, "error: failed to write data to disk:", err) internalServerError(w, r) return } @@ -323,10 +326,10 @@ func (s HTTPServer) histExists(w http.ResponseWriter, r *http.Request) { gameID := chi.URLParam(r, "id") uuid := chi.URLParam(r, "uuid") - finfo, err := data.ArchiveInfo(gameID, s.documentRoot, uuid) + finfo, err := s.Service.Backup(gameID, uuid) if err != nil { - if errors.Is(err, data.ErrBackupNotExists) { - notFound("backup not found", w, r) + if errors.Is(err, repository.ErrNotFound) { + notFound("not found", w, r) return } fmt.Fprintln(os.Stderr, "error: failed to read data:", err) @@ -340,10 +343,10 @@ func (s HTTPServer) histExists(w http.ResponseWriter, r *http.Request) { func (s HTTPServer) hash(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "id") - sum, err := data.Hash(id, s.documentRoot) + m, err := s.Service.One(id) if err != nil { - if errors.Is(err, data.ErrNotExists) { - notFound("id not found", w, r) + if errors.Is(err, repository.ErrNotFound) { + notFound("not found", w, r) return } fmt.Fprintln(os.Stderr, "error: an error occured while calculating the hash:", err) @@ -351,7 +354,7 @@ func (s HTTPServer) hash(w http.ResponseWriter, r *http.Request) { return } - ok(sum, w, r) + ok(m.MD5, w, r) } func (s HTTPServer) metadata(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/server/runner.go b/cmd/server/runner.go index 5063dfa..3e532f2 100644 --- a/cmd/server/runner.go +++ b/cmd/server/runner.go @@ -4,6 +4,8 @@ import ( "cloudsave/cmd/server/api" "cloudsave/cmd/server/security/htpasswd" "cloudsave/pkg/constants" + "cloudsave/pkg/data" + "cloudsave/pkg/repository" "flag" "fmt" "path/filepath" @@ -16,16 +18,36 @@ func run() { var documentRoot string var port int + var noCache bool flag.StringVar(&documentRoot, "document-root", defaultDocumentRoot, "Define the path to the document root") flag.IntVar(&port, "port", 8080, "Define the port of the server") + flag.BoolVar(&noCache, "no-cache", false, "Disable the cache") flag.Parse() h, err := htpasswd.Open(filepath.Join(documentRoot, ".htpasswd")) if err != nil { fatal("failed to load .htpasswd: "+err.Error(), 1) } + var repo repository.Repository + if noCache { + r, err := repository.NewEagerRepository(filepath.Join(documentRoot, "data")) + if err != nil { + fatal("failed to load datastore: "+err.Error(), 1) + } + if err := r.Preload(); err != nil { + fatal("failed to load datastore: "+err.Error(), 1) + } + repo = r + } else { + repo, err = repository.NewLazyRepository(filepath.Join(documentRoot, "data")) + if err != nil { + fatal("failed to load datastore: "+err.Error(), 1) + } + } - server := api.NewServer(documentRoot, h.Content(), port) + s := data.NewService(repo) + + server := api.NewServer(documentRoot, s, h.Content(), port) fmt.Println("starting server at :" + strconv.Itoa(port)) if err := server.Server.ListenAndServe(); err != nil { diff --git a/pkg/data/data.go b/pkg/data/data.go index 6c5d28d..525ffbe 100644 --- a/pkg/data/data.go +++ b/pkg/data/data.go @@ -61,12 +61,20 @@ func (s *Service) One(gameID string) (repository.Metadata, error) { func (s *Service) Backup(gameID, backupID string) (repository.Backup, error) { id := repository.NewBackupIdentifier(gameID, backupID) + if err := s.repo.Mkdir(id); err != nil { + return repository.Backup{}, fmt.Errorf("failed to make game dir: %w", err) + } + return s.repo.Backup(id) } func (s *Service) UpdateMetadata(gameID string, m repository.Metadata) error { id := repository.NewGameIdentifier(gameID) + if err := s.repo.Mkdir(id); err != nil { + return fmt.Errorf("failed to make game dir: %w", err) + } + if err := s.repo.WriteMetadata(id, m); err != nil { return fmt.Errorf("failed to write metadate: %w", err) } @@ -117,6 +125,38 @@ func (s *Service) Scan(gameID string) error { return nil } +func (s *Service) MakeBackup(gameID string) error { + var id repository.Identifier = repository.NewGameIdentifier(gameID) + + src, err := s.repo.ReadBlob(id) + if err != nil { + return err + } + if v, ok := src.(io.Closer); ok { + defer v.Close() + } + + id = repository.NewBackupIdentifier(gameID, uuid.NewString()) + + if err := s.repo.Mkdir(id); err != nil { + return err + } + + dst, err := s.repo.WriteBlob(id) + if err != nil { + return err + } + if v, ok := dst.(io.Closer); ok { + defer v.Close() + } + + if _, err := io.Copy(dst, src); err != nil { + return err + } + + return nil +} + func (s *Service) AllGames() ([]repository.Metadata, error) { ids, err := s.repo.All() if err != nil { @@ -163,6 +203,21 @@ func (l Service) PullArchive(gameID, backupID string, cli *client.Client) error return cli.Pull(gameID, filepath.Join(path, "data.tar.gz")) } +func (l Service) PushArchive(gameID, backupID string, cli *client.Client) error { + m, err := l.repo.Metadata(repository.NewGameIdentifier(gameID)) + if err != nil { + return err + } + + if len(backupID) > 0 { + path := l.repo.DataPath(repository.NewBackupIdentifier(gameID, backupID)) + return cli.PushSave(filepath.Join(path, "data.taz.gz"), m) + } + + path := l.repo.DataPath(repository.NewGameIdentifier(gameID)) + return cli.PushSave(filepath.Join(path, "data.tar.gz"), m) +} + func (l Service) PullCurrent(id, path string, cli *client.Client) error { gameID := repository.NewGameIdentifier(id) if err := l.repo.Mkdir(gameID); err != nil { @@ -251,3 +306,41 @@ func IsDirectoryChanged(path string, lastRun time.Time) bool { }) return changed } + +func (l Service) Copy(id string, src io.Reader) error { + dst, err := l.repo.WriteBlob(repository.NewGameIdentifier(id)) + if err != nil { + return err + } + if v, ok := dst.(io.Closer); ok { + defer v.Close() + } + + if _, err := io.Copy(dst, src); err != nil { + return err + } + + return nil +} + +func (l Service) CopyBackup(gameID, backupID string, src io.Reader) error { + id := repository.NewBackupIdentifier(gameID, backupID) + + if err := l.repo.Mkdir(id); err != nil { + return err + } + + dst, err := l.repo.WriteBlob(id) + if err != nil { + return err + } + if v, ok := dst.(io.Closer); ok { + defer v.Close() + } + + if _, err := io.Copy(dst, src); err != nil { + return err + } + + return nil +} diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index c07882c..50d5b53 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -35,8 +35,9 @@ type ( Data struct { Metadata Metadata + Remote *Remote DataPath string - Backup map[string]Data + Backup map[string]Backup } GameIdentifier struct { @@ -57,7 +58,7 @@ type ( } EagerRepository struct { - LazyRepository + Repository data map[string]Data } @@ -75,6 +76,7 @@ type ( LastScan(gameID GameIdentifier) (time.Time, error) ReadBlob(gameID Identifier) (io.Reader, error) Backup(id BackupIdentifier) (Backup, error) + Remote(id GameIdentifier) (*Remote, error) SetRemote(gameID GameIdentifier, url string) error ResetLastScan(id GameIdentifier) error @@ -86,8 +88,7 @@ type ( ) var ( - roaming string - datastorepath string + ErrNotFound error = errors.New("not found") ) func NewGameIdentifier(gameID string) GameIdentifier { @@ -110,14 +111,19 @@ func (bi BackupIdentifier) Key() string { return bi.gameID + ":" + bi.backupID } -func NewLazyRepository(dataRootPath string) (Repository, error) { - m, err := os.Stat(dataRootPath) - if err != nil { - return nil, fmt.Errorf("failed to open datastore: %w", err) - } - - if !m.IsDir() { - return nil, fmt.Errorf("failed to open datastore: not a directory") +func NewLazyRepository(dataRootPath string) (*LazyRepository, error) { + if m, err := os.Stat(dataRootPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(dataRootPath, 0740); err != nil { + return nil, fmt.Errorf("failed to make the directory: %w", err) + } + } else { + return nil, fmt.Errorf("failed to open datastore: %w", err) + } + } else { + if !m.IsDir() { + return nil, fmt.Errorf("failed to open datastore: not a directory") + } } return &LazyRepository{ @@ -125,11 +131,11 @@ func NewLazyRepository(dataRootPath string) (Repository, error) { }, nil } -func (l LazyRepository) Mkdir(id Identifier) error { +func (l *LazyRepository) Mkdir(id Identifier) error { return os.MkdirAll(l.DataPath(id), 0740) } -func (l LazyRepository) All() ([]string, error) { +func (l *LazyRepository) All() ([]string, error) { dir, err := os.ReadDir(l.dataRoot) if err != nil { return nil, fmt.Errorf("failed to open directory: %w", err) @@ -143,7 +149,7 @@ func (l LazyRepository) All() ([]string, error) { return res, nil } -func (l LazyRepository) AllHist(id GameIdentifier) ([]string, error) { +func (l *LazyRepository) AllHist(id GameIdentifier) ([]string, error) { path := l.DataPath(id) dir, err := os.ReadDir(filepath.Join(path, "hist")) @@ -162,7 +168,7 @@ func (l LazyRepository) AllHist(id GameIdentifier) ([]string, error) { return res, nil } -func (l LazyRepository) WriteBlob(ID Identifier) (io.Writer, error) { +func (l *LazyRepository) WriteBlob(ID Identifier) (io.Writer, error) { path := l.DataPath(ID) dst, err := os.OpenFile(filepath.Join(path, "data.tar.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) @@ -173,7 +179,7 @@ func (l LazyRepository) WriteBlob(ID Identifier) (io.Writer, error) { return dst, nil } -func (l LazyRepository) WriteMetadata(id GameIdentifier, m Metadata) error { +func (l *LazyRepository) WriteMetadata(id GameIdentifier, m Metadata) error { path := l.DataPath(id) dst, err := os.OpenFile(filepath.Join(path, "metadata.json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) @@ -190,11 +196,14 @@ func (l LazyRepository) WriteMetadata(id GameIdentifier, m Metadata) error { return nil } -func (l LazyRepository) Metadata(id GameIdentifier) (Metadata, error) { +func (l *LazyRepository) Metadata(id GameIdentifier) (Metadata, error) { path := l.DataPath(id) src, err := os.OpenFile(filepath.Join(path, "metadata.json"), os.O_RDONLY, 0) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return Metadata{}, ErrNotFound + } return Metadata{}, fmt.Errorf("corrupted datastore: failed to open metadata: %w", err) } @@ -212,11 +221,14 @@ func (l LazyRepository) Metadata(id GameIdentifier) (Metadata, error) { return m, nil } -func (l LazyRepository) Backup(id BackupIdentifier) (Backup, error) { +func (l *LazyRepository) Backup(id BackupIdentifier) (Backup, error) { path := l.DataPath(id) fs, err := os.Stat(filepath.Join(path, "data.tar.gz")) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return Backup{}, ErrNotFound + } return Backup{}, fmt.Errorf("corrupted datastore: failed to open metadata: %w", err) } @@ -233,7 +245,7 @@ func (l LazyRepository) Backup(id BackupIdentifier) (Backup, error) { }, nil } -func (l LazyRepository) LastScan(id GameIdentifier) (time.Time, error) { +func (l *LazyRepository) LastScan(id GameIdentifier) (time.Time, error) { path := l.DataPath(id) data, err := os.ReadFile(filepath.Join(path, ".last_run")) @@ -252,7 +264,7 @@ func (l LazyRepository) LastScan(id GameIdentifier) (time.Time, error) { return lastRun, nil } -func (l LazyRepository) ResetLastScan(id GameIdentifier) error { +func (l *LazyRepository) ResetLastScan(id GameIdentifier) error { path := l.DataPath(id) f, err := os.OpenFile(filepath.Join(path, ".last_run"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) @@ -270,7 +282,7 @@ func (l LazyRepository) ResetLastScan(id GameIdentifier) error { return nil } -func (l LazyRepository) ReadBlob(id Identifier) (io.Reader, error) { +func (l *LazyRepository) ReadBlob(id Identifier) (io.Reader, error) { path := l.DataPath(id) dst, err := os.OpenFile(filepath.Join(path, "data.tar.gz"), os.O_RDONLY, 0) @@ -281,7 +293,7 @@ func (l LazyRepository) ReadBlob(id Identifier) (io.Reader, error) { return dst, nil } -func (l LazyRepository) SetRemote(id GameIdentifier, url string) error { +func (l *LazyRepository) SetRemote(id GameIdentifier, url string) error { path := l.DataPath(id) src, err := os.OpenFile(filepath.Join(path, "remote.json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) @@ -291,15 +303,38 @@ func (l LazyRepository) SetRemote(id GameIdentifier, url string) error { defer src.Close() var r Remote - d := json.NewEncoder(src) - if err := d.Encode(r); err != nil { + r.URL = url + + e := json.NewEncoder(src) + if err := e.Encode(r); err != nil { return fmt.Errorf("failed to marshall remote description: %w", err) } return nil } -func (l LazyRepository) Remove(id GameIdentifier) error { +func (l *LazyRepository) Remote(id GameIdentifier) (*Remote, error) { + path := l.DataPath(id) + + src, err := os.OpenFile(filepath.Join(path, "remote.json"), os.O_RDONLY, 0) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to open remote description: %w", err) + } + defer src.Close() + + var r Remote + e := json.NewDecoder(src) + if err := e.Decode(&r); err != nil { + return nil, fmt.Errorf("failed to marshall remote description: %w", err) + } + + return &r, nil +} + +func (l *LazyRepository) Remove(id GameIdentifier) error { path := l.DataPath(id) if err := os.RemoveAll(path); err != nil { @@ -309,7 +344,7 @@ func (l LazyRepository) Remove(id GameIdentifier) error { return nil } -func (r LazyRepository) DataPath(id Identifier) string { +func (r *LazyRepository) DataPath(id Identifier) string { switch identifier := id.(type) { case GameIdentifier: return filepath.Join(r.dataRoot, identifier.gameID) @@ -319,3 +354,131 @@ func (r LazyRepository) DataPath(id Identifier) string { panic("identifier type not supported") } + +func NewEagerRepository(dataRootPath string) (*EagerRepository, error) { + r, err := NewLazyRepository(dataRootPath) + if err != nil { + return nil, err + } + + return &EagerRepository{ + Repository: r, + data: make(map[string]Data), + }, nil +} + +func (r *EagerRepository) Preload() error { + games, err := r.Repository.All() + if err != nil { + return fmt.Errorf("failed to load all data: %w", err) + } + + for _, g := range games { + backup, err := r.Repository.AllHist(NewGameIdentifier(g)) + if err != nil { + return fmt.Errorf("[%s] failed to load hist data: %w", g, err) + } + + remote, err := r.Repository.Remote(NewGameIdentifier(g)) + if err != nil { + return fmt.Errorf("[%s] failed to load remote metadata: %w", g, err) + } + + m, err := r.Repository.Metadata(NewGameIdentifier(g)) + if err != nil { + return fmt.Errorf("[%s] failed to load metadata: %w", g, err) + } + + backups := make(map[string]Backup) + for _, b := range backup { + info, err := r.Repository.Backup(NewBackupIdentifier(g, b)) + if err != nil { + return fmt.Errorf("[%s] failed to get backup information: %w", g, err) + } + + backups[b] = info + } + + r.data[g] = Data{ + Metadata: m, + Remote: remote, + DataPath: r.DataPath(NewGameIdentifier(g)), + Backup: backups, + } + } + + return nil +} + +func (r *EagerRepository) All() ([]string, error) { + var res []string + for _, g := range r.data { + res = append(res, g.Metadata.ID) + } + + return res, nil +} + +func (r *EagerRepository) AllHist(id GameIdentifier) ([]string, error) { + var res []string + if d, ok := r.data[id.gameID]; ok { + for _, b := range d.Backup { + res = append(res, b.UUID) + } + } + return res, nil +} + +func (r *EagerRepository) WriteMetadata(id GameIdentifier, m Metadata) error { + err := r.Repository.WriteMetadata(id, m) + if err != nil { + return err + } + + d := r.data[id.gameID] + d.Metadata = m + r.data[id.gameID] = d + + return nil +} + +func (r *EagerRepository) Metadata(id GameIdentifier) (Metadata, error) { + if d, ok := r.data[id.gameID]; ok { + return d.Metadata, nil + } + return Metadata{}, ErrNotFound +} + +func (r *EagerRepository) Backup(id BackupIdentifier) (Backup, error) { + if d, ok := r.data[id.gameID]; ok { + if b, ok := d.Backup[id.backupID]; ok { + return b, nil + } + } + return Backup{}, ErrNotFound +} + +func (r *EagerRepository) SetRemote(id GameIdentifier, url string) error { + err := r.Repository.SetRemote(id, url) + if err != nil { + return err + } + + d := r.data[id.gameID] + d.Remote = &Remote{ + URL: url, + GameID: d.Metadata.ID, + } + r.data[id.gameID] = d + + return nil +} + +func (r *EagerRepository) Remove(id GameIdentifier) error { + if err := r.Repository.Remove(id); err != nil { + return err + } + + delete(r.data, id.gameID) + return nil +} From ab857bd0ddb6f363bfe6a261f0196d564bc8ffa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Sun, 10 Aug 2025 02:09:34 +0200 Subject: [PATCH 3/7] ver num --- pkg/constants/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index ba987f3..b9ec354 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -1,5 +1,5 @@ package constants -const Version = "0.0.3" +const Version = "0.0.4" const ApiVersion = 1 From 822c93bf4c79badd33cd1458dc6a5d9ce07483a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Sun, 10 Aug 2025 02:32:56 +0200 Subject: [PATCH 4/7] opti --- cmd/server/api/api.go | 15 ++-------- cmd/web/server/server.go | 64 +++++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/cmd/server/api/api.go b/cmd/server/api/api.go index d7fb2cf..816a199 100644 --- a/cmd/server/api/api.go +++ b/cmd/server/api/api.go @@ -215,20 +215,9 @@ func (s HTTPServer) upload(w http.ResponseWriter, r *http.Request) { func (s HTTPServer) allHist(w http.ResponseWriter, r *http.Request) { gameID := chi.URLParam(r, "id") - path := filepath.Join(s.documentRoot, "data", gameID, "hist") datastore := make([]string, 0) - if _, err := os.Stat(path); err != nil { - if errors.Is(err, os.ErrNotExist) { - ok(datastore, w, r) - return - } - fmt.Fprintln(os.Stderr, "failed to open datastore (", s.documentRoot, "):", err) - internalServerError(w, r) - return - } - - ds, err := os.ReadDir(path) + ds, err := s.Service.AllBackups(gameID) if err != nil { fmt.Fprintln(os.Stderr, "failed to open datastore (", s.documentRoot, "):", err) internalServerError(w, r) @@ -236,7 +225,7 @@ func (s HTTPServer) allHist(w http.ResponseWriter, r *http.Request) { } for _, d := range ds { - datastore = append(datastore, d.Name()) + datastore = append(datastore, d.UUID) } ok(datastore, w, r) diff --git a/cmd/web/server/server.go b/cmd/web/server/server.go index 7eee69a..6c0d427 100644 --- a/cmd/web/server/server.go +++ b/cmd/web/server/server.go @@ -12,6 +12,7 @@ import ( "net/http" "runtime" "slices" + "sync" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" @@ -160,38 +161,59 @@ func (s *HTTPServer) detailled(w http.ResponseWriter, r *http.Request) { return } - save, err := cli.Metadata(id) - if err != nil { - if errors.Is(err, client.ErrUnauthorized) { + var wg sync.WaitGroup + var err1, err2, err3 error + var save repository.Metadata + var h string + var ids []string + + wg.Add(1) + go func() { + save, err1 = cli.Metadata(id) + wg.Done() + }() + + wg.Add(1) + go func() { + h, err2 = cli.Hash(id) + wg.Done() + }() + + wg.Add(1) + go func() { + ids, err3 = cli.ListArchives(id) + wg.Done() + }() + + wg.Wait() + + if err1 != nil || err2 != nil || err3 != nil { + if errors.Is(err1, client.ErrUnauthorized) { unauthorized("Unable to access resources", w, r) return } - slog.Error("unable to connect to the remote", "err", err) + slog.Error("unable to connect to the remote", "err", err1) return } - h, err := cli.Hash(id) - if err != nil { - slog.Error("unable to connect to the remote", "err", err) - return - } - - ids, err := cli.ListArchives(id) - if err != nil { - slog.Error("unable to connect to the remote", "err", err) - return - } + wg = sync.WaitGroup{} var bm []repository.Backup for _, i := range ids { - b, err := cli.ArchiveInfo(id, i) - if err != nil { - slog.Error("unable to connect to the remote", "err", err) - return - } - bm = append(bm, b) + wg.Add(1) + go func() { + defer wg.Done() + b, err := cli.ArchiveInfo(id, i) + if err != nil { + slog.Error("unable to connect to the remote", "err", err) + return + } + bm = append(bm, b) + }() } + wg.Wait() + payload := DetaillePayload{ Save: save, Hash: h, From aa29fae900d7535ac66ffed4c5cee58bf3301a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Sun, 10 Aug 2025 12:34:08 +0200 Subject: [PATCH 5/7] fix windows sync --- cmd/cli/commands/sync/sync.go | 2 +- pkg/remote/client/client.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/cli/commands/sync/sync.go b/cmd/cli/commands/sync/sync.go index ad26bed..bc3bd91 100644 --- a/cmd/cli/commands/sync/sync.go +++ b/cmd/cli/commands/sync/sync.go @@ -264,7 +264,7 @@ func (p *SyncCmd) pullBackup(m repository.Metadata, cli *client.Client) error { return err } - if linfo != rinfo { + if linfo.MD5 != rinfo.MD5 { if err := p.Service.PullBackup(m.ID, uuid, cli); err != nil { return err } diff --git a/pkg/remote/client/client.go b/pkg/remote/client/client.go index c161939..83a741d 100644 --- a/pkg/remote/client/client.go +++ b/pkg/remote/client/client.go @@ -276,15 +276,16 @@ func (c *Client) PullBackup(gameID, uuid, archivePath string) error { if err != nil { return fmt.Errorf("failed to open file: %w", err) } - defer f.Close() res, err := cli.Do(req) if err != nil { + f.Close() return fmt.Errorf("cannot connect to remote: %w", err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { + f.Close() return fmt.Errorf("cannot connect to remote: server return code: %s", res.Status) } @@ -295,8 +296,10 @@ func (c *Client) PullBackup(gameID, uuid, archivePath string) error { defer bar.Close() if _, err := io.Copy(io.MultiWriter(f, bar), res.Body); err != nil { + f.Close() return fmt.Errorf("an error occured while copying the file from the remote: %w", err) } + f.Close() if err := os.Rename(archivePath+".part", archivePath); err != nil { return fmt.Errorf("failed to move temporary data: %w", err) From 851ff898869641e51e715194799ad238dcc08c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Sun, 17 Aug 2025 00:30:02 +0200 Subject: [PATCH 6/7] implement applu --- cmd/cli/commands/apply/apply.go | 23 ++++++++++++++----- cmd/cli/main.go | 3 +++ pkg/constants/constants.go | 2 +- pkg/data/data.go | 39 +++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/cmd/cli/commands/apply/apply.go b/cmd/cli/commands/apply/apply.go index 041b805..c0c3de9 100644 --- a/cmd/cli/commands/apply/apply.go +++ b/cmd/cli/commands/apply/apply.go @@ -1,6 +1,7 @@ package apply import ( + "cloudsave/pkg/data" "context" "flag" "fmt" @@ -11,13 +12,14 @@ import ( type ( ListCmd struct { + Service *data.Service } ) func (*ListCmd) Name() string { return "apply" } func (*ListCmd) Synopsis() string { return "apply a backup" } func (*ListCmd) Usage() string { - return `Usage: cloudsave apply + return `Usage: cloudsave apply [BACKUP_ID] Apply a backup ` @@ -27,15 +29,26 @@ func (p *ListCmd) SetFlags(f *flag.FlagSet) { } func (p *ListCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { - if f.NArg() != 2 { + if f.NArg() < 1 { fmt.Fprintln(os.Stderr, "error: missing game ID and/or backup uuid") return subcommands.ExitUsageError } - //gameID := f.Arg(0) - //uuid := f.Arg(1) + gameID := f.Arg(0) + uuid := f.Arg(1) - panic("not implemented") + if len(uuid) == 0 { + if err := p.Service.ApplyCurrent(gameID); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to apply: %s", err) + return subcommands.ExitFailure + } + return subcommands.ExitSuccess + } + + if err := p.Service.ApplyBackup(gameID, uuid); err != nil { + fmt.Fprintf(os.Stderr, "error: failed to apply: %s", err) + return subcommands.ExitFailure + } return subcommands.ExitSuccess } diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 2b43b79..8ec7531 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -2,6 +2,7 @@ package main import ( "cloudsave/cmd/cli/commands/add" + "cloudsave/cmd/cli/commands/apply" "cloudsave/cmd/cli/commands/list" "cloudsave/cmd/cli/commands/pull" "cloudsave/cmd/cli/commands/remote" @@ -48,6 +49,8 @@ func main() { subcommands.Register(&list.ListCmd{Service: s}, "management") subcommands.Register(&remove.RemoveCmd{Service: s}, "management") + subcommands.Register(&apply.ListCmd{Service: s}, "restore") + subcommands.Register(&remote.RemoteCmd{Service: s}, "remote") subcommands.Register(&sync.SyncCmd{Service: s}, "remote") subcommands.Register(&pull.PullCmd{Service: s}, "remote") diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index b9ec354..cd821e8 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -1,5 +1,5 @@ package constants -const Version = "0.0.4" +const Version = "0.0.5" const ApiVersion = 1 diff --git a/pkg/data/data.go b/pkg/data/data.go index 525ffbe..8a6c9b3 100644 --- a/pkg/data/data.go +++ b/pkg/data/data.go @@ -344,3 +344,42 @@ func (l Service) CopyBackup(gameID, backupID string, src io.Reader) error { return nil } + +func (l Service) ApplyCurrent(gameID string) error { + id := repository.NewGameIdentifier(gameID) + path := l.repo.DataPath(id) + + g, err := l.repo.Metadata(id) + if err != nil { + return err + } + + return l.apply(path, g.Path) +} + +func (l Service) ApplyBackup(gameID, backupID string) error { + id := repository.NewGameIdentifier(gameID) + fullID := repository.NewBackupIdentifier(gameID, backupID) + path := l.repo.DataPath(fullID) + + g, err := l.repo.Metadata(id) + if err != nil { + return err + } + + return l.apply(path, g.Path) +} + +func (l Service) apply(src, dst string) error { + if err := os.RemoveAll(dst); err != nil { + return fmt.Errorf("failed to remove old save: %w", err) + } + + f, err := os.OpenFile(src, os.O_RDONLY, 0) + if err != nil { + return fmt.Errorf("failed to open archive: %w", err) + } + defer f.Close() + + return archive.Untar(f, dst) +} From 54dcc7d00690cda8bf78c983e3fc4307540788ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Sun, 17 Aug 2025 00:42:12 +0200 Subject: [PATCH 7/7] show cmd --- cmd/cli/commands/show/show.go | 51 +++++++++++++++++++++++++++++++++++ cmd/cli/main.go | 2 ++ pkg/data/data.go | 4 +-- 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 cmd/cli/commands/show/show.go diff --git a/cmd/cli/commands/show/show.go b/cmd/cli/commands/show/show.go new file mode 100644 index 0000000..4086c92 --- /dev/null +++ b/cmd/cli/commands/show/show.go @@ -0,0 +1,51 @@ +package show + +import ( + "cloudsave/pkg/data" + "context" + "flag" + "fmt" + "os" + + "github.com/google/subcommands" +) + +type ( + ShowCmd struct { + Service *data.Service + } +) + +func (*ShowCmd) Name() string { return "show" } +func (*ShowCmd) Synopsis() string { return "show metadata about game" } +func (*ShowCmd) Usage() string { + return `Usage: cloudsave show + +Show metdata about a game +` +} + +func (p *ShowCmd) SetFlags(f *flag.FlagSet) { +} + +func (p *ShowCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus { + if f.NArg() != 1 { + fmt.Fprintln(os.Stderr, "error: missing game ID") + return subcommands.ExitUsageError + } + + gameID := f.Arg(0) + g, err := p.Service.One(gameID) + if err != nil { + fmt.Fprintf(os.Stderr, "error: failed to apply: %s", err) + return subcommands.ExitFailure + } + + fmt.Println(g.Name) + fmt.Println("------") + fmt.Println("Version: ", g.Version) + fmt.Println("Path: ", g.Path) + fmt.Println("MD5: ", g.MD5) + + return subcommands.ExitSuccess +} diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 8ec7531..255e06d 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -8,6 +8,7 @@ import ( "cloudsave/cmd/cli/commands/remote" "cloudsave/cmd/cli/commands/remove" "cloudsave/cmd/cli/commands/run" + "cloudsave/cmd/cli/commands/show" "cloudsave/cmd/cli/commands/sync" "cloudsave/cmd/cli/commands/version" "cloudsave/pkg/data" @@ -48,6 +49,7 @@ func main() { subcommands.Register(&run.RunCmd{Service: s}, "management") subcommands.Register(&list.ListCmd{Service: s}, "management") subcommands.Register(&remove.RemoveCmd{Service: s}, "management") + subcommands.Register(&show.ShowCmd{Service: s}, "management") subcommands.Register(&apply.ListCmd{Service: s}, "restore") diff --git a/pkg/data/data.go b/pkg/data/data.go index 8a6c9b3..ff8063f 100644 --- a/pkg/data/data.go +++ b/pkg/data/data.go @@ -354,7 +354,7 @@ func (l Service) ApplyCurrent(gameID string) error { return err } - return l.apply(path, g.Path) + return l.apply(filepath.Join(path, "data.tar.gz"), g.Path) } func (l Service) ApplyBackup(gameID, backupID string) error { @@ -367,7 +367,7 @@ func (l Service) ApplyBackup(gameID, backupID string) error { return err } - return l.apply(path, g.Path) + return l.apply(filepath.Join(path, "data.tar.gz"), g.Path) } func (l Service) apply(src, dst string) error {