From e34dc704ca5e8fa2e49db753f6f8ff77d869480c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Sun, 10 Aug 2025 02:03:27 +0200 Subject: [PATCH] wip refactoring --- cmd/cli/commands/run/run.go | 4 + cmd/cli/commands/sync/sync.go | 14 +-- cmd/server/api/api.go | 29 +++-- cmd/server/runner.go | 24 +++- pkg/data/data.go | 93 +++++++++++++++ pkg/repository/repository.go | 217 +++++++++++++++++++++++++++++----- 6 files changed, 333 insertions(+), 48 deletions(-) diff --git a/cmd/cli/commands/run/run.go b/cmd/cli/commands/run/run.go index e381a4f..ed645bc 100644 --- a/cmd/cli/commands/run/run.go +++ b/cmd/cli/commands/run/run.go @@ -37,6 +37,10 @@ func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s } for _, metadata := range datastore { + if err := p.Service.MakeBackup(metadata.ID); err != nil { + fmt.Fprintln(os.Stderr, "error: failed to make backup:", err) + return subcommands.ExitFailure + } if err := p.Service.Scan(metadata.ID); err != nil { fmt.Fprintln(os.Stderr, "error: failed to scan:", err) return subcommands.ExitFailure diff --git a/cmd/cli/commands/sync/sync.go b/cmd/cli/commands/sync/sync.go index 09d8b0d..ad26bed 100644 --- a/cmd/cli/commands/sync/sync.go +++ b/cmd/cli/commands/sync/sync.go @@ -78,13 +78,13 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) if !exists { pg.Describe(fmt.Sprintf("[%s] Pushing data...", g.Name)) - if err := push(g, cli); err != nil { + if err := p.push(g, cli); err != nil { destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure } pg.Describe(fmt.Sprintf("[%s] Pushing backup...", g.Name)) - if err := pushBackup(g, cli); err != nil { + if err := p.pushBackup(g, cli); err != nil { destroyPg() slog.Warn("failed to push backup files", "err", err) } @@ -109,12 +109,12 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) } pg.Describe(fmt.Sprintf("[%s] Pulling backup...", g.Name)) - if err := pullBackup(g, cli); err != nil { + if err := p.pullBackup(g, cli); err != nil { slog.Warn("failed to pull backup files", "err", err) } pg.Describe(fmt.Sprintf("[%s] Pushing backup...", g.Name)) - if err := pushBackup(g, cli); err != nil { + if err := p.pushBackup(g, cli); err != nil { slog.Warn("failed to push backup files", "err", err) } @@ -133,7 +133,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) if g.Version > remoteMetadata.Version { pg.Describe(fmt.Sprintf("[%s] Pushing data...", g.Name)) - if err := push(g, cli); err != nil { + if err := p.push(g, cli); err != nil { destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure @@ -145,7 +145,7 @@ func (p *SyncCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) if g.Version < remoteMetadata.Version { destroyPg() - if err := pull(r.GameID, cli); err != nil { + if err := p.pull(r.GameID, cli); err != nil { destroyPg() fmt.Fprintln(os.Stderr, "failed to push:", err) return subcommands.ExitFailure @@ -220,7 +220,7 @@ func (p *SyncCmd) conflict(gameID string, m, remoteMetadata repository.Metadata, } func (p *SyncCmd) push(m repository.Metadata, cli *client.Client) error { - + return p.Service.PushArchive(m.ID, "", cli) } func (p *SyncCmd) pushBackup(m repository.Metadata, cli *client.Client) error { diff --git a/cmd/server/api/api.go b/cmd/server/api/api.go index c041293..d7fb2cf 100644 --- a/cmd/server/api/api.go +++ b/cmd/server/api/api.go @@ -1,6 +1,7 @@ package api import ( + "cloudsave/pkg/data" "cloudsave/pkg/repository" "encoding/json" "errors" @@ -19,16 +20,18 @@ import ( type ( HTTPServer struct { Server *http.Server + Service *data.Service documentRoot string } ) // NewServer start the http server -func NewServer(documentRoot string, creds map[string]string, port int) *HTTPServer { +func NewServer(documentRoot string, srv *data.Service, creds map[string]string, port int) *HTTPServer { if !filepath.IsAbs(documentRoot) { panic("the document root is not an absolute path") } s := &HTTPServer{ + Service: srv, documentRoot: documentRoot, } router := chi.NewRouter() @@ -194,14 +197,14 @@ func (s HTTPServer) upload(w http.ResponseWriter, r *http.Request) { defer file.Close() //TODO make a transaction - if err := data.UpdateMetadata(id, s.documentRoot, m); err != nil { + if err := s.Service.UpdateMetadata(id, m); err != nil { fmt.Fprintln(os.Stderr, "error: failed to write metadata to disk:", err) internalServerError(w, r) return } - if err := data.Write(id, s.documentRoot, file); err != nil { - fmt.Fprintln(os.Stderr, "error: failed to write file to disk:", err) + if err := s.Service.Copy(id, file); err != nil { + fmt.Fprintln(os.Stderr, "error: failed to write data to disk:", err) internalServerError(w, r) return } @@ -267,8 +270,8 @@ func (s HTTPServer) histUpload(w http.ResponseWriter, r *http.Request) { } defer file.Close() - if err := data.WriteHist(gameID, s.documentRoot, uuid, file); err != nil { - fmt.Fprintln(os.Stderr, "error: failed to write file to disk:", err) + if err := s.Service.CopyBackup(gameID, uuid, file); err != nil { + fmt.Fprintln(os.Stderr, "error: failed to write data to disk:", err) internalServerError(w, r) return } @@ -323,10 +326,10 @@ func (s HTTPServer) histExists(w http.ResponseWriter, r *http.Request) { gameID := chi.URLParam(r, "id") uuid := chi.URLParam(r, "uuid") - finfo, err := data.ArchiveInfo(gameID, s.documentRoot, uuid) + finfo, err := s.Service.Backup(gameID, uuid) if err != nil { - if errors.Is(err, data.ErrBackupNotExists) { - notFound("backup not found", w, r) + if errors.Is(err, repository.ErrNotFound) { + notFound("not found", w, r) return } fmt.Fprintln(os.Stderr, "error: failed to read data:", err) @@ -340,10 +343,10 @@ func (s HTTPServer) histExists(w http.ResponseWriter, r *http.Request) { func (s HTTPServer) hash(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "id") - sum, err := data.Hash(id, s.documentRoot) + m, err := s.Service.One(id) if err != nil { - if errors.Is(err, data.ErrNotExists) { - notFound("id not found", w, r) + if errors.Is(err, repository.ErrNotFound) { + notFound("not found", w, r) return } fmt.Fprintln(os.Stderr, "error: an error occured while calculating the hash:", err) @@ -351,7 +354,7 @@ func (s HTTPServer) hash(w http.ResponseWriter, r *http.Request) { return } - ok(sum, w, r) + ok(m.MD5, w, r) } func (s HTTPServer) metadata(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/server/runner.go b/cmd/server/runner.go index 5063dfa..3e532f2 100644 --- a/cmd/server/runner.go +++ b/cmd/server/runner.go @@ -4,6 +4,8 @@ import ( "cloudsave/cmd/server/api" "cloudsave/cmd/server/security/htpasswd" "cloudsave/pkg/constants" + "cloudsave/pkg/data" + "cloudsave/pkg/repository" "flag" "fmt" "path/filepath" @@ -16,16 +18,36 @@ func run() { var documentRoot string var port int + var noCache bool flag.StringVar(&documentRoot, "document-root", defaultDocumentRoot, "Define the path to the document root") flag.IntVar(&port, "port", 8080, "Define the port of the server") + flag.BoolVar(&noCache, "no-cache", false, "Disable the cache") flag.Parse() h, err := htpasswd.Open(filepath.Join(documentRoot, ".htpasswd")) if err != nil { fatal("failed to load .htpasswd: "+err.Error(), 1) } + var repo repository.Repository + if noCache { + r, err := repository.NewEagerRepository(filepath.Join(documentRoot, "data")) + if err != nil { + fatal("failed to load datastore: "+err.Error(), 1) + } + if err := r.Preload(); err != nil { + fatal("failed to load datastore: "+err.Error(), 1) + } + repo = r + } else { + repo, err = repository.NewLazyRepository(filepath.Join(documentRoot, "data")) + if err != nil { + fatal("failed to load datastore: "+err.Error(), 1) + } + } - server := api.NewServer(documentRoot, h.Content(), port) + s := data.NewService(repo) + + server := api.NewServer(documentRoot, s, h.Content(), port) fmt.Println("starting server at :" + strconv.Itoa(port)) if err := server.Server.ListenAndServe(); err != nil { diff --git a/pkg/data/data.go b/pkg/data/data.go index 6c5d28d..525ffbe 100644 --- a/pkg/data/data.go +++ b/pkg/data/data.go @@ -61,12 +61,20 @@ func (s *Service) One(gameID string) (repository.Metadata, error) { func (s *Service) Backup(gameID, backupID string) (repository.Backup, error) { id := repository.NewBackupIdentifier(gameID, backupID) + if err := s.repo.Mkdir(id); err != nil { + return repository.Backup{}, fmt.Errorf("failed to make game dir: %w", err) + } + return s.repo.Backup(id) } func (s *Service) UpdateMetadata(gameID string, m repository.Metadata) error { id := repository.NewGameIdentifier(gameID) + if err := s.repo.Mkdir(id); err != nil { + return fmt.Errorf("failed to make game dir: %w", err) + } + if err := s.repo.WriteMetadata(id, m); err != nil { return fmt.Errorf("failed to write metadate: %w", err) } @@ -117,6 +125,38 @@ func (s *Service) Scan(gameID string) error { return nil } +func (s *Service) MakeBackup(gameID string) error { + var id repository.Identifier = repository.NewGameIdentifier(gameID) + + src, err := s.repo.ReadBlob(id) + if err != nil { + return err + } + if v, ok := src.(io.Closer); ok { + defer v.Close() + } + + id = repository.NewBackupIdentifier(gameID, uuid.NewString()) + + if err := s.repo.Mkdir(id); err != nil { + return err + } + + dst, err := s.repo.WriteBlob(id) + if err != nil { + return err + } + if v, ok := dst.(io.Closer); ok { + defer v.Close() + } + + if _, err := io.Copy(dst, src); err != nil { + return err + } + + return nil +} + func (s *Service) AllGames() ([]repository.Metadata, error) { ids, err := s.repo.All() if err != nil { @@ -163,6 +203,21 @@ func (l Service) PullArchive(gameID, backupID string, cli *client.Client) error return cli.Pull(gameID, filepath.Join(path, "data.tar.gz")) } +func (l Service) PushArchive(gameID, backupID string, cli *client.Client) error { + m, err := l.repo.Metadata(repository.NewGameIdentifier(gameID)) + if err != nil { + return err + } + + if len(backupID) > 0 { + path := l.repo.DataPath(repository.NewBackupIdentifier(gameID, backupID)) + return cli.PushSave(filepath.Join(path, "data.taz.gz"), m) + } + + path := l.repo.DataPath(repository.NewGameIdentifier(gameID)) + return cli.PushSave(filepath.Join(path, "data.tar.gz"), m) +} + func (l Service) PullCurrent(id, path string, cli *client.Client) error { gameID := repository.NewGameIdentifier(id) if err := l.repo.Mkdir(gameID); err != nil { @@ -251,3 +306,41 @@ func IsDirectoryChanged(path string, lastRun time.Time) bool { }) return changed } + +func (l Service) Copy(id string, src io.Reader) error { + dst, err := l.repo.WriteBlob(repository.NewGameIdentifier(id)) + if err != nil { + return err + } + if v, ok := dst.(io.Closer); ok { + defer v.Close() + } + + if _, err := io.Copy(dst, src); err != nil { + return err + } + + return nil +} + +func (l Service) CopyBackup(gameID, backupID string, src io.Reader) error { + id := repository.NewBackupIdentifier(gameID, backupID) + + if err := l.repo.Mkdir(id); err != nil { + return err + } + + dst, err := l.repo.WriteBlob(id) + if err != nil { + return err + } + if v, ok := dst.(io.Closer); ok { + defer v.Close() + } + + if _, err := io.Copy(dst, src); err != nil { + return err + } + + return nil +} diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index c07882c..50d5b53 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -35,8 +35,9 @@ type ( Data struct { Metadata Metadata + Remote *Remote DataPath string - Backup map[string]Data + Backup map[string]Backup } GameIdentifier struct { @@ -57,7 +58,7 @@ type ( } EagerRepository struct { - LazyRepository + Repository data map[string]Data } @@ -75,6 +76,7 @@ type ( LastScan(gameID GameIdentifier) (time.Time, error) ReadBlob(gameID Identifier) (io.Reader, error) Backup(id BackupIdentifier) (Backup, error) + Remote(id GameIdentifier) (*Remote, error) SetRemote(gameID GameIdentifier, url string) error ResetLastScan(id GameIdentifier) error @@ -86,8 +88,7 @@ type ( ) var ( - roaming string - datastorepath string + ErrNotFound error = errors.New("not found") ) func NewGameIdentifier(gameID string) GameIdentifier { @@ -110,14 +111,19 @@ func (bi BackupIdentifier) Key() string { return bi.gameID + ":" + bi.backupID } -func NewLazyRepository(dataRootPath string) (Repository, error) { - m, err := os.Stat(dataRootPath) - if err != nil { - return nil, fmt.Errorf("failed to open datastore: %w", err) - } - - if !m.IsDir() { - return nil, fmt.Errorf("failed to open datastore: not a directory") +func NewLazyRepository(dataRootPath string) (*LazyRepository, error) { + if m, err := os.Stat(dataRootPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(dataRootPath, 0740); err != nil { + return nil, fmt.Errorf("failed to make the directory: %w", err) + } + } else { + return nil, fmt.Errorf("failed to open datastore: %w", err) + } + } else { + if !m.IsDir() { + return nil, fmt.Errorf("failed to open datastore: not a directory") + } } return &LazyRepository{ @@ -125,11 +131,11 @@ func NewLazyRepository(dataRootPath string) (Repository, error) { }, nil } -func (l LazyRepository) Mkdir(id Identifier) error { +func (l *LazyRepository) Mkdir(id Identifier) error { return os.MkdirAll(l.DataPath(id), 0740) } -func (l LazyRepository) All() ([]string, error) { +func (l *LazyRepository) All() ([]string, error) { dir, err := os.ReadDir(l.dataRoot) if err != nil { return nil, fmt.Errorf("failed to open directory: %w", err) @@ -143,7 +149,7 @@ func (l LazyRepository) All() ([]string, error) { return res, nil } -func (l LazyRepository) AllHist(id GameIdentifier) ([]string, error) { +func (l *LazyRepository) AllHist(id GameIdentifier) ([]string, error) { path := l.DataPath(id) dir, err := os.ReadDir(filepath.Join(path, "hist")) @@ -162,7 +168,7 @@ func (l LazyRepository) AllHist(id GameIdentifier) ([]string, error) { return res, nil } -func (l LazyRepository) WriteBlob(ID Identifier) (io.Writer, error) { +func (l *LazyRepository) WriteBlob(ID Identifier) (io.Writer, error) { path := l.DataPath(ID) dst, err := os.OpenFile(filepath.Join(path, "data.tar.gz"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) @@ -173,7 +179,7 @@ func (l LazyRepository) WriteBlob(ID Identifier) (io.Writer, error) { return dst, nil } -func (l LazyRepository) WriteMetadata(id GameIdentifier, m Metadata) error { +func (l *LazyRepository) WriteMetadata(id GameIdentifier, m Metadata) error { path := l.DataPath(id) dst, err := os.OpenFile(filepath.Join(path, "metadata.json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) @@ -190,11 +196,14 @@ func (l LazyRepository) WriteMetadata(id GameIdentifier, m Metadata) error { return nil } -func (l LazyRepository) Metadata(id GameIdentifier) (Metadata, error) { +func (l *LazyRepository) Metadata(id GameIdentifier) (Metadata, error) { path := l.DataPath(id) src, err := os.OpenFile(filepath.Join(path, "metadata.json"), os.O_RDONLY, 0) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return Metadata{}, ErrNotFound + } return Metadata{}, fmt.Errorf("corrupted datastore: failed to open metadata: %w", err) } @@ -212,11 +221,14 @@ func (l LazyRepository) Metadata(id GameIdentifier) (Metadata, error) { return m, nil } -func (l LazyRepository) Backup(id BackupIdentifier) (Backup, error) { +func (l *LazyRepository) Backup(id BackupIdentifier) (Backup, error) { path := l.DataPath(id) fs, err := os.Stat(filepath.Join(path, "data.tar.gz")) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return Backup{}, ErrNotFound + } return Backup{}, fmt.Errorf("corrupted datastore: failed to open metadata: %w", err) } @@ -233,7 +245,7 @@ func (l LazyRepository) Backup(id BackupIdentifier) (Backup, error) { }, nil } -func (l LazyRepository) LastScan(id GameIdentifier) (time.Time, error) { +func (l *LazyRepository) LastScan(id GameIdentifier) (time.Time, error) { path := l.DataPath(id) data, err := os.ReadFile(filepath.Join(path, ".last_run")) @@ -252,7 +264,7 @@ func (l LazyRepository) LastScan(id GameIdentifier) (time.Time, error) { return lastRun, nil } -func (l LazyRepository) ResetLastScan(id GameIdentifier) error { +func (l *LazyRepository) ResetLastScan(id GameIdentifier) error { path := l.DataPath(id) f, err := os.OpenFile(filepath.Join(path, ".last_run"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) @@ -270,7 +282,7 @@ func (l LazyRepository) ResetLastScan(id GameIdentifier) error { return nil } -func (l LazyRepository) ReadBlob(id Identifier) (io.Reader, error) { +func (l *LazyRepository) ReadBlob(id Identifier) (io.Reader, error) { path := l.DataPath(id) dst, err := os.OpenFile(filepath.Join(path, "data.tar.gz"), os.O_RDONLY, 0) @@ -281,7 +293,7 @@ func (l LazyRepository) ReadBlob(id Identifier) (io.Reader, error) { return dst, nil } -func (l LazyRepository) SetRemote(id GameIdentifier, url string) error { +func (l *LazyRepository) SetRemote(id GameIdentifier, url string) error { path := l.DataPath(id) src, err := os.OpenFile(filepath.Join(path, "remote.json"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0740) @@ -291,15 +303,38 @@ func (l LazyRepository) SetRemote(id GameIdentifier, url string) error { defer src.Close() var r Remote - d := json.NewEncoder(src) - if err := d.Encode(r); err != nil { + r.URL = url + + e := json.NewEncoder(src) + if err := e.Encode(r); err != nil { return fmt.Errorf("failed to marshall remote description: %w", err) } return nil } -func (l LazyRepository) Remove(id GameIdentifier) error { +func (l *LazyRepository) Remote(id GameIdentifier) (*Remote, error) { + path := l.DataPath(id) + + src, err := os.OpenFile(filepath.Join(path, "remote.json"), os.O_RDONLY, 0) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("failed to open remote description: %w", err) + } + defer src.Close() + + var r Remote + e := json.NewDecoder(src) + if err := e.Decode(&r); err != nil { + return nil, fmt.Errorf("failed to marshall remote description: %w", err) + } + + return &r, nil +} + +func (l *LazyRepository) Remove(id GameIdentifier) error { path := l.DataPath(id) if err := os.RemoveAll(path); err != nil { @@ -309,7 +344,7 @@ func (l LazyRepository) Remove(id GameIdentifier) error { return nil } -func (r LazyRepository) DataPath(id Identifier) string { +func (r *LazyRepository) DataPath(id Identifier) string { switch identifier := id.(type) { case GameIdentifier: return filepath.Join(r.dataRoot, identifier.gameID) @@ -319,3 +354,131 @@ func (r LazyRepository) DataPath(id Identifier) string { panic("identifier type not supported") } + +func NewEagerRepository(dataRootPath string) (*EagerRepository, error) { + r, err := NewLazyRepository(dataRootPath) + if err != nil { + return nil, err + } + + return &EagerRepository{ + Repository: r, + data: make(map[string]Data), + }, nil +} + +func (r *EagerRepository) Preload() error { + games, err := r.Repository.All() + if err != nil { + return fmt.Errorf("failed to load all data: %w", err) + } + + for _, g := range games { + backup, err := r.Repository.AllHist(NewGameIdentifier(g)) + if err != nil { + return fmt.Errorf("[%s] failed to load hist data: %w", g, err) + } + + remote, err := r.Repository.Remote(NewGameIdentifier(g)) + if err != nil { + return fmt.Errorf("[%s] failed to load remote metadata: %w", g, err) + } + + m, err := r.Repository.Metadata(NewGameIdentifier(g)) + if err != nil { + return fmt.Errorf("[%s] failed to load metadata: %w", g, err) + } + + backups := make(map[string]Backup) + for _, b := range backup { + info, err := r.Repository.Backup(NewBackupIdentifier(g, b)) + if err != nil { + return fmt.Errorf("[%s] failed to get backup information: %w", g, err) + } + + backups[b] = info + } + + r.data[g] = Data{ + Metadata: m, + Remote: remote, + DataPath: r.DataPath(NewGameIdentifier(g)), + Backup: backups, + } + } + + return nil +} + +func (r *EagerRepository) All() ([]string, error) { + var res []string + for _, g := range r.data { + res = append(res, g.Metadata.ID) + } + + return res, nil +} + +func (r *EagerRepository) AllHist(id GameIdentifier) ([]string, error) { + var res []string + if d, ok := r.data[id.gameID]; ok { + for _, b := range d.Backup { + res = append(res, b.UUID) + } + } + return res, nil +} + +func (r *EagerRepository) WriteMetadata(id GameIdentifier, m Metadata) error { + err := r.Repository.WriteMetadata(id, m) + if err != nil { + return err + } + + d := r.data[id.gameID] + d.Metadata = m + r.data[id.gameID] = d + + return nil +} + +func (r *EagerRepository) Metadata(id GameIdentifier) (Metadata, error) { + if d, ok := r.data[id.gameID]; ok { + return d.Metadata, nil + } + return Metadata{}, ErrNotFound +} + +func (r *EagerRepository) Backup(id BackupIdentifier) (Backup, error) { + if d, ok := r.data[id.gameID]; ok { + if b, ok := d.Backup[id.backupID]; ok { + return b, nil + } + } + return Backup{}, ErrNotFound +} + +func (r *EagerRepository) SetRemote(id GameIdentifier, url string) error { + err := r.Repository.SetRemote(id, url) + if err != nil { + return err + } + + d := r.data[id.gameID] + d.Remote = &Remote{ + URL: url, + GameID: d.Metadata.ID, + } + r.data[id.gameID] = d + + return nil +} + +func (r *EagerRepository) Remove(id GameIdentifier) error { + if err := r.Repository.Remove(id); err != nil { + return err + } + + delete(r.data, id.gameID) + return nil +}