From 4a3fe068a395797bac7c716b3e956b4012b8ee11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lie=20DELHAIE?= Date: Tue, 28 Oct 2025 19:22:21 +0100 Subject: [PATCH] first working version --- .vscode/launch.json | 18 +++ cmd/server/api/api.go | 19 ++- cmd/server/core/git/git.go | 54 ++++++- cmd/server/core/runtime/runtime.go | 86 +++++++++++ .../storage/migrations/001_first_schema.sql | 8 + cmd/server/core/storage/storage.go | 142 +++++++++++++++--- cmd/server/main.go | 20 ++- pkg/project/file.go | 89 +++++++++-- pkg/project/project.go | 19 ++- 9 files changed, 400 insertions(+), 55 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 cmd/server/core/runtime/runtime.go diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..6708b7a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "Server", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/server", + "args": ["-db-path", "./data.db"], + "cwd": "${workspaceFolder}" + } + ] +} \ No newline at end of file diff --git a/cmd/server/api/api.go b/cmd/server/api/api.go index 73a63d3..1470585 100644 --- a/cmd/server/api/api.go +++ b/cmd/server/api/api.go @@ -11,20 +11,24 @@ import ( "net/http" "runtime" + cronruntime "mirror-sync/cmd/server/core/runtime" + "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" ) type ( HTTPServer struct { - Server *http.Server - data *storage.Repository + Server *http.Server + data *storage.Repository + scheduler *cronruntime.Scheduler } ) -func NewServer(data *storage.Repository, port int) *HTTPServer { +func NewServer(data *storage.Repository, scheduler *cronruntime.Scheduler, port int) *HTTPServer { s := &HTTPServer{ - data: data, + data: data, + scheduler: scheduler, } router := chi.NewRouter() router.NotFound(func(writer http.ResponseWriter, request *http.Request) { @@ -90,6 +94,13 @@ func (s *HTTPServer) ProjectPostHandler(w http.ResponseWriter, r *http.Request) return } + s.scheduler.Remove(pr) + if err := s.scheduler.Add(pr); err != nil { + slog.Error("failed to run project", "err", err) + internalServerError(err, w, r) + return + } + w.WriteHeader(201) } diff --git a/cmd/server/core/git/git.go b/cmd/server/core/git/git.go index 6a15dcf..552a59a 100644 --- a/cmd/server/core/git/git.go +++ b/cmd/server/core/git/git.go @@ -1,6 +1,7 @@ package git import ( + "errors" "fmt" "github.com/go-git/go-git/v6" @@ -12,9 +13,10 @@ import ( type ( Repository struct { - src string - dst string - auth Authentication + src string + dst string + srcAuth Authentication + dstAuth Authentication } Authentication interface { @@ -22,16 +24,42 @@ type ( } TokenAuthentication struct { - username string - token string + token string + } + + BasicAuthentication struct { + username, password string } NoAuthentication struct{} ) +func NewRepository(src, dst string, srcAuth, dstAuth Authentication) Repository { + return Repository{ + src: src, + dst: dst, + srcAuth: srcAuth, + dstAuth: dstAuth, + } +} + +func NewTokenAuthentication(token string) TokenAuthentication { + return TokenAuthentication{ + token: token, + } +} + +func NewBasicAuthentication(username, password string) BasicAuthentication { + return BasicAuthentication{ + username: username, + password: password, + } +} + func Sync(r Repository) error { repo, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{ - URL: r.src, + URL: r.src, + Auth: r.srcAuth.Value(), }) if err != nil { return fmt.Errorf("failed to clone repository from source: %w", err) @@ -50,11 +78,14 @@ func Sync(r Repository) error { err = m.Push(&git.PushOptions{ RemoteName: "mirror", - Auth: r.auth.Value(), + Auth: r.dstAuth.Value(), RefSpecs: []config.RefSpec{"+refs/*:refs/*"}, Force: true, }) if err != nil { + if errors.Is(err, git.NoErrAlreadyUpToDate) { + return nil + } return fmt.Errorf("failed to push to mirror server: %w", err) } @@ -63,11 +94,18 @@ func Sync(r Repository) error { func (a TokenAuthentication) Value() transport.AuthMethod { return &http.BasicAuth{ - Username: a.username, + Username: "git", Password: a.token, } } +func (a BasicAuthentication) Value() transport.AuthMethod { + return &http.BasicAuth{ + Username: a.username, + Password: a.password, + } +} + func (NoAuthentication) Value() transport.AuthMethod { return nil } diff --git a/cmd/server/core/runtime/runtime.go b/cmd/server/core/runtime/runtime.go new file mode 100644 index 0000000..82aef3d --- /dev/null +++ b/cmd/server/core/runtime/runtime.go @@ -0,0 +1,86 @@ +package runtime + +import ( + "fmt" + "log/slog" + "mirror-sync/cmd/server/core/git" + "mirror-sync/pkg/project" + + "github.com/robfig/cron/v3" +) + +type ( + Scheduler struct { + cr *cron.Cron + ids map[string]map[string]cron.EntryID + } +) + +func New(prs []project.Project) (*Scheduler, error) { + s := &Scheduler{ + cr: cron.New(), + ids: make(map[string]map[string]cron.EntryID), + } + + for _, pr := range prs { + if err := s.Add(pr); err != nil { + return nil, err + } + } + + return s, nil +} + +func (s *Scheduler) Add(pr project.Project) error { + s.ids[pr.Name] = make(map[string]cron.EntryID) + for _, repo := range pr.Repositories { + var srcAuth git.Authentication = git.NoAuthentication{} + var dstAuth git.Authentication = git.NoAuthentication{} + if v, ok := repo.Authentications["source"]; ok { + if len(v.Token) > 0 { + srcAuth = git.NewTokenAuthentication(v.Token) + } else if v.Basic != nil { + srcAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password) + } + } + if v, ok := repo.Authentications["mirror"]; ok { + if len(v.Token) > 0 { + dstAuth = git.NewTokenAuthentication(v.Token) + } else if v.Basic != nil { + dstAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password) + } + } + r := git.NewRepository(repo.Source, repo.Destination, srcAuth, dstAuth) + id, err := s.cr.AddFunc(repo.Schedule, func() { + slog.Info(fmt.Sprintf("[%s] starting sync...", repo.Name)) + if err := git.Sync(r); err != nil { + slog.Error(fmt.Sprintf("[%s] failed to sync repository: %s", repo.Name, err)) + return + } + slog.Info(fmt.Sprintf("[%s] synced", repo.Name)) + }) + if err != nil { + return err + } + s.ids[pr.Name][repo.Name] = id + + slog.Info(fmt.Sprintf("[%s] scheduled with '%s'", repo.Name, repo.Schedule)) + } + + return nil +} + +func (s *Scheduler) Remove(pr project.Project) { + if v, ok := s.ids[pr.Name]; ok { + for name, id := range v { + s.cr.Remove(id) + slog.Info(fmt.Sprintf("[%s] remove from being run in the future.", name)) + } + } + delete(s.ids, pr.Name) +} + +// Run the cron scheduler, or no-op if already running. +func (s *Scheduler) Run() { + s.cr.Run() +} diff --git a/cmd/server/core/storage/migrations/001_first_schema.sql b/cmd/server/core/storage/migrations/001_first_schema.sql index 598056b..21822bb 100644 --- a/cmd/server/core/storage/migrations/001_first_schema.sql +++ b/cmd/server/core/storage/migrations/001_first_schema.sql @@ -15,6 +15,14 @@ CREATE TABLE Repositories ( ); CREATE INDEX Repositories_uuid_IDX ON Repositories (uuid); +CREATE TABLE Authentication ( + repository TEXT NOT NULL, + ref TEXT NOT NULL, + username TEXT, + "password" TEXT, + token TEXT +); + -- +goose Down DROP TABLE Projects; DROP TABLE Repositories; \ No newline at end of file diff --git a/cmd/server/core/storage/storage.go b/cmd/server/core/storage/storage.go index d3840af..375a1c7 100644 --- a/cmd/server/core/storage/storage.go +++ b/cmd/server/core/storage/storage.go @@ -117,16 +117,47 @@ func (r *Repository) Create(pr project.Project) error { } // Create repositories entries - stmt, err = tx.Prepare("INSERT INTO Repositories (uuid, name, source, destination, schedule, project) VALUES (?, ?, ?, ?, ?, ?)") + + for _, repo := range pr.Repositories { + if err := r.createRepository(tx, projectUUID, repo); err != nil { + return err + } + } + + return nil +} + +func (r Repository) createRepository(tx *sql.Tx, projectUuid string, repo project.Repository) error { + repoUUID := uuid.NewString() + + stmt, err := tx.Prepare("INSERT INTO Repositories (uuid, name, source, destination, schedule, project) VALUES (?, ?, ?, ?, ?, ?)") if err != nil { return fmt.Errorf("failed to create statement: %s", err) } - for _, repo := range pr.Repositories { - repoUUID := uuid.NewString() + if _, err := stmt.Exec(repoUUID, repo.Name, repo.Source, repo.Destination, repo.Schedule, projectUuid); err != nil { + return fmt.Errorf("failed to execute sql query: %s", err) + } - if _, err := stmt.Exec(repoUUID, repo.Name, repo.Source, repo.Destination, repo.Schedule, projectUUID); err != nil { - return fmt.Errorf("failed to execute sql query: %s", err) + for ref, auth := range repo.Authentications { + if len(auth.Token) > 0 { + stmt, err := tx.Prepare("INSERT INTO Authentication (repository, ref, token) VALUES (?, ?, ?)") + if err != nil { + return fmt.Errorf("failed to create statement: %s", err) + } + + if _, err := stmt.Exec(repoUUID, ref, auth.Token); err != nil { + return fmt.Errorf("failed to execute sql query: %s", err) + } + } else if auth.Basic != nil { + stmt, err := tx.Prepare("INSERT INTO Authentication (repository, ref, username, password) VALUES (?, ?, ?, ?)") + if err != nil { + return fmt.Errorf("failed to create statement: %s", err) + } + + if _, err := stmt.Exec(repoUUID, ref, auth.Basic.Username, auth.Basic.Password); err != nil { + return fmt.Errorf("failed to execute sql query: %s", err) + } } } @@ -185,11 +216,6 @@ func (r *Repository) Update(pr project.Project) error { return fmt.Errorf("failed to get project uuid: %w", err) } - stmt, err := tx.Prepare("UPDATE Repositories SET schedule = ?, source = ?, destination = ? WHERE uuid = ?") - if err != nil { - return fmt.Errorf("failed to create statement: %w", err) - } - // this loop does NOT remove orphan for _, repo := range pr.Repositories { // checks if the repo exists @@ -200,24 +226,52 @@ func (r *Repository) Update(pr project.Project) error { if exists { // if it exists, just update it - uuid, err := r.RepositoryUUID(repo.Name) - if err != nil { - return fmt.Errorf("failed to get uuid from database: %w", err) - } - - if _, err := stmt.Exec(repo.Schedule, repo.Source, repo.Destination, uuid); err != nil { - return fmt.Errorf("failed to update repository entry for %s::'%s'", uuid, repo.Name) + if err := r.updateRepository(tx, repo); err != nil { + return err } } else { // if not, create a new uuid and create the entry - repoUUID := uuid.NewString() - - if _, err := stmt.Exec(repoUUID, repo.Name, repo.Source, repo.Destination, repo.Schedule, projectUUID); err != nil { - return fmt.Errorf("failed to execute sql query: %s", err) + if err := r.createRepository(tx, projectUUID, repo); err != nil { + return err } } - if _, err := stmt.Exec(repo.Schedule, repo.Source, repo.Destination, repo.Name); err != nil { - return fmt.Errorf("failed to update repository entry for '%s'", repo.Name) + } + + return nil +} + +func (r *Repository) updateRepository(tx *sql.Tx, repo project.Repository) error { + uuid, err := r.RepositoryUUID(repo.Name) + if err != nil { + return fmt.Errorf("failed to get uuid from database: %w", err) + } + + stmt, err := tx.Prepare("UPDATE Repositories SET schedule = ?, source = ?, destination = ? WHERE uuid = ?") + if err != nil { + return fmt.Errorf("failed to create statement: %w", err) + } + + if _, err := stmt.Exec(repo.Schedule, repo.Source, repo.Destination, uuid); err != nil { + return fmt.Errorf("failed to update repository entry for %s::'%s'", uuid, repo.Name) + } + + for ref, auth := range repo.Authentications { + if auth.Basic != nil { + stmt, err := tx.Prepare("UPDATE Authentication SET username = ?, password = ?, token = null WHERE repository = ? AND ref = ?") + if err != nil { + return fmt.Errorf("failed to create statement: %w", err) + } + if _, err := stmt.Exec(auth.Basic.Username, auth.Basic.Password, uuid, ref); err != nil { + return fmt.Errorf("failed to execut sql query: %s", err) + } + } else { + stmt, err := tx.Prepare("UPDATE Authentication SET username = null, password = null, token = ? WHERE repository = ? AND ref = ?") + if err != nil { + return fmt.Errorf("failed to create statement: %w", err) + } + if _, err := stmt.Exec(auth.Token, uuid, ref); err != nil { + return fmt.Errorf("failed to execute sql query: %s", err) + } } } @@ -233,7 +287,12 @@ func (r *Repository) List() ([]project.Project, error) { } defer rows.Close() - stmt, err := r.db.Prepare("SELECT name, schedule, source, destination FROM Repositories WHERE project = ?") + stmt, err := r.db.Prepare("SELECT uuid, name, schedule, source, destination FROM Repositories WHERE project = ?") + if err != nil { + return nil, fmt.Errorf("invalid syntax: %w", err) + } + + authStmt, err := r.db.Prepare("SELECT ref, username, password, token FROM Authentication WHERE repository = ?") if err != nil { return nil, fmt.Errorf("invalid syntax: %w", err) } @@ -250,11 +309,44 @@ func (r *Repository) List() ([]project.Project, error) { } for repoRows.Next() { + var uuid string var repo project.Repository - if err := repoRows.Scan(&repo.Name, &repo.Schedule, &repo.Source, &repo.Destination); err != nil { + if err := repoRows.Scan(&uuid, &repo.Name, &repo.Schedule, &repo.Source, &repo.Destination); err != nil { repoRows.Close() return nil, fmt.Errorf("failed to scan repository entry: %w", err) } + + authRows, err := authStmt.Query(uuid) + if err != nil { + repoRows.Close() + return nil, fmt.Errorf("failed to query repositories for the project %s: %w", prUUID, err) + } + + auth := make(map[string]project.AuthenticationSettings) + for authRows.Next() { + var ref string + var username, password, token *string + if err := authRows.Scan(&ref, &username, &password, &token); err != nil { + authRows.Close() + repoRows.Close() + return nil, fmt.Errorf("failed to scan authentication entry: %s", err) + } + if token != nil { + auth[ref] = project.AuthenticationSettings{ + Token: *token, + } + } else if username != nil { + auth[ref] = project.AuthenticationSettings{ + Basic: &project.BasicAuthenticationSettings{ + Username: *username, + Password: *password, + }, + } + } + } + + authRows.Close() + repo.Authentications = auth pr.Repositories = append(pr.Repositories, repo) } diff --git a/cmd/server/main.go b/cmd/server/main.go index 81a6f52..59fac53 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "mirror-sync/cmd/server/api" + cronruntime "mirror-sync/cmd/server/core/runtime" "mirror-sync/cmd/server/core/storage" "mirror-sync/pkg/constants" "os" @@ -29,7 +30,24 @@ func main() { os.Exit(1) } - s := api.NewServer(data, 8080) + // runtime + prs, err := data.List() + if err != nil { + fmt.Fprintln(os.Stderr, "failed to start server:", err.Error()) + os.Exit(1) + } + + scheduler, err := cronruntime.New(prs) + if err != nil { + fmt.Fprintln(os.Stderr, "failed to start server:", err.Error()) + os.Exit(1) + } + + go scheduler.Run() + slog.Info("daemon scheduler is running") + + // api + s := api.NewServer(data, scheduler, 8080) slog.Info("daemon listening to :8080") if err := s.Server.ListenAndServe(); err != nil { diff --git a/pkg/project/file.go b/pkg/project/file.go index b468c7c..99911e8 100644 --- a/pkg/project/file.go +++ b/pkg/project/file.go @@ -1,6 +1,7 @@ package project import ( + "encoding/json" "errors" "fmt" "os" @@ -30,8 +31,23 @@ type ( } GitStorage struct { - Source string `yaml:"source"` - Mirror string `yaml:"mirror"` + Source StorageSettings `yaml:"source"` + Mirror StorageSettings `yaml:"mirror"` + } + + StorageSettings struct { + URL string `yaml:"url"` + Authentication AuthenticationDescriptor `yaml:"authentication"` + } + + AuthenticationDescriptor struct { + Basic BasicAuthenticationDescriptor `yaml:"basic"` + Token string `yaml:"token"` + } + + BasicAuthenticationDescriptor struct { + Username string `yaml:"username"` + Password string `yaml:"password"` } ) @@ -42,11 +58,6 @@ var ( ) func LoadCurrent() (Project, error) { - wd, err := os.Getwd() - if err != nil { - return Project{}, fmt.Errorf("%w: cannot get current working directory path: %s", ErrOS, err) - } - f, err := os.OpenFile("./git-compose.yaml", os.O_RDONLY, 0) if err != nil { return Project{}, fmt.Errorf("%w: %s", ErrIO, err) @@ -59,6 +70,24 @@ func LoadCurrent() (Project, error) { return Project{}, fmt.Errorf("%w: %s", ErrParsing, err) } + return decode(mainFile) +} + +func LoadBytes(b []byte) (Project, error) { + var mainFile MainFile + if err := json.Unmarshal(b, &mainFile); err != nil { + return Project{}, fmt.Errorf("%w: %s", ErrParsing, err) + } + + return decode(mainFile) +} + +func decode(mainFile MainFile) (Project, error) { + wd, err := os.Getwd() + if err != nil { + return Project{}, fmt.Errorf("%w: cannot get current working directory path: %s", ErrOS, err) + } + if err := checkConfig(mainFile); err != nil { return Project{}, fmt.Errorf("failed to validate configuration: %w", err) } @@ -85,25 +114,52 @@ func LoadCurrent() (Project, error) { } for repoName, repo := range mainFile.Repositories { - pr.Repositories = append(pr.Repositories, Repository{ + r := Repository{ Name: fmt.Sprintf("%s-%s", pr.Name, strings.ToLower(repoName)), - Source: repo.Storage.Source, - Destination: repo.Storage.Mirror, + Source: repo.Storage.Source.URL, + Destination: repo.Storage.Mirror.URL, Schedule: repo.Schedule, - }) + } + + r.Authentications = make(map[string]AuthenticationSettings) + setAuthentication(r.Authentications, "source", repo.Storage.Source.Authentication) + setAuthentication(r.Authentications, "mirror", repo.Storage.Mirror.Authentication) + + pr.Repositories = append(pr.Repositories, r) } return pr, nil } +func setAuthentication(m map[string]AuthenticationSettings, key string, auth AuthenticationDescriptor) { + if len(auth.Token) > 0 { + m[key] = AuthenticationSettings{ + Token: auth.Token, + } + } else if len(auth.Basic.Username) > 0 { + m[key] = AuthenticationSettings{ + Basic: &BasicAuthenticationSettings{ + Username: auth.Basic.Username, + Password: auth.Basic.Password, + }, + } + } +} + func checkConfig(mf MainFile) error { for _, r := range mf.Repositories { - if len(strings.TrimSpace(r.Storage.Source)) == 0 { + if len(strings.TrimSpace(r.Storage.Source.URL)) == 0 { return fmt.Errorf("source is empty") } - if len(strings.TrimSpace(r.Storage.Mirror)) == 0 { + if err := checkAuthenticationConfig(r.Storage.Source); err != nil { + return err + } + if len(strings.TrimSpace(r.Storage.Mirror.URL)) == 0 { return fmt.Errorf("mirror is empty") } + if err := checkAuthenticationConfig(r.Storage.Mirror); err != nil { + return err + } if len(strings.TrimSpace(r.Schedule)) == 0 { return fmt.Errorf("schedule is empty") } @@ -114,3 +170,10 @@ func checkConfig(mf MainFile) error { return nil } + +func checkAuthenticationConfig(ss StorageSettings) error { + if len(ss.Authentication.Token) > 0 && (len(ss.Authentication.Basic.Username) > 0 || len(ss.Authentication.Basic.Password) > 0) { + return fmt.Errorf("cannot use token and basic authentication in the same repository") + } + return nil +} diff --git a/pkg/project/project.go b/pkg/project/project.go index 3946103..a9ac869 100644 --- a/pkg/project/project.go +++ b/pkg/project/project.go @@ -8,9 +8,20 @@ type ( } Repository struct { - Name string `json:"name"` - Schedule string `json:"schedule"` - Source string `json:"source"` - Destination string `json:"destination"` + Name string `json:"name"` + Schedule string `json:"schedule"` + Source string `json:"source"` + Destination string `json:"destination"` + Authentications map[string]AuthenticationSettings `json:"authentications"` + } + + AuthenticationSettings struct { + Basic *BasicAuthenticationSettings `json:"basic,omitempty"` + Token string `json:"token,omitempty"` + } + + BasicAuthenticationSettings struct { + Username string `json:"username"` + Password string `json:"password"` } )