thelilfrog/mirror-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Aurélie DELHAIE
2025-11-01 20:53:38 +01:00
parent aa8851aca0
commit 6e4452ddb9
12 changed files with 457 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/cli/commands/apply/apply.go
View File

@@ -4,15 +4,18 @@ import (
"context"
"flag"
"fmt"
"mirror-sync/cmd/cli/config"
"mirror-sync/pkg/client"
"mirror-sync/pkg/project"
"os"
"path/filepath"
"github.com/google/subcommands"
)
type (
ApplyCmd struct {
projectName string
}
)
@@ -28,10 +31,22 @@ Options:
}
func (p *ApplyCmd) SetFlags(f *flag.FlagSet) {
wd, err := os.Getwd()
if err != nil {
panic(err)
}
f.StringVar(&p.projectName, "project-name", filepath.Base(wd), "set the project name")
}
func (p *ApplyCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
projectConfig, err := project.LoadCurrent()
clientConfig := config.Load()
defaultValues := project.DefaultValues{
DaemonURL: clientConfig.Deamon.URL,
ProjectName: p.projectName,
}
projectConfig, err := project.LoadCurrent(defaultValues)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure

2
cmd/cli/commands/list/list.go
View File

@@ -56,6 +56,6 @@ func print(pr project.Project) {
fmt.Println("------------------")
for _, repo := range pr.Repositories {
fmt.Printf("%-20s | %s -> %s | %s\n", repo.Name, repo.Source, repo.Destination, repo.Schedule)
fmt.Printf("%s | %-20s | %s -> %s | %s\n", repo.UUID, repo.Name, repo.Source, repo.Destination, repo.Schedule)
}
}

60
cmd/cli/commands/remove/remove.go Normal file
View File

@@ -0,0 +1,60 @@
package remove
import (
"context"
"flag"
"fmt"
"mirror-sync/cmd/cli/config"
"mirror-sync/pkg/client"
"mirror-sync/pkg/project"
"os"
"path/filepath"
"github.com/google/subcommands"
)
type (
DownCmd struct {
projectName string
}
)
func (*DownCmd) Name() string { return "down" }
func (*DownCmd) Synopsis() string { return "remove the current project schedule" }
func (*DownCmd) Usage() string {
return `Usage: mirror-sync down
remove the current project
`
}
func (p *DownCmd) SetFlags(f *flag.FlagSet) {
wd, err := os.Getwd()
if err != nil {
panic(err)
}
f.StringVar(&p.projectName, "project-name", filepath.Base(wd), "set the project name")
}
func (p *DownCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
clientConfig := config.Load()
defaultValues := project.DefaultValues{
DaemonURL: clientConfig.Deamon.URL,
ProjectName: p.projectName,
}
projectConfig, err := project.LoadCurrent(defaultValues)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
cli := client.New(projectConfig.ServerURL)
if err := cli.Remove(projectConfig); err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
return subcommands.ExitSuccess
}

60
cmd/cli/commands/run/run.go Normal file
View File

@@ -0,0 +1,60 @@
package run
import (
"context"
"flag"
"fmt"
"mirror-sync/cmd/cli/config"
"mirror-sync/pkg/client"
"mirror-sync/pkg/project"
"os"
"path/filepath"
"github.com/google/subcommands"
)
type (
RunCmd struct {
}
)
func (*RunCmd) Name() string { return "run" }
func (*RunCmd) Synopsis() string { return "run the current project schedule" }
func (*RunCmd) Usage() string {
return `Usage: mirror-sync run
run the current project
`
}
func (p *RunCmd) SetFlags(f *flag.FlagSet) {
}
func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
clientConfig := config.Load()
wd, err := os.Getwd()
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
defaultValues := project.DefaultValues{
DaemonURL: clientConfig.Deamon.URL,
ProjectName: filepath.Base(wd),
}
projectConfig, err := project.LoadCurrent(defaultValues)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
cli := client.New(projectConfig.ServerURL)
if err := cli.RunOne(projectConfig); err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
return subcommands.ExitSuccess
}

30
cmd/cli/commands/version/version.go
View File

@@ -4,7 +4,10 @@ import (
"context"
"flag"
"fmt"
"mirror-sync/cmd/cli/config"
"mirror-sync/pkg/client"
"mirror-sync/pkg/constants"
"os"
"runtime"
"strconv"
@@ -31,14 +34,31 @@ func (p *VersionCmd) SetFlags(f *flag.FlagSet) {
}
func (p *VersionCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
local()
return subcommands.ExitSuccess
}
func local() {
fmt.Println("Client: mirror-sync cli")
fmt.Println(" Version: " + constants.Version)
fmt.Println(" API version: " + strconv.Itoa(constants.ApiVersion))
fmt.Println(" Go version: " + runtime.Version())
fmt.Println(" OS/Arch: " + runtime.GOOS + "/" + runtime.GOARCH)
clientConfig := config.Load()
cli := client.New(clientConfig.Deamon.URL)
systemInfoDaemon, err := cli.Version()
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
fmt.Println()
fmt.Println("Daemon:")
fmt.Println(" Version: " + systemInfoDaemon.Version)
fmt.Println(" API version: " + strconv.Itoa(systemInfoDaemon.APIVersion))
fmt.Println(" Go version: " + systemInfoDaemon.GoVersion)
fmt.Println(" OS/Arch: " + systemInfoDaemon.OSName + "/" + systemInfoDaemon.OSArchitecture)
return subcommands.ExitSuccess
}
func local() {
}

4
cmd/cli/main.go
View File

@@ -6,6 +6,8 @@ import (
"fmt"
"mirror-sync/cmd/cli/commands/apply"
"mirror-sync/cmd/cli/commands/list"
"mirror-sync/cmd/cli/commands/remove"
"mirror-sync/cmd/cli/commands/run"
"mirror-sync/cmd/cli/commands/version"
"os"
@@ -25,6 +27,8 @@ func main() {
subcommands.Register(&version.VersionCmd{}, "help")
subcommands.Register(&apply.ApplyCmd{}, "projects")
subcommands.Register(&run.RunCmd{}, "projects")
subcommands.Register(&remove.DownCmd{}, "projects")
subcommands.Register(&list.ListCmd{}, "management")

56
cmd/server/api/api.go
View File

@@ -37,6 +37,7 @@ func NewServer(data *storage.Repository, scheduler *cronruntime.Scheduler, port
router.MethodNotAllowed(func(writer http.ResponseWriter, request *http.Request) {
methodNotAllowed(writer, request)
})
chi.RegisterMethod("EXECUTE")
router.Use(middleware.Logger)
router.Use(recoverMiddleware)
router.Use(middleware.GetHead)
@@ -46,11 +47,12 @@ func NewServer(data *storage.Repository, scheduler *cronruntime.Scheduler, port
routerAPI.Route("/v1", func(r chi.Router) {
// Get information about the server
r.Get("/version", s.Information)
r.MethodFunc("EXECUTE", "/run", s.RunProjectHandler)
r.Route("/projects", func(r chi.Router) {
r.Get("/all", s.ProjectsHandler)
r.Get("/{name}", func(w http.ResponseWriter, r *http.Request) {})
r.Post("/{name}", s.ProjectPostHandler)
r.Delete("/{name}", func(w http.ResponseWriter, r *http.Request) {})
r.Get("/all", s.ProjectsGetHandler)
r.Get("/", func(w http.ResponseWriter, r *http.Request) {})
r.Post("/", s.ProjectPostHandler)
r.Delete("/", s.ProjectDeleteHandler)
})
})
})
@@ -73,13 +75,6 @@ func (s *HTTPServer) Information(w http.ResponseWriter, r *http.Request) {
}
func (s *HTTPServer) ProjectPostHandler(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
if len(name) == 0 {
badRequest("project name cannot be empty", w, r)
return
}
var pr project.Project
d := json.NewDecoder(r.Body)
if err := d.Decode(&pr); err != nil {
@@ -104,7 +99,26 @@ func (s *HTTPServer) ProjectPostHandler(w http.ResponseWriter, r *http.Request)
w.WriteHeader(201)
}
func (s *HTTPServer) ProjectsHandler(w http.ResponseWriter, r *http.Request) {
func (s *HTTPServer) ProjectDeleteHandler(w http.ResponseWriter, r *http.Request) {
var pr project.Project
d := json.NewDecoder(r.Body)
if err := d.Decode(&pr); err != nil {
slog.Error("failed to parse project description", "err", err)
internalServerError(err, w, r)
return
}
s.scheduler.Remove(pr)
if err := s.data.Remove(pr); err != nil {
slog.Error("failed to remove project", "err", err, "uuid", pr.UUID)
internalServerError(err, w, r)
return
}
w.WriteHeader(204)
}
func (s *HTTPServer) ProjectsGetHandler(w http.ResponseWriter, r *http.Request) {
prs, err := s.data.List()
if err != nil {
slog.Error("failed to fetch all the projects from the database", "err", err)
@@ -114,3 +128,21 @@ func (s *HTTPServer) ProjectsHandler(w http.ResponseWriter, r *http.Request) {
ok(prs, w, r)
}
func (s *HTTPServer) RunProjectHandler(w http.ResponseWriter, r *http.Request) {
var pr project.Project
d := json.NewDecoder(r.Body)
if err := d.Decode(&pr); err != nil {
slog.Error("failed to parse project description", "err", err)
internalServerError(err, w, r)
return
}
if err := s.scheduler.RunOnce(pr); err != nil {
slog.Error("failed to run the project", "err", err)
internalServerError(err, w, r)
return
}
ok("ok", w, r)
}

53
cmd/server/core/runtime/runtime.go
View File

@@ -34,26 +34,10 @@ func New(prs []project.Project) (*Scheduler, error) {
func (s *Scheduler) Add(pr project.Project) error {
s.ids[pr.Name] = make(map[string]cron.EntryID)
for _, repo := range pr.Repositories {
var srcAuth git.Authentication = git.NoAuthentication{}
var dstAuth git.Authentication = git.NoAuthentication{}
if v, ok := repo.Authentications["source"]; ok {
if len(v.Token) > 0 {
srcAuth = git.NewTokenAuthentication(v.Token)
} else if v.Basic != nil {
srcAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
}
}
if v, ok := repo.Authentications["mirror"]; ok {
if len(v.Token) > 0 {
dstAuth = git.NewTokenAuthentication(v.Token)
} else if v.Basic != nil {
dstAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
}
}
r := git.NewRepository(repo.Source, repo.Destination, srcAuth, dstAuth)
gr := s.prepare(repo)
id, err := s.cr.AddFunc(repo.Schedule, func() {
slog.Info(fmt.Sprintf("[%s] starting sync...", repo.Name))
if err := git.Sync(r); err != nil {
if err := git.Sync(gr); err != nil {
slog.Error(fmt.Sprintf("[%s] failed to sync repository: %s", repo.Name, err))
return
}
@@ -80,6 +64,39 @@ func (s *Scheduler) Remove(pr project.Project) {
delete(s.ids, pr.Name)
}
func (s *Scheduler) RunOnce(pr project.Project) error {
for _, repo := range pr.Repositories {
gr := s.prepare(repo)
slog.Info(fmt.Sprintf("[%s] starting sync...", repo.Name))
if err := git.Sync(gr); err != nil {
slog.Error(fmt.Sprintf("[%s] failed to sync repository: %s", repo.Name, err))
continue
}
slog.Info(fmt.Sprintf("[%s] synced", repo.Name))
}
return nil
}
func (s *Scheduler) prepare(repo project.Repository) git.Repository {
var srcAuth git.Authentication = git.NoAuthentication{}
var dstAuth git.Authentication = git.NoAuthentication{}
if v, ok := repo.Authentications["source"]; ok {
if len(v.Token) > 0 {
srcAuth = git.NewTokenAuthentication(v.Token)
} else if v.Basic != nil {
srcAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
}
}
if v, ok := repo.Authentications["mirror"]; ok {
if len(v.Token) > 0 {
dstAuth = git.NewTokenAuthentication(v.Token)
} else if v.Basic != nil {
dstAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
}
}
return git.NewRepository(repo.Source, repo.Destination, srcAuth, dstAuth)
}
// Run the cron scheduler, or no-op if already running.
func (s *Scheduler) Run() {
s.cr.Run()

163
cmd/server/core/storage/storage.go
View File

@@ -279,6 +279,43 @@ func (r *Repository) updateRepository(tx *sql.Tx, repo project.Repository) error
return nil
}
func (r *Repository) Remove(pr project.Project) error {
tx, err := r.db.Begin()
if err != nil {
return fmt.Errorf("failed to create transaction: %s", err)
}
defer func() {
if err != nil {
tx.Rollback()
return
}
tx.Commit()
}()
uuid, err := r.ProjectUUID(pr.Name)
repos, err := r.listRepositories(uuid)
if err != nil {
return err
}
for _, repo := range repos {
if _, err := tx.Exec("DELETE FROM Authentication WHERE repository = ?", repo.UUID); err != nil {
return fmt.Errorf("failed to delete the authentication entries from the database: %s", err)
}
}
if _, err := tx.Exec("DELETE FROM Repositories WHERE project = ?", uuid); err != nil {
return fmt.Errorf("failed to delete the repositories from the database: %s", err)
}
if _, err := tx.Exec("DELETE FROM Projects WHERE uuid = ?", uuid); err != nil {
return fmt.Errorf("failed to delete the project from the database: %s", err)
}
return nil
}
func (r *Repository) List() ([]project.Project, error) {
var prs []project.Project
@@ -288,72 +325,88 @@ func (r *Repository) List() ([]project.Project, error) {
}
defer rows.Close()
stmt, err := r.db.Prepare("SELECT uuid, name, schedule, source, destination FROM Repositories WHERE project = ?")
if err != nil {
return nil, fmt.Errorf("invalid syntax: %w", err)
}
authStmt, err := r.db.Prepare("SELECT ref, username, password, token FROM Authentication WHERE repository = ?")
if err != nil {
return nil, fmt.Errorf("invalid syntax: %w", err)
}
for rows.Next() {
var pr project.Project
var prUUID string
if err := rows.Scan(&prUUID, &pr.Name); err != nil {
if err := rows.Scan(&pr.UUID, &pr.Name); err != nil {
return nil, fmt.Errorf("failed to scan project name: %w", err)
}
repoRows, err := stmt.Query(prUUID)
repos, err := r.listRepositories(pr.UUID)
if err != nil {
return nil, fmt.Errorf("failed to query repositories for the project %s: %w", prUUID, err)
return nil, err
}
for repoRows.Next() {
var uuid string
var repo project.Repository
if err := repoRows.Scan(&uuid, &repo.Name, &repo.Schedule, &repo.Source, &repo.Destination); err != nil {
repoRows.Close()
return nil, fmt.Errorf("failed to scan repository entry: %w", err)
}
pr.Repositories = repos
authRows, err := authStmt.Query(uuid)
if err != nil {
repoRows.Close()
return nil, fmt.Errorf("failed to query repositories for the project %s: %w", prUUID, err)
}
auth := make(map[string]project.AuthenticationSettings)
for authRows.Next() {
var ref string
var username, password, token *string
if err := authRows.Scan(&ref, &username, &password, &token); err != nil {
authRows.Close()
repoRows.Close()
return nil, fmt.Errorf("failed to scan authentication entry: %s", err)
}
if token != nil {
auth[ref] = project.AuthenticationSettings{
Token: *token,
}
} else if username != nil {
auth[ref] = project.AuthenticationSettings{
Basic: &project.BasicAuthenticationSettings{
Username: *username,
Password: *password,
},
}
}
}
authRows.Close()
repo.Authentications = auth
pr.Repositories = append(pr.Repositories, repo)
}
repoRows.Close()
prs = append(prs, pr)
}
return prs, nil
}
func (r *Repository) listRepositories(projectUUID string) ([]project.Repository, error) {
stmt, err := r.db.Prepare("SELECT uuid, name, schedule, source, destination FROM Repositories WHERE project = ?")
if err != nil {
return nil, fmt.Errorf("invalid syntax: %w", err)
}
rows, err := stmt.Query(projectUUID)
if err != nil {
return nil, fmt.Errorf("failed to query repositories for the project %s: %w", projectUUID, err)
}
defer rows.Close()
var repositories []project.Repository
for rows.Next() {
var repo project.Repository
if err := rows.Scan(&repo.UUID, &repo.Name, &repo.Schedule, &repo.Source, &repo.Destination); err != nil {
return nil, fmt.Errorf("failed to scan repository entry: %w", err)
}
auth, err := r.listAuthentications(repo.UUID)
if err != nil {
return nil, err
}
repo.Authentications = auth
repositories = append(repositories, repo)
}
return repositories, nil
}
func (r *Repository) listAuthentications(repositoryUUID string) (map[string]project.AuthenticationSettings, error) {
stmt, err := r.db.Prepare("SELECT ref, username, password, token FROM Authentication WHERE repository = ?")
if err != nil {
return nil, fmt.Errorf("invalid syntax: %w", err)
}
rows, err := stmt.Query(repositoryUUID)
if err != nil {
return nil, fmt.Errorf("failed to query repositories for the project %s: %w", repositoryUUID, err)
}
defer rows.Close()
res := make(map[string]project.AuthenticationSettings)
for rows.Next() {
var ref string
var username, password, token *string
if err := rows.Scan(&ref, &username, &password, &token); err != nil {
return nil, fmt.Errorf("failed to scan authentication entry: %s", err)
}
if token != nil {
res[ref] = project.AuthenticationSettings{
Token: *token,
}
} else if username != nil {
res[ref] = project.AuthenticationSettings{
Basic: &project.BasicAuthenticationSettings{
Username: *username,
Password: *password,
},
}
}
}
return res, nil
}