thelilfrog/mirror-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Aurélie DELHAIE
2025-11-01 20:53:38 +01:00
parent aa8851aca0
commit 6e4452ddb9
12 changed files with 457 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/cli/commands/apply/apply.go
View File

@@ -4,15 +4,18 @@ import (
"context"
"flag"
"fmt"
"mirror-sync/cmd/cli/config"
"mirror-sync/pkg/client"
"mirror-sync/pkg/project"
"os"
"path/filepath"
"github.com/google/subcommands"
)
type (
ApplyCmd struct {
projectName string
}
)
@@ -28,10 +31,22 @@ Options:
}
func (p *ApplyCmd) SetFlags(f *flag.FlagSet) {
wd, err := os.Getwd()
if err != nil {
panic(err)
}
f.StringVar(&p.projectName, "project-name", filepath.Base(wd), "set the project name")
}
func (p *ApplyCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
projectConfig, err := project.LoadCurrent()
clientConfig := config.Load()
defaultValues := project.DefaultValues{
DaemonURL: clientConfig.Deamon.URL,
ProjectName: p.projectName,
}
projectConfig, err := project.LoadCurrent(defaultValues)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure

2
cmd/cli/commands/list/list.go
View File

@@ -56,6 +56,6 @@ func print(pr project.Project) {
fmt.Println("------------------")
for _, repo := range pr.Repositories {
fmt.Printf("%-20s | %s -> %s | %s\n", repo.Name, repo.Source, repo.Destination, repo.Schedule)
fmt.Printf("%s | %-20s | %s -> %s | %s\n", repo.UUID, repo.Name, repo.Source, repo.Destination, repo.Schedule)
}
}

60
cmd/cli/commands/remove/remove.go Normal file
View File

@@ -0,0 +1,60 @@
package remove
import (
"context"
"flag"
"fmt"
"mirror-sync/cmd/cli/config"
"mirror-sync/pkg/client"
"mirror-sync/pkg/project"
"os"
"path/filepath"
"github.com/google/subcommands"
)
type (
DownCmd struct {
projectName string
}
)
func (*DownCmd) Name() string { return "down" }
func (*DownCmd) Synopsis() string { return "remove the current project schedule" }
func (*DownCmd) Usage() string {
return `Usage: mirror-sync down
remove the current project
`
}
func (p *DownCmd) SetFlags(f *flag.FlagSet) {
wd, err := os.Getwd()
if err != nil {
panic(err)
}
f.StringVar(&p.projectName, "project-name", filepath.Base(wd), "set the project name")
}
func (p *DownCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
clientConfig := config.Load()
defaultValues := project.DefaultValues{
DaemonURL: clientConfig.Deamon.URL,
ProjectName: p.projectName,
}
projectConfig, err := project.LoadCurrent(defaultValues)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
cli := client.New(projectConfig.ServerURL)
if err := cli.Remove(projectConfig); err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
return subcommands.ExitSuccess
}

60
cmd/cli/commands/run/run.go Normal file
View File

@@ -0,0 +1,60 @@
package run
import (
"context"
"flag"
"fmt"
"mirror-sync/cmd/cli/config"
"mirror-sync/pkg/client"
"mirror-sync/pkg/project"
"os"
"path/filepath"
"github.com/google/subcommands"
)
type (
RunCmd struct {
}
)
func (*RunCmd) Name() string { return "run" }
func (*RunCmd) Synopsis() string { return "run the current project schedule" }
func (*RunCmd) Usage() string {
return `Usage: mirror-sync run
run the current project
`
}
func (p *RunCmd) SetFlags(f *flag.FlagSet) {
}
func (p *RunCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
clientConfig := config.Load()
wd, err := os.Getwd()
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
defaultValues := project.DefaultValues{
DaemonURL: clientConfig.Deamon.URL,
ProjectName: filepath.Base(wd),
}
projectConfig, err := project.LoadCurrent(defaultValues)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
cli := client.New(projectConfig.ServerURL)
if err := cli.RunOne(projectConfig); err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
return subcommands.ExitSuccess
}

30
cmd/cli/commands/version/version.go
View File

@@ -4,7 +4,10 @@ import (
"context"
"flag"
"fmt"
"mirror-sync/cmd/cli/config"
"mirror-sync/pkg/client"
"mirror-sync/pkg/constants"
"os"
"runtime"
"strconv"
@@ -31,14 +34,31 @@ func (p *VersionCmd) SetFlags(f *flag.FlagSet) {
}
func (p *VersionCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
local()
return subcommands.ExitSuccess
}
func local() {
fmt.Println("Client: mirror-sync cli")
fmt.Println(" Version: " + constants.Version)
fmt.Println(" API version: " + strconv.Itoa(constants.ApiVersion))
fmt.Println(" Go version: " + runtime.Version())
fmt.Println(" OS/Arch: " + runtime.GOOS + "/" + runtime.GOARCH)
clientConfig := config.Load()
cli := client.New(clientConfig.Deamon.URL)
systemInfoDaemon, err := cli.Version()
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return subcommands.ExitFailure
}
fmt.Println()
fmt.Println("Daemon:")
fmt.Println(" Version: " + systemInfoDaemon.Version)
fmt.Println(" API version: " + strconv.Itoa(systemInfoDaemon.APIVersion))
fmt.Println(" Go version: " + systemInfoDaemon.GoVersion)
fmt.Println(" OS/Arch: " + systemInfoDaemon.OSName + "/" + systemInfoDaemon.OSArchitecture)
return subcommands.ExitSuccess
}
func local() {
}

4
cmd/cli/main.go
View File

@@ -6,6 +6,8 @@ import (
"fmt"
"mirror-sync/cmd/cli/commands/apply"
"mirror-sync/cmd/cli/commands/list"
"mirror-sync/cmd/cli/commands/remove"
"mirror-sync/cmd/cli/commands/run"
"mirror-sync/cmd/cli/commands/version"
"os"
@@ -25,6 +27,8 @@ func main() {
subcommands.Register(&version.VersionCmd{}, "help")
subcommands.Register(&apply.ApplyCmd{}, "projects")
subcommands.Register(&run.RunCmd{}, "projects")
subcommands.Register(&remove.DownCmd{}, "projects")
subcommands.Register(&list.ListCmd{}, "management")

56
cmd/server/api/api.go
View File

@@ -37,6 +37,7 @@ func NewServer(data *storage.Repository, scheduler *cronruntime.Scheduler, port
router.MethodNotAllowed(func(writer http.ResponseWriter, request *http.Request) {
methodNotAllowed(writer, request)
})
chi.RegisterMethod("EXECUTE")
router.Use(middleware.Logger)
router.Use(recoverMiddleware)
router.Use(middleware.GetHead)
@@ -46,11 +47,12 @@ func NewServer(data *storage.Repository, scheduler *cronruntime.Scheduler, port
routerAPI.Route("/v1", func(r chi.Router) {
// Get information about the server
r.Get("/version", s.Information)
r.MethodFunc("EXECUTE", "/run", s.RunProjectHandler)
r.Route("/projects", func(r chi.Router) {
r.Get("/all", s.ProjectsHandler)
r.Get("/{name}", func(w http.ResponseWriter, r *http.Request) {})
r.Post("/{name}", s.ProjectPostHandler)
r.Delete("/{name}", func(w http.ResponseWriter, r *http.Request) {})
r.Get("/all", s.ProjectsGetHandler)
r.Get("/", func(w http.ResponseWriter, r *http.Request) {})
r.Post("/", s.ProjectPostHandler)
r.Delete("/", s.ProjectDeleteHandler)
})
})
})
@@ -73,13 +75,6 @@ func (s *HTTPServer) Information(w http.ResponseWriter, r *http.Request) {
}
func (s *HTTPServer) ProjectPostHandler(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
if len(name) == 0 {
badRequest("project name cannot be empty", w, r)
return
}
var pr project.Project
d := json.NewDecoder(r.Body)
if err := d.Decode(&pr); err != nil {
@@ -104,7 +99,26 @@ func (s *HTTPServer) ProjectPostHandler(w http.ResponseWriter, r *http.Request)
w.WriteHeader(201)
}
func (s *HTTPServer) ProjectsHandler(w http.ResponseWriter, r *http.Request) {
func (s *HTTPServer) ProjectDeleteHandler(w http.ResponseWriter, r *http.Request) {
var pr project.Project
d := json.NewDecoder(r.Body)
if err := d.Decode(&pr); err != nil {
slog.Error("failed to parse project description", "err", err)
internalServerError(err, w, r)
return
}
s.scheduler.Remove(pr)
if err := s.data.Remove(pr); err != nil {
slog.Error("failed to remove project", "err", err, "uuid", pr.UUID)
internalServerError(err, w, r)
return
}
w.WriteHeader(204)
}
func (s *HTTPServer) ProjectsGetHandler(w http.ResponseWriter, r *http.Request) {
prs, err := s.data.List()
if err != nil {
slog.Error("failed to fetch all the projects from the database", "err", err)
@@ -114,3 +128,21 @@ func (s *HTTPServer) ProjectsHandler(w http.ResponseWriter, r *http.Request) {
ok(prs, w, r)
}
func (s *HTTPServer) RunProjectHandler(w http.ResponseWriter, r *http.Request) {
var pr project.Project
d := json.NewDecoder(r.Body)
if err := d.Decode(&pr); err != nil {
slog.Error("failed to parse project description", "err", err)
internalServerError(err, w, r)
return
}
if err := s.scheduler.RunOnce(pr); err != nil {
slog.Error("failed to run the project", "err", err)
internalServerError(err, w, r)
return
}
ok("ok", w, r)
}

53
cmd/server/core/runtime/runtime.go
View File

@@ -34,26 +34,10 @@ func New(prs []project.Project) (*Scheduler, error) {
func (s *Scheduler) Add(pr project.Project) error {
s.ids[pr.Name] = make(map[string]cron.EntryID)
for _, repo := range pr.Repositories {
var srcAuth git.Authentication = git.NoAuthentication{}
var dstAuth git.Authentication = git.NoAuthentication{}
if v, ok := repo.Authentications["source"]; ok {
if len(v.Token) > 0 {
srcAuth = git.NewTokenAuthentication(v.Token)
} else if v.Basic != nil {
srcAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
}
}
if v, ok := repo.Authentications["mirror"]; ok {
if len(v.Token) > 0 {
dstAuth = git.NewTokenAuthentication(v.Token)
} else if v.Basic != nil {
dstAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
}
}
r := git.NewRepository(repo.Source, repo.Destination, srcAuth, dstAuth)
gr := s.prepare(repo)
id, err := s.cr.AddFunc(repo.Schedule, func() {
slog.Info(fmt.Sprintf("[%s] starting sync...", repo.Name))
if err := git.Sync(r); err != nil {
if err := git.Sync(gr); err != nil {
slog.Error(fmt.Sprintf("[%s] failed to sync repository: %s", repo.Name, err))
return
}
@@ -80,6 +64,39 @@ func (s *Scheduler) Remove(pr project.Project) {
delete(s.ids, pr.Name)
}
func (s *Scheduler) RunOnce(pr project.Project) error {
for _, repo := range pr.Repositories {
gr := s.prepare(repo)
slog.Info(fmt.Sprintf("[%s] starting sync...", repo.Name))
if err := git.Sync(gr); err != nil {
slog.Error(fmt.Sprintf("[%s] failed to sync repository: %s", repo.Name, err))
continue
}
slog.Info(fmt.Sprintf("[%s] synced", repo.Name))
}
return nil
}
func (s *Scheduler) prepare(repo project.Repository) git.Repository {
var srcAuth git.Authentication = git.NoAuthentication{}
var dstAuth git.Authentication = git.NoAuthentication{}
if v, ok := repo.Authentications["source"]; ok {
if len(v.Token) > 0 {
srcAuth = git.NewTokenAuthentication(v.Token)
} else if v.Basic != nil {
srcAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
}
}
if v, ok := repo.Authentications["mirror"]; ok {
if len(v.Token) > 0 {
dstAuth = git.NewTokenAuthentication(v.Token)
} else if v.Basic != nil {
dstAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
}
}
return git.NewRepository(repo.Source, repo.Destination, srcAuth, dstAuth)
}
// Run the cron scheduler, or no-op if already running.
func (s *Scheduler) Run() {
s.cr.Run()

127
cmd/server/core/storage/storage.go
View File

@@ -279,6 +279,43 @@ func (r *Repository) updateRepository(tx *sql.Tx, repo project.Repository) error
return nil
}
func (r *Repository) Remove(pr project.Project) error {
tx, err := r.db.Begin()
if err != nil {
return fmt.Errorf("failed to create transaction: %s", err)
}
defer func() {
if err != nil {
tx.Rollback()
return
}
tx.Commit()
}()
uuid, err := r.ProjectUUID(pr.Name)
repos, err := r.listRepositories(uuid)
if err != nil {
return err
}
for _, repo := range repos {
if _, err := tx.Exec("DELETE FROM Authentication WHERE repository = ?", repo.UUID); err != nil {
return fmt.Errorf("failed to delete the authentication entries from the database: %s", err)
}
}
if _, err := tx.Exec("DELETE FROM Repositories WHERE project = ?", uuid); err != nil {
return fmt.Errorf("failed to delete the repositories from the database: %s", err)
}
if _, err := tx.Exec("DELETE FROM Projects WHERE uuid = ?", uuid); err != nil {
return fmt.Errorf("failed to delete the project from the database: %s", err)
}
return nil
}
func (r *Repository) List() ([]project.Project, error) {
var prs []project.Project
@@ -288,56 +325,81 @@ func (r *Repository) List() ([]project.Project, error) {
}
defer rows.Close()
for rows.Next() {
var pr project.Project
if err := rows.Scan(&pr.UUID, &pr.Name); err != nil {
return nil, fmt.Errorf("failed to scan project name: %w", err)
}
repos, err := r.listRepositories(pr.UUID)
if err != nil {
return nil, err
}
pr.Repositories = repos
prs = append(prs, pr)
}
return prs, nil
}
func (r *Repository) listRepositories(projectUUID string) ([]project.Repository, error) {
stmt, err := r.db.Prepare("SELECT uuid, name, schedule, source, destination FROM Repositories WHERE project = ?")
if err != nil {
return nil, fmt.Errorf("invalid syntax: %w", err)
}
authStmt, err := r.db.Prepare("SELECT ref, username, password, token FROM Authentication WHERE repository = ?")
rows, err := stmt.Query(projectUUID)
if err != nil {
return nil, fmt.Errorf("invalid syntax: %w", err)
return nil, fmt.Errorf("failed to query repositories for the project %s: %w", projectUUID, err)
}
defer rows.Close()
var repositories []project.Repository
for rows.Next() {
var pr project.Project
var prUUID string
if err := rows.Scan(&prUUID, &pr.Name); err != nil {
return nil, fmt.Errorf("failed to scan project name: %w", err)
}
repoRows, err := stmt.Query(prUUID)
if err != nil {
return nil, fmt.Errorf("failed to query repositories for the project %s: %w", prUUID, err)
}
for repoRows.Next() {
var uuid string
var repo project.Repository
if err := repoRows.Scan(&uuid, &repo.Name, &repo.Schedule, &repo.Source, &repo.Destination); err != nil {
repoRows.Close()
if err := rows.Scan(&repo.UUID, &repo.Name, &repo.Schedule, &repo.Source, &repo.Destination); err != nil {
return nil, fmt.Errorf("failed to scan repository entry: %w", err)
}
authRows, err := authStmt.Query(uuid)
auth, err := r.listAuthentications(repo.UUID)
if err != nil {
repoRows.Close()
return nil, fmt.Errorf("failed to query repositories for the project %s: %w", prUUID, err)
return nil, err
}
repo.Authentications = auth
repositories = append(repositories, repo)
}
auth := make(map[string]project.AuthenticationSettings)
for authRows.Next() {
return repositories, nil
}
func (r *Repository) listAuthentications(repositoryUUID string) (map[string]project.AuthenticationSettings, error) {
stmt, err := r.db.Prepare("SELECT ref, username, password, token FROM Authentication WHERE repository = ?")
if err != nil {
return nil, fmt.Errorf("invalid syntax: %w", err)
}
rows, err := stmt.Query(repositoryUUID)
if err != nil {
return nil, fmt.Errorf("failed to query repositories for the project %s: %w", repositoryUUID, err)
}
defer rows.Close()
res := make(map[string]project.AuthenticationSettings)
for rows.Next() {
var ref string
var username, password, token *string
if err := authRows.Scan(&ref, &username, &password, &token); err != nil {
authRows.Close()
repoRows.Close()
if err := rows.Scan(&ref, &username, &password, &token); err != nil {
return nil, fmt.Errorf("failed to scan authentication entry: %s", err)
}
if token != nil {
auth[ref] = project.AuthenticationSettings{
res[ref] = project.AuthenticationSettings{
Token: *token,
}
} else if username != nil {
auth[ref] = project.AuthenticationSettings{
res[ref] = project.AuthenticationSettings{
Basic: &project.BasicAuthenticationSettings{
Username: *username,
Password: *password,
@@ -346,14 +408,5 @@ func (r *Repository) List() ([]project.Project, error) {
}
}
authRows.Close()
repo.Authentications = auth
pr.Repositories = append(pr.Repositories, repo)
}
repoRows.Close()
prs = append(prs, pr)
}
return prs, nil
return res, nil
}

91
pkg/client/client.go
View File

@@ -28,7 +28,7 @@ func New(url string) *Client {
}
func (c *Client) Apply(pr project.Project) error {
url, err := url.JoinPath(c.url, "api", "v1", "projects", pr.Name)
url, err := url.JoinPath(c.url, "api", "v1", "projects")
if err != nil {
return fmt.Errorf("failed to make url: %s", err)
}
@@ -60,6 +60,31 @@ func (c *Client) Apply(pr project.Project) error {
return nil
}
func (c *Client) Version() (obj.SystemInformation, error) {
url, err := url.JoinPath(c.url, "api", "v1", "version")
if err != nil {
return obj.SystemInformation{}, fmt.Errorf("failed to make url: %s", err)
}
res, err := http.Get(url)
if err != nil {
return obj.SystemInformation{}, fmt.Errorf("failed to send the request to the server: %s", err)
}
defer res.Body.Close()
if res.StatusCode != 200 {
return obj.SystemInformation{}, fmt.Errorf("failed to send the request to the server: %s: %s", res.Status, toError(res.Body))
}
var payload obj.HTTPObject[obj.SystemInformation]
d := json.NewDecoder(res.Body)
if err := d.Decode(&payload); err != nil {
return obj.SystemInformation{}, fmt.Errorf("failed to parse the server response, is your client up-to-date? (reason: %s)", err)
}
return payload.Data, nil
}
func (c *Client) List() ([]project.Project, error) {
url, err := url.JoinPath(c.url, "api", "v1", "projects", "all")
if err != nil {
@@ -92,6 +117,70 @@ func (c *Client) List() ([]project.Project, error) {
return prs, nil
}
func (c *Client) Remove(pr project.Project) error {
url, err := url.JoinPath(c.url, "api", "v1", "projects")
if err != nil {
return fmt.Errorf("failed to make url: %s", err)
}
data, err := json.Marshal(pr)
if err != nil {
return fmt.Errorf("failed to marshal project data: %s", err)
}
r := bytes.NewReader(data)
req, err := http.NewRequest("DELETE", url, r)
if err != nil {
return fmt.Errorf("failed to make request: %s", err)
}
cli := http.Client{}
res, err := cli.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request to the server: %s", err)
}
defer res.Body.Close()
if res.StatusCode != 204 {
return fmt.Errorf("failed to send the request to the server: %s: %s", res.Status, toError(res.Body))
}
return nil
}
func (c *Client) RunOne(pr project.Project) error {
url, err := url.JoinPath(c.url, "api", "v1", "run")
if err != nil {
return fmt.Errorf("failed to make url: %s", err)
}
data, err := json.Marshal(pr)
if err != nil {
return fmt.Errorf("failed to marshal project data: %s", err)
}
r := bytes.NewReader(data)
req, err := http.NewRequest("EXECUTE", url, r)
if err != nil {
return fmt.Errorf("failed to make request: %s", err)
}
cli := http.Client{}
res, err := cli.Do(req)
if err != nil {
return fmt.Errorf("failed to send the request to the server: %s", err)
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("failed to send the request to the server: %s: %s", res.Status, toError(res.Body))
}
return nil
}
func toError(body io.ReadCloser) error {
var msg SimpleError

25
pkg/project/file.go
View File

@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/goccy/go-yaml"
@@ -49,6 +48,11 @@ type (
Username string `yaml:"username"`
Password string `yaml:"password"`
}
DefaultValues struct {
DaemonURL string
ProjectName string
}
)
var (
@@ -57,7 +61,7 @@ var (
ErrParsing error = errors.New("failed to parse file")
)
func LoadCurrent() (Project, error) {
func LoadCurrent(defaultValues DefaultValues) (Project, error) {
f, err := os.OpenFile("./git-compose.yaml", os.O_RDONLY, 0)
if err != nil {
return Project{}, fmt.Errorf("%w: %s", ErrIO, err)
@@ -70,31 +74,26 @@ func LoadCurrent() (Project, error) {
return Project{}, fmt.Errorf("%w: %s", ErrParsing, err)
}
return decode(mainFile)
return decode(mainFile, defaultValues)
}
func LoadBytes(b []byte) (Project, error) {
func LoadBytes(b []byte, defaultValues DefaultValues) (Project, error) {
var mainFile MainFile
if err := json.Unmarshal(b, &mainFile); err != nil {
return Project{}, fmt.Errorf("%w: %s", ErrParsing, err)
}
return decode(mainFile)
}
func decode(mainFile MainFile) (Project, error) {
wd, err := os.Getwd()
if err != nil {
return Project{}, fmt.Errorf("%w: cannot get current working directory path: %s", ErrOS, err)
return decode(mainFile, defaultValues)
}
func decode(mainFile MainFile, defaultValues DefaultValues) (Project, error) {
if err := checkConfig(mainFile); err != nil {
return Project{}, fmt.Errorf("failed to validate configuration: %w", err)
}
pr := Project{
Name: filepath.Base(wd),
ServerURL: "http://localhost:25697",
Name: defaultValues.ProjectName,
ServerURL: defaultValues.DaemonURL,
}
if len(strings.TrimSpace(mainFile.ProjectName)) > 0 {

2
pkg/project/project.go
View File

@@ -2,12 +2,14 @@ package project
type (
Project struct {
UUID string `json:"uuid"`
Name string `json:"name"`
Repositories []Repository `json:"repositories"`
ServerURL string `json:"-"`
}
Repository struct {
UUID string `json:"uuid"`
Name string `json:"name"`
Schedule string `json:"schedule"`
Source string `json:"source"`