thelilfrog/mirror-sync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first working version

Browse Source
This commit is contained in:
Aurélie DELHAIE
2025-10-28 19:22:21 +01:00
parent 9406ffda01
commit 4a3fe068a3
9 changed files with 400 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
cmd/server/core/git/git.go
View File

@@ -1,6 +1,7 @@
package git
import (
"errors"
"fmt"
"github.com/go-git/go-git/v6"
@@ -12,9 +13,10 @@ import (
type (
Repository struct {
src string
dst string
auth Authentication
src string
dst string
srcAuth Authentication
dstAuth Authentication
}
Authentication interface {
@@ -22,16 +24,42 @@ type (
}
TokenAuthentication struct {
username string
token string
token string
}
BasicAuthentication struct {
username, password string
}
NoAuthentication struct{}
)
func NewRepository(src, dst string, srcAuth, dstAuth Authentication) Repository {
return Repository{
src: src,
dst: dst,
srcAuth: srcAuth,
dstAuth: dstAuth,
}
}
func NewTokenAuthentication(token string) TokenAuthentication {
return TokenAuthentication{
token: token,
}
}
func NewBasicAuthentication(username, password string) BasicAuthentication {
return BasicAuthentication{
username: username,
password: password,
}
}
func Sync(r Repository) error {
repo, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
URL: r.src,
URL: r.src,
Auth: r.srcAuth.Value(),
})
if err != nil {
return fmt.Errorf("failed to clone repository from source: %w", err)
@@ -50,11 +78,14 @@ func Sync(r Repository) error {
err = m.Push(&git.PushOptions{
RemoteName: "mirror",
Auth: r.auth.Value(),
Auth: r.dstAuth.Value(),
RefSpecs: []config.RefSpec{"+refs/*:refs/*"},
Force: true,
})
if err != nil {
if errors.Is(err, git.NoErrAlreadyUpToDate) {
return nil
}
return fmt.Errorf("failed to push to mirror server: %w", err)
}
@@ -63,11 +94,18 @@ func Sync(r Repository) error {
func (a TokenAuthentication) Value() transport.AuthMethod {
return &http.BasicAuth{
Username: a.username,
Username: "git",
Password: a.token,
}
}
func (a BasicAuthentication) Value() transport.AuthMethod {
return &http.BasicAuth{
Username: a.username,
Password: a.password,
}
}
func (NoAuthentication) Value() transport.AuthMethod {
return nil
}

86
cmd/server/core/runtime/runtime.go Normal file
View File

@@ -0,0 +1,86 @@
package runtime
import (
"fmt"
"log/slog"
"mirror-sync/cmd/server/core/git"
"mirror-sync/pkg/project"
"github.com/robfig/cron/v3"
)
type (
Scheduler struct {
cr *cron.Cron
ids map[string]map[string]cron.EntryID
}
)
func New(prs []project.Project) (*Scheduler, error) {
s := &Scheduler{
cr: cron.New(),
ids: make(map[string]map[string]cron.EntryID),
}
for _, pr := range prs {
if err := s.Add(pr); err != nil {
return nil, err
}
}
return s, nil
}
func (s *Scheduler) Add(pr project.Project) error {
s.ids[pr.Name] = make(map[string]cron.EntryID)
for _, repo := range pr.Repositories {
var srcAuth git.Authentication = git.NoAuthentication{}
var dstAuth git.Authentication = git.NoAuthentication{}
if v, ok := repo.Authentications["source"]; ok {
if len(v.Token) > 0 {
srcAuth = git.NewTokenAuthentication(v.Token)
} else if v.Basic != nil {
srcAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
}
}
if v, ok := repo.Authentications["mirror"]; ok {
if len(v.Token) > 0 {
dstAuth = git.NewTokenAuthentication(v.Token)
} else if v.Basic != nil {
dstAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
}
}
r := git.NewRepository(repo.Source, repo.Destination, srcAuth, dstAuth)
id, err := s.cr.AddFunc(repo.Schedule, func() {
slog.Info(fmt.Sprintf("[%s] starting sync...", repo.Name))
if err := git.Sync(r); err != nil {
slog.Error(fmt.Sprintf("[%s] failed to sync repository: %s", repo.Name, err))
return
}
slog.Info(fmt.Sprintf("[%s] synced", repo.Name))
})
if err != nil {
return err
}
s.ids[pr.Name][repo.Name] = id
slog.Info(fmt.Sprintf("[%s] scheduled with '%s'", repo.Name, repo.Schedule))
}
return nil
}
func (s *Scheduler) Remove(pr project.Project) {
if v, ok := s.ids[pr.Name]; ok {
for name, id := range v {
s.cr.Remove(id)
slog.Info(fmt.Sprintf("[%s] remove from being run in the future.", name))
}
}
delete(s.ids, pr.Name)
}
// Run the cron scheduler, or no-op if already running.
func (s *Scheduler) Run() {
s.cr.Run()
}

8
cmd/server/core/storage/migrations/001_first_schema.sql
View File

@@ -15,6 +15,14 @@ CREATE TABLE Repositories (
);
CREATE INDEX Repositories_uuid_IDX ON Repositories (uuid);
CREATE TABLE Authentication (
repository TEXT NOT NULL,
ref TEXT NOT NULL,
username TEXT,
"password" TEXT,
token TEXT
);
-- +goose Down
DROP TABLE Projects;
DROP TABLE Repositories;

142
cmd/server/core/storage/storage.go
View File

@@ -117,16 +117,47 @@ func (r *Repository) Create(pr project.Project) error {
}
// Create repositories entries
stmt, err = tx.Prepare("INSERT INTO Repositories (uuid, name, source, destination, schedule, project) VALUES (?, ?, ?, ?, ?, ?)")
for _, repo := range pr.Repositories {
if err := r.createRepository(tx, projectUUID, repo); err != nil {
return err
}
}
return nil
}
func (r Repository) createRepository(tx *sql.Tx, projectUuid string, repo project.Repository) error {
repoUUID := uuid.NewString()
stmt, err := tx.Prepare("INSERT INTO Repositories (uuid, name, source, destination, schedule, project) VALUES (?, ?, ?, ?, ?, ?)")
if err != nil {
return fmt.Errorf("failed to create statement: %s", err)
}
for _, repo := range pr.Repositories {
repoUUID := uuid.NewString()
if _, err := stmt.Exec(repoUUID, repo.Name, repo.Source, repo.Destination, repo.Schedule, projectUuid); err != nil {
return fmt.Errorf("failed to execute sql query: %s", err)
}
if _, err := stmt.Exec(repoUUID, repo.Name, repo.Source, repo.Destination, repo.Schedule, projectUUID); err != nil {
return fmt.Errorf("failed to execute sql query: %s", err)
for ref, auth := range repo.Authentications {
if len(auth.Token) > 0 {
stmt, err := tx.Prepare("INSERT INTO Authentication (repository, ref, token) VALUES (?, ?, ?)")
if err != nil {
return fmt.Errorf("failed to create statement: %s", err)
}
if _, err := stmt.Exec(repoUUID, ref, auth.Token); err != nil {
return fmt.Errorf("failed to execute sql query: %s", err)
}
} else if auth.Basic != nil {
stmt, err := tx.Prepare("INSERT INTO Authentication (repository, ref, username, password) VALUES (?, ?, ?, ?)")
if err != nil {
return fmt.Errorf("failed to create statement: %s", err)
}
if _, err := stmt.Exec(repoUUID, ref, auth.Basic.Username, auth.Basic.Password); err != nil {
return fmt.Errorf("failed to execute sql query: %s", err)
}
}
}
@@ -185,11 +216,6 @@ func (r *Repository) Update(pr project.Project) error {
return fmt.Errorf("failed to get project uuid: %w", err)
}
stmt, err := tx.Prepare("UPDATE Repositories SET schedule = ?, source = ?, destination = ? WHERE uuid = ?")
if err != nil {
return fmt.Errorf("failed to create statement: %w", err)
}
// this loop does NOT remove orphan
for _, repo := range pr.Repositories {
// checks if the repo exists
@@ -200,24 +226,52 @@ func (r *Repository) Update(pr project.Project) error {
if exists {
// if it exists, just update it
uuid, err := r.RepositoryUUID(repo.Name)
if err != nil {
return fmt.Errorf("failed to get uuid from database: %w", err)
}
if _, err := stmt.Exec(repo.Schedule, repo.Source, repo.Destination, uuid); err != nil {
return fmt.Errorf("failed to update repository entry for %s::'%s'", uuid, repo.Name)
if err := r.updateRepository(tx, repo); err != nil {
return err
}
} else {
// if not, create a new uuid and create the entry
repoUUID := uuid.NewString()
if _, err := stmt.Exec(repoUUID, repo.Name, repo.Source, repo.Destination, repo.Schedule, projectUUID); err != nil {
return fmt.Errorf("failed to execute sql query: %s", err)
if err := r.createRepository(tx, projectUUID, repo); err != nil {
return err
}
}
if _, err := stmt.Exec(repo.Schedule, repo.Source, repo.Destination, repo.Name); err != nil {
return fmt.Errorf("failed to update repository entry for '%s'", repo.Name)
}
return nil
}
func (r *Repository) updateRepository(tx *sql.Tx, repo project.Repository) error {
uuid, err := r.RepositoryUUID(repo.Name)
if err != nil {
return fmt.Errorf("failed to get uuid from database: %w", err)
}
stmt, err := tx.Prepare("UPDATE Repositories SET schedule = ?, source = ?, destination = ? WHERE uuid = ?")
if err != nil {
return fmt.Errorf("failed to create statement: %w", err)
}
if _, err := stmt.Exec(repo.Schedule, repo.Source, repo.Destination, uuid); err != nil {
return fmt.Errorf("failed to update repository entry for %s::'%s'", uuid, repo.Name)
}
for ref, auth := range repo.Authentications {
if auth.Basic != nil {
stmt, err := tx.Prepare("UPDATE Authentication SET username = ?, password = ?, token = null WHERE repository = ? AND ref = ?")
if err != nil {
return fmt.Errorf("failed to create statement: %w", err)
}
if _, err := stmt.Exec(auth.Basic.Username, auth.Basic.Password, uuid, ref); err != nil {
return fmt.Errorf("failed to execut sql query: %s", err)
}
} else {
stmt, err := tx.Prepare("UPDATE Authentication SET username = null, password = null, token = ? WHERE repository = ? AND ref = ?")
if err != nil {
return fmt.Errorf("failed to create statement: %w", err)
}
if _, err := stmt.Exec(auth.Token, uuid, ref); err != nil {
return fmt.Errorf("failed to execute sql query: %s", err)
}
}
}
@@ -233,7 +287,12 @@ func (r *Repository) List() ([]project.Project, error) {
}
defer rows.Close()
stmt, err := r.db.Prepare("SELECT name, schedule, source, destination FROM Repositories WHERE project = ?")
stmt, err := r.db.Prepare("SELECT uuid, name, schedule, source, destination FROM Repositories WHERE project = ?")
if err != nil {
return nil, fmt.Errorf("invalid syntax: %w", err)
}
authStmt, err := r.db.Prepare("SELECT ref, username, password, token FROM Authentication WHERE repository = ?")
if err != nil {
return nil, fmt.Errorf("invalid syntax: %w", err)
}
@@ -250,11 +309,44 @@ func (r *Repository) List() ([]project.Project, error) {
}
for repoRows.Next() {
var uuid string
var repo project.Repository
if err := repoRows.Scan(&repo.Name, &repo.Schedule, &repo.Source, &repo.Destination); err != nil {
if err := repoRows.Scan(&uuid, &repo.Name, &repo.Schedule, &repo.Source, &repo.Destination); err != nil {
repoRows.Close()
return nil, fmt.Errorf("failed to scan repository entry: %w", err)
}
authRows, err := authStmt.Query(uuid)
if err != nil {
repoRows.Close()
return nil, fmt.Errorf("failed to query repositories for the project %s: %w", prUUID, err)
}
auth := make(map[string]project.AuthenticationSettings)
for authRows.Next() {
var ref string
var username, password, token *string
if err := authRows.Scan(&ref, &username, &password, &token); err != nil {
authRows.Close()
repoRows.Close()
return nil, fmt.Errorf("failed to scan authentication entry: %s", err)
}
if token != nil {
auth[ref] = project.AuthenticationSettings{
Token: *token,
}
} else if username != nil {
auth[ref] = project.AuthenticationSettings{
Basic: &project.BasicAuthenticationSettings{
Username: *username,
Password: *password,
},
}
}
}
authRows.Close()
repo.Authentications = auth
pr.Repositories = append(pr.Repositories, repo)
}