first working version

.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+
+        {
+            "name": "Server",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cmd/server",
+            "args": ["-db-path", "./data.db"],
+            "cwd": "${workspaceFolder}"
+        }
+    ]
+}

cmd/server/api/api.go

@@ -11,20 +11,24 @@ import (
 	"net/http"
 	"runtime"
 
+	cronruntime "mirror-sync/cmd/server/core/runtime"
+
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 )
 
 type (
 	HTTPServer struct {
-		Server *http.Server
+		Server    *http.Server
-		data   *storage.Repository
+		data      *storage.Repository
+		scheduler *cronruntime.Scheduler
 	}
 )
 
-func NewServer(data *storage.Repository, port int) *HTTPServer {
+func NewServer(data *storage.Repository, scheduler *cronruntime.Scheduler, port int) *HTTPServer {
 	s := &HTTPServer{
-		data: data,
+		data:      data,
+		scheduler: scheduler,
 	}
 	router := chi.NewRouter()
 	router.NotFound(func(writer http.ResponseWriter, request *http.Request) {
@@ -90,6 +94,13 @@ func (s *HTTPServer) ProjectPostHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	s.scheduler.Remove(pr)
+	if err := s.scheduler.Add(pr); err != nil {
+		slog.Error("failed to run project", "err", err)
+		internalServerError(err, w, r)
+		return
+	}
+
 	w.WriteHeader(201)
 }
 

cmd/server/core/git/git.go

@@ -1,6 +1,7 @@
 package git
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/go-git/go-git/v6"
@@ -12,9 +13,10 @@ import (
 
 type (
 	Repository struct {
-		src  string
+		src     string
-		dst  string
+		dst     string
-		auth Authentication
+		srcAuth Authentication
+		dstAuth Authentication
 	}
 
 	Authentication interface {
@@ -22,16 +24,42 @@ type (
 	}
 
 	TokenAuthentication struct {
-		username string
+		token string
-		token    string
+	}
+
+	BasicAuthentication struct {
+		username, password string
 	}
 
 	NoAuthentication struct{}
 )
 
+func NewRepository(src, dst string, srcAuth, dstAuth Authentication) Repository {
+	return Repository{
+		src:     src,
+		dst:     dst,
+		srcAuth: srcAuth,
+		dstAuth: dstAuth,
+	}
+}
+
+func NewTokenAuthentication(token string) TokenAuthentication {
+	return TokenAuthentication{
+		token: token,
+	}
+}
+
+func NewBasicAuthentication(username, password string) BasicAuthentication {
+	return BasicAuthentication{
+		username: username,
+		password: password,
+	}
+}
+
 func Sync(r Repository) error {
 	repo, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
-		URL: r.src,
+		URL:  r.src,
+		Auth: r.srcAuth.Value(),
 	})
 	if err != nil {
 		return fmt.Errorf("failed to clone repository from source: %w", err)
@@ -50,11 +78,14 @@ func Sync(r Repository) error {
 
 	err = m.Push(&git.PushOptions{
 		RemoteName: "mirror",
-		Auth:       r.auth.Value(),
+		Auth:       r.dstAuth.Value(),
 		RefSpecs:   []config.RefSpec{"+refs/*:refs/*"},
 		Force:      true,
 	})
 	if err != nil {
+		if errors.Is(err, git.NoErrAlreadyUpToDate) {
+			return nil
+		}
 		return fmt.Errorf("failed to push to mirror server: %w", err)
 	}
 
@@ -63,11 +94,18 @@ func Sync(r Repository) error {
 
 func (a TokenAuthentication) Value() transport.AuthMethod {
 	return &http.BasicAuth{
-		Username: a.username,
+		Username: "git",
 		Password: a.token,
 	}
 }
 
+func (a BasicAuthentication) Value() transport.AuthMethod {
+	return &http.BasicAuth{
+		Username: a.username,
+		Password: a.password,
+	}
+}
+
 func (NoAuthentication) Value() transport.AuthMethod {
 	return nil
 }

cmd/server/core/runtime/runtime.go

@@ -0,0 +1,86 @@
+package runtime
+
+import (
+	"fmt"
+	"log/slog"
+	"mirror-sync/cmd/server/core/git"
+	"mirror-sync/pkg/project"
+
+	"github.com/robfig/cron/v3"
+)
+
+type (
+	Scheduler struct {
+		cr  *cron.Cron
+		ids map[string]map[string]cron.EntryID
+	}
+)
+
+func New(prs []project.Project) (*Scheduler, error) {
+	s := &Scheduler{
+		cr:  cron.New(),
+		ids: make(map[string]map[string]cron.EntryID),
+	}
+
+	for _, pr := range prs {
+		if err := s.Add(pr); err != nil {
+			return nil, err
+		}
+	}
+
+	return s, nil
+}
+
+func (s *Scheduler) Add(pr project.Project) error {
+	s.ids[pr.Name] = make(map[string]cron.EntryID)
+	for _, repo := range pr.Repositories {
+		var srcAuth git.Authentication = git.NoAuthentication{}
+		var dstAuth git.Authentication = git.NoAuthentication{}
+		if v, ok := repo.Authentications["source"]; ok {
+			if len(v.Token) > 0 {
+				srcAuth = git.NewTokenAuthentication(v.Token)
+			} else if v.Basic != nil {
+				srcAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
+			}
+		}
+		if v, ok := repo.Authentications["mirror"]; ok {
+			if len(v.Token) > 0 {
+				dstAuth = git.NewTokenAuthentication(v.Token)
+			} else if v.Basic != nil {
+				dstAuth = git.NewBasicAuthentication(v.Basic.Username, v.Basic.Password)
+			}
+		}
+		r := git.NewRepository(repo.Source, repo.Destination, srcAuth, dstAuth)
+		id, err := s.cr.AddFunc(repo.Schedule, func() {
+			slog.Info(fmt.Sprintf("[%s] starting sync...", repo.Name))
+			if err := git.Sync(r); err != nil {
+				slog.Error(fmt.Sprintf("[%s] failed to sync repository: %s", repo.Name, err))
+				return
+			}
+			slog.Info(fmt.Sprintf("[%s] synced", repo.Name))
+		})
+		if err != nil {
+			return err
+		}
+		s.ids[pr.Name][repo.Name] = id
+
+		slog.Info(fmt.Sprintf("[%s] scheduled with '%s'", repo.Name, repo.Schedule))
+	}
+
+	return nil
+}
+
+func (s *Scheduler) Remove(pr project.Project) {
+	if v, ok := s.ids[pr.Name]; ok {
+		for name, id := range v {
+			s.cr.Remove(id)
+			slog.Info(fmt.Sprintf("[%s] remove from being run in the future.", name))
+		}
+	}
+	delete(s.ids, pr.Name)
+}
+
+// Run the cron scheduler, or no-op if already running.
+func (s *Scheduler) Run() {
+	s.cr.Run()
+}

cmd/server/core/storage/migrations/001_first_schema.sql

@@ -15,6 +15,14 @@ CREATE TABLE Repositories (
 );
 CREATE INDEX Repositories_uuid_IDX ON Repositories (uuid);
 
+CREATE TABLE Authentication (
+	repository TEXT NOT NULL,
+	ref TEXT NOT NULL,
+	username TEXT,
+	"password" TEXT,
+	token TEXT
+);
+
 -- +goose Down
 DROP TABLE Projects;
 DROP TABLE Repositories;

cmd/server/core/storage/storage.go

@@ -117,16 +117,47 @@ func (r *Repository) Create(pr project.Project) error {
 	}
 
 	// Create repositories entries
-	stmt, err = tx.Prepare("INSERT INTO Repositories (uuid, name, source, destination, schedule, project) VALUES (?, ?, ?, ?, ?, ?)")
+
+	for _, repo := range pr.Repositories {
+		if err := r.createRepository(tx, projectUUID, repo); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r Repository) createRepository(tx *sql.Tx, projectUuid string, repo project.Repository) error {
+	repoUUID := uuid.NewString()
+
+	stmt, err := tx.Prepare("INSERT INTO Repositories (uuid, name, source, destination, schedule, project) VALUES (?, ?, ?, ?, ?, ?)")
 	if err != nil {
 		return fmt.Errorf("failed to create statement: %s", err)
 	}
 
-	for _, repo := range pr.Repositories {
+	if _, err := stmt.Exec(repoUUID, repo.Name, repo.Source, repo.Destination, repo.Schedule, projectUuid); err != nil {
-		repoUUID := uuid.NewString()
+		return fmt.Errorf("failed to execute sql query: %s", err)
+	}
 
-		if _, err := stmt.Exec(repoUUID, repo.Name, repo.Source, repo.Destination, repo.Schedule, projectUUID); err != nil {
+	for ref, auth := range repo.Authentications {
-			return fmt.Errorf("failed to execute sql query: %s", err)
+		if len(auth.Token) > 0 {
+			stmt, err := tx.Prepare("INSERT INTO Authentication (repository, ref, token) VALUES (?, ?, ?)")
+			if err != nil {
+				return fmt.Errorf("failed to create statement: %s", err)
+			}
+
+			if _, err := stmt.Exec(repoUUID, ref, auth.Token); err != nil {
+				return fmt.Errorf("failed to execute sql query: %s", err)
+			}
+		} else if auth.Basic != nil {
+			stmt, err := tx.Prepare("INSERT INTO Authentication (repository, ref, username, password) VALUES (?, ?, ?, ?)")
+			if err != nil {
+				return fmt.Errorf("failed to create statement: %s", err)
+			}
+
+			if _, err := stmt.Exec(repoUUID, ref, auth.Basic.Username, auth.Basic.Password); err != nil {
+				return fmt.Errorf("failed to execute sql query: %s", err)
+			}
 		}
 	}
 
@@ -185,11 +216,6 @@ func (r *Repository) Update(pr project.Project) error {
 		return fmt.Errorf("failed to get project uuid: %w", err)
 	}
 
-	stmt, err := tx.Prepare("UPDATE Repositories SET schedule = ?, source = ?, destination = ? WHERE uuid = ?")
-	if err != nil {
-		return fmt.Errorf("failed to create statement: %w", err)
-	}
-
 	// this loop does NOT remove orphan
 	for _, repo := range pr.Repositories {
 		// checks if the repo exists
@@ -200,24 +226,52 @@ func (r *Repository) Update(pr project.Project) error {
 
 		if exists {
 			// if it exists, just update it
-			uuid, err := r.RepositoryUUID(repo.Name)
+			if err := r.updateRepository(tx, repo); err != nil {
-			if err != nil {
+				return err
-				return fmt.Errorf("failed to get uuid from database: %w", err)
-			}
-
-			if _, err := stmt.Exec(repo.Schedule, repo.Source, repo.Destination, uuid); err != nil {
-				return fmt.Errorf("failed to update repository entry for %s::'%s'", uuid, repo.Name)
 			}
 		} else {
 			// if not, create a new uuid and create the entry
-			repoUUID := uuid.NewString()
+			if err := r.createRepository(tx, projectUUID, repo); err != nil {
-
+				return err
-			if _, err := stmt.Exec(repoUUID, repo.Name, repo.Source, repo.Destination, repo.Schedule, projectUUID); err != nil {
-				return fmt.Errorf("failed to execute sql query: %s", err)
 			}
 		}
-		if _, err := stmt.Exec(repo.Schedule, repo.Source, repo.Destination, repo.Name); err != nil {
+	}
-			return fmt.Errorf("failed to update repository entry for '%s'", repo.Name)
+
+	return nil
+}
+
+func (r *Repository) updateRepository(tx *sql.Tx, repo project.Repository) error {
+	uuid, err := r.RepositoryUUID(repo.Name)
+	if err != nil {
+		return fmt.Errorf("failed to get uuid from database: %w", err)
+	}
+
+	stmt, err := tx.Prepare("UPDATE Repositories SET schedule = ?, source = ?, destination = ? WHERE uuid = ?")
+	if err != nil {
+		return fmt.Errorf("failed to create statement: %w", err)
+	}
+
+	if _, err := stmt.Exec(repo.Schedule, repo.Source, repo.Destination, uuid); err != nil {
+		return fmt.Errorf("failed to update repository entry for %s::'%s'", uuid, repo.Name)
+	}
+
+	for ref, auth := range repo.Authentications {
+		if auth.Basic != nil {
+			stmt, err := tx.Prepare("UPDATE Authentication SET username = ?, password = ?, token = null WHERE repository = ? AND ref = ?")
+			if err != nil {
+				return fmt.Errorf("failed to create statement: %w", err)
+			}
+			if _, err := stmt.Exec(auth.Basic.Username, auth.Basic.Password, uuid, ref); err != nil {
+				return fmt.Errorf("failed to execut sql query: %s", err)
+			}
+		} else {
+			stmt, err := tx.Prepare("UPDATE Authentication SET username = null, password = null, token = ? WHERE repository = ? AND ref = ?")
+			if err != nil {
+				return fmt.Errorf("failed to create statement: %w", err)
+			}
+			if _, err := stmt.Exec(auth.Token, uuid, ref); err != nil {
+				return fmt.Errorf("failed to execute sql query: %s", err)
+			}
 		}
 	}
 
@@ -233,7 +287,12 @@ func (r *Repository) List() ([]project.Project, error) {
 	}
 	defer rows.Close()
 
-	stmt, err := r.db.Prepare("SELECT name, schedule, source, destination FROM Repositories WHERE project = ?")
+	stmt, err := r.db.Prepare("SELECT uuid, name, schedule, source, destination FROM Repositories WHERE project = ?")
+	if err != nil {
+		return nil, fmt.Errorf("invalid syntax: %w", err)
+	}
+
+	authStmt, err := r.db.Prepare("SELECT ref, username, password, token FROM Authentication WHERE repository = ?")
 	if err != nil {
 		return nil, fmt.Errorf("invalid syntax: %w", err)
 	}
@@ -250,11 +309,44 @@ func (r *Repository) List() ([]project.Project, error) {
 		}
 
 		for repoRows.Next() {
+			var uuid string
 			var repo project.Repository
-			if err := repoRows.Scan(&repo.Name, &repo.Schedule, &repo.Source, &repo.Destination); err != nil {
+			if err := repoRows.Scan(&uuid, &repo.Name, &repo.Schedule, &repo.Source, &repo.Destination); err != nil {
 				repoRows.Close()
 				return nil, fmt.Errorf("failed to scan repository entry: %w", err)
 			}
+
+			authRows, err := authStmt.Query(uuid)
+			if err != nil {
+				repoRows.Close()
+				return nil, fmt.Errorf("failed to query repositories for the project %s: %w", prUUID, err)
+			}
+
+			auth := make(map[string]project.AuthenticationSettings)
+			for authRows.Next() {
+				var ref string
+				var username, password, token *string
+				if err := authRows.Scan(&ref, &username, &password, &token); err != nil {
+					authRows.Close()
+					repoRows.Close()
+					return nil, fmt.Errorf("failed to scan authentication entry: %s", err)
+				}
+				if token != nil {
+					auth[ref] = project.AuthenticationSettings{
+						Token: *token,
+					}
+				} else if username != nil {
+					auth[ref] = project.AuthenticationSettings{
+						Basic: &project.BasicAuthenticationSettings{
+							Username: *username,
+							Password: *password,
+						},
+					}
+				}
+			}
+
+			authRows.Close()
+			repo.Authentications = auth
 			pr.Repositories = append(pr.Repositories, repo)
 		}
 

cmd/server/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log/slog"
 	"mirror-sync/cmd/server/api"
+	cronruntime "mirror-sync/cmd/server/core/runtime"
 	"mirror-sync/cmd/server/core/storage"
 	"mirror-sync/pkg/constants"
 	"os"
@@ -29,7 +30,24 @@ func main() {
 		os.Exit(1)
 	}
 
-	s := api.NewServer(data, 8080)
+	// runtime
+	prs, err := data.List()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "failed to start server:", err.Error())
+		os.Exit(1)
+	}
+
+	scheduler, err := cronruntime.New(prs)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "failed to start server:", err.Error())
+		os.Exit(1)
+	}
+
+	go scheduler.Run()
+	slog.Info("daemon scheduler is running")
+
+	// api
+	s := api.NewServer(data, scheduler, 8080)
 
 	slog.Info("daemon listening to :8080")
 	if err := s.Server.ListenAndServe(); err != nil {

pkg/project/file.go

@@ -1,6 +1,7 @@
 package project
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -30,8 +31,23 @@ type (
 	}
 
 	GitStorage struct {
-		Source string `yaml:"source"`
+		Source StorageSettings `yaml:"source"`
-		Mirror string `yaml:"mirror"`
+		Mirror StorageSettings `yaml:"mirror"`
+	}
+
+	StorageSettings struct {
+		URL            string                   `yaml:"url"`
+		Authentication AuthenticationDescriptor `yaml:"authentication"`
+	}
+
+	AuthenticationDescriptor struct {
+		Basic BasicAuthenticationDescriptor `yaml:"basic"`
+		Token string                        `yaml:"token"`
+	}
+
+	BasicAuthenticationDescriptor struct {
+		Username string `yaml:"username"`
+		Password string `yaml:"password"`
 	}
 )
 
@@ -42,11 +58,6 @@ var (
 )
 
 func LoadCurrent() (Project, error) {
-	wd, err := os.Getwd()
-	if err != nil {
-		return Project{}, fmt.Errorf("%w: cannot get current working directory path: %s", ErrOS, err)
-	}
-
 	f, err := os.OpenFile("./git-compose.yaml", os.O_RDONLY, 0)
 	if err != nil {
 		return Project{}, fmt.Errorf("%w: %s", ErrIO, err)
@@ -59,6 +70,24 @@ func LoadCurrent() (Project, error) {
 		return Project{}, fmt.Errorf("%w: %s", ErrParsing, err)
 	}
 
+	return decode(mainFile)
+}
+
+func LoadBytes(b []byte) (Project, error) {
+	var mainFile MainFile
+	if err := json.Unmarshal(b, &mainFile); err != nil {
+		return Project{}, fmt.Errorf("%w: %s", ErrParsing, err)
+	}
+
+	return decode(mainFile)
+}
+
+func decode(mainFile MainFile) (Project, error) {
+	wd, err := os.Getwd()
+	if err != nil {
+		return Project{}, fmt.Errorf("%w: cannot get current working directory path: %s", ErrOS, err)
+	}
+
 	if err := checkConfig(mainFile); err != nil {
 		return Project{}, fmt.Errorf("failed to validate configuration: %w", err)
 	}
@@ -85,25 +114,52 @@ func LoadCurrent() (Project, error) {
 	}
 
 	for repoName, repo := range mainFile.Repositories {
-		pr.Repositories = append(pr.Repositories, Repository{
+		r := Repository{
 			Name:        fmt.Sprintf("%s-%s", pr.Name, strings.ToLower(repoName)),
-			Source:      repo.Storage.Source,
+			Source:      repo.Storage.Source.URL,
-			Destination: repo.Storage.Mirror,
+			Destination: repo.Storage.Mirror.URL,
 			Schedule:    repo.Schedule,
-		})
+		}
+
+		r.Authentications = make(map[string]AuthenticationSettings)
+		setAuthentication(r.Authentications, "source", repo.Storage.Source.Authentication)
+		setAuthentication(r.Authentications, "mirror", repo.Storage.Mirror.Authentication)
+
+		pr.Repositories = append(pr.Repositories, r)
 	}
 
 	return pr, nil
 }
 
+func setAuthentication(m map[string]AuthenticationSettings, key string, auth AuthenticationDescriptor) {
+	if len(auth.Token) > 0 {
+		m[key] = AuthenticationSettings{
+			Token: auth.Token,
+		}
+	} else if len(auth.Basic.Username) > 0 {
+		m[key] = AuthenticationSettings{
+			Basic: &BasicAuthenticationSettings{
+				Username: auth.Basic.Username,
+				Password: auth.Basic.Password,
+			},
+		}
+	}
+}
+
 func checkConfig(mf MainFile) error {
 	for _, r := range mf.Repositories {
-		if len(strings.TrimSpace(r.Storage.Source)) == 0 {
+		if len(strings.TrimSpace(r.Storage.Source.URL)) == 0 {
 			return fmt.Errorf("source is empty")
 		}
-		if len(strings.TrimSpace(r.Storage.Mirror)) == 0 {
+		if err := checkAuthenticationConfig(r.Storage.Source); err != nil {
+			return err
+		}
+		if len(strings.TrimSpace(r.Storage.Mirror.URL)) == 0 {
 			return fmt.Errorf("mirror is empty")
 		}
+		if err := checkAuthenticationConfig(r.Storage.Mirror); err != nil {
+			return err
+		}
 		if len(strings.TrimSpace(r.Schedule)) == 0 {
 			return fmt.Errorf("schedule is empty")
 		}
@@ -114,3 +170,10 @@ func checkConfig(mf MainFile) error {
 
 	return nil
 }
+
+func checkAuthenticationConfig(ss StorageSettings) error {
+	if len(ss.Authentication.Token) > 0 && (len(ss.Authentication.Basic.Username) > 0 || len(ss.Authentication.Basic.Password) > 0) {
+		return fmt.Errorf("cannot use token and basic authentication in the same repository")
+	}
+	return nil
+}

pkg/project/project.go

@@ -8,9 +8,20 @@ type (
 	}
 
 	Repository struct {
-		Name        string `json:"name"`
+		Name            string                            `json:"name"`
-		Schedule    string `json:"schedule"`
+		Schedule        string                            `json:"schedule"`
-		Source      string `json:"source"`
+		Source          string                            `json:"source"`
-		Destination string `json:"destination"`
+		Destination     string                            `json:"destination"`
+		Authentications map[string]AuthenticationSettings `json:"authentications"`
+	}
+
+	AuthenticationSettings struct {
+		Basic *BasicAuthenticationSettings `json:"basic,omitempty"`
+		Token string                       `json:"token,omitempty"`
+	}
+
+	BasicAuthenticationSettings struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
 	}
 )