From 98bc0df9b87c90bb7733cc51bd7aa10b645a14df Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 25 Jul 2025 11:07:37 -0700 Subject: [PATCH] some improvements --- modules/gitrepo/config.go | 43 +++++++++++++++++------ modules/templates/util_misc.go | 11 +++--- services/mirror/mirror_pull.go | 64 +++++++++++++++++++++++++++++++++- services/mirror/mirror_push.go | 6 ++-- 4 files changed, 103 insertions(+), 21 deletions(-) diff --git a/modules/gitrepo/config.go b/modules/gitrepo/config.go index e2f35343b31..d19dcfea212 100644 --- a/modules/gitrepo/config.go +++ b/modules/gitrepo/config.go @@ -5,6 +5,7 @@ package gitrepo import ( "context" + "errors" "io" "time" @@ -60,7 +61,14 @@ func UpdateGitConfig(ctx context.Context, repo Repository, key, value string) (s return value, err1 } -func AddGitRemote(ctx context.Context, repo Repository, remoteName, remoteURL string, options ...string) error { +type RemoteOption string + +const ( + RemoteOptionMirrorPush RemoteOption = "--mirror=push" + RemoteOptionMirrorFetch RemoteOption = "--mirror=fetch" +) + +func AddGitRemote(ctx context.Context, repo Repository, remoteName, remoteURL string, options ...RemoteOption) error { releaser, err := globallock.Lock(ctx, getRepoConfigLockKey(repo.RelativePath())) if err != nil { return err @@ -69,7 +77,14 @@ func AddGitRemote(ctx context.Context, repo Repository, remoteName, remoteURL st cmd := git.NewCommand("remote", "add") if len(options) > 0 { - cmd.AddDynamicArguments(options...) + switch options[0] { + case RemoteOptionMirrorPush: + cmd.AddArguments("--mirror=push") + case RemoteOptionMirrorFetch: + cmd.AddArguments("--mirror=fetch") + default: + return errors.New("unknown remote option: " + string(options[0])) + } } _, _, err = cmd. AddDynamicArguments(remoteName, remoteURL). @@ -95,17 +110,28 @@ func GetRemoteURL(ctx context.Context, repo Repository, remoteName string) (*git if err != nil { return nil, err } + if addr == "" { + return nil, nil + } return giturl.ParseGitURL(addr) } -// PruneRemote prunes the remote branches that no longer exist in the remote repository. -func PruneRemote(ctx context.Context, repo Repository, remoteName string, timeout time.Duration, stdout, stderr io.Writer) error { +func SetRemoteURL(ctx context.Context, repo Repository, remoteName, remoteURL string) error { releaser, err := globallock.Lock(ctx, getRepoConfigLockKey(repo.RelativePath())) if err != nil { return err } defer releaser() + cmd := git.NewCommand("remote", "set-url").AddDynamicArguments(remoteName, remoteURL) + _, _, err = cmd.RunStdString(ctx, &git.RunOpts{Dir: repoPath(repo)}) + return err +} + +// PruneRemote prunes the remote branches that no longer exist in the remote repository. +// No lock is needed because the remote remoteName will be checked before invoking this function. +// Then it will not update the remote automatically if the remote does not exist. +func PruneRemote(ctx context.Context, repo Repository, remoteName string, timeout time.Duration, stdout, stderr io.Writer) error { return git.NewCommand("remote", "prune").AddDynamicArguments(remoteName). Run(ctx, &git.RunOpts{ Timeout: timeout, @@ -115,13 +141,10 @@ func PruneRemote(ctx context.Context, repo Repository, remoteName string, timeou }) } +// UpdateRemotePrune updates the remote branches and prunes the ones that no longer exist in the remote repository. +// No lock is needed because the remote remoteName will be checked before invoking this function. +// Then it will not update the remote automatically if the remote does not exist. func UpdateRemotePrune(ctx context.Context, repo Repository, remoteName string, timeout time.Duration, stdout, stderr io.Writer) error { - releaser, err := globallock.Lock(ctx, getRepoConfigLockKey(repo.RelativePath())) - if err != nil { - return err - } - defer releaser() - return git.NewCommand("remote", "update", "--prune").AddDynamicArguments(remoteName). Run(ctx, &git.RunOpts{ Timeout: timeout, diff --git a/modules/templates/util_misc.go b/modules/templates/util_misc.go index cc5bf67b42b..8e01cc109c1 100644 --- a/modules/templates/util_misc.go +++ b/modules/templates/util_misc.go @@ -14,8 +14,7 @@ import ( activities_model "code.gitea.io/gitea/models/activities" repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/git" - giturl "code.gitea.io/gitea/modules/git/url" + "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/repository" @@ -145,15 +144,13 @@ type remoteAddress struct { func mirrorRemoteAddress(ctx context.Context, m *repo_model.Repository, remoteName string) remoteAddress { ret := remoteAddress{} - remoteURL, err := git.GetRemoteAddress(ctx, m.RepoPath(), remoteName) + u, err := gitrepo.GetRemoteURL(ctx, m, remoteName) if err != nil { log.Error("GetRemoteURL %v", err) return ret } - - u, err := giturl.ParseGitURL(remoteURL) - if err != nil { - log.Error("giturl.Parse %v", err) + if u == nil { + log.Error("GetRemoteURL %s returned nil", remoteName) return ret } diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index 47f6d9da99f..026d4d7b19b 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -208,6 +208,24 @@ func pruneBrokenReferences(ctx context.Context, stderrBuilder.Reset() stdoutBuilder.Reset() + // check whether the remote still exists before pruning to avoid prune creating a new remote + // this is needed because prune will not fail if the remote does not exist + u, err := gitrepo.GetRemoteURL(ctx, storageRepo, m.GetRemoteName()) + if err != nil { + return err + } + if u == nil { + return fmt.Errorf("remote %s does not exist for %srepository %s", m.GetRemoteName(), wiki, storageRepo.RelativePath()) + } + + fetchConfig, err := gitrepo.GetGitConfig(ctx, storageRepo, "remote.origin.fetch") + if err != nil { + return err + } + if fetchConfig == "" { + return fmt.Errorf("remote %s has no fetch config for %srepository %s", m.GetRemoteName(), wiki, storageRepo.RelativePath()) + } + pruneErr := gitrepo.PruneRemote(ctx, storageRepo, m.GetRemoteName(), timeout, stdoutBuilder, stderrBuilder) if pruneErr != nil { stdout := stdoutBuilder.String() @@ -263,7 +281,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo remoteURL, remoteErr := gitrepo.GetRemoteURL(ctx, m.Repo, m.GetRemoteName()) if remoteErr != nil { - log.Error("SyncMirrors [repo: %-v]: GetRemoteAddress Error %v", m.Repo, remoteErr) + log.Error("SyncMirrors [repo: %-v]: GetRemoteURL Error %v", m.Repo, remoteErr) return nil, false } @@ -365,6 +383,28 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo stderrBuilder.Reset() stdoutBuilder.Reset() + // check whether the remote still exists before pruning to avoid prune creating a new remote + // this is needed because prune will not fail if the remote does not exist + u, err := gitrepo.GetRemoteURL(ctx, m.Repo.WikiStorageRepo(), m.GetRemoteName()) + if err != nil { + log.Error("SyncMirrors [repo: %-v Wiki]: GetRemoteURL Error %v", m.Repo, err) + return nil, false + } + if u == nil { + log.Error("remote %s does not exist for repository %s", m.GetRemoteName(), m.Repo.WikiStorageRepo().RelativePath()) + return nil, false + } + + fetchConfig, err := gitrepo.GetGitConfig(ctx, m.Repo.WikiStorageRepo(), "remote.origin.fetch") + if err != nil { + log.Error("SyncMirrors [repo: %-v Wiki]: GetGitConfig Error %v", m.Repo, err) + return nil, false + } + if fetchConfig == "" { + log.Error("remote %s has no fetch config for repository %s", m.GetRemoteName(), m.Repo.WikiStorageRepo().RelativePath()) + return nil, false + } + if err := gitrepo.UpdateRemotePrune(ctx, m.Repo.WikiStorageRepo(), m.GetRemoteName(), timeout, &stdoutBuilder, &stderrBuilder); err != nil { stdout := stdoutBuilder.String() @@ -386,6 +426,28 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*mirrorSyncResult, bo stderrBuilder.Reset() stdoutBuilder.Reset() + // check whether the remote still exists before pruning to avoid prune creating a new remote + // this is needed because prune will not fail if the remote does not exist + u, err := gitrepo.GetRemoteURL(ctx, m.Repo.WikiStorageRepo(), m.GetRemoteName()) + if err != nil { + log.Error("SyncMirrors [repo: %-v Wiki]: GetRemoteURL Error %v", m.Repo, err) + return nil, false + } + if u == nil { + log.Error("remote %s does not exist for repository %s", m.GetRemoteName(), m.Repo.WikiStorageRepo().RelativePath()) + return nil, false + } + + fetchConfig, err := gitrepo.GetGitConfig(ctx, m.Repo.WikiStorageRepo(), "remote.origin.fetch") + if err != nil { + log.Error("SyncMirrors [repo: %-v Wiki]: GetGitConfig Error %v", m.Repo, err) + return nil, false + } + if fetchConfig == "" { + log.Error("remote %s has no fetch config for repository %s", m.GetRemoteName(), m.Repo.WikiStorageRepo().RelativePath()) + return nil, false + } + if err = gitrepo.UpdateRemotePrune(ctx, m.Repo.WikiStorageRepo(), m.GetRemoteName(), timeout, &stdoutBuilder, &stderrBuilder); err != nil { stdout := stdoutBuilder.String() diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 00bdfff8de5..bb0ea5bc7a1 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -130,7 +130,7 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { } remoteURL, err := gitrepo.GetRemoteURL(ctx, storageRepo, m.RemoteName) if err != nil { - log.Error("GetRemoteAddress(%s) Error %v", path, err) + log.Error("GetRemoteURL(%s) Error %v", path, err) return errors.New("Unexpected error") } @@ -175,8 +175,8 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { } if m.Repo.HasWiki() { - _, err := git.GetRemoteAddress(ctx, m.Repo.WikiPath(), m.RemoteName) - if err == nil { + u, err := gitrepo.GetRemoteURL(ctx, m.Repo.WikiStorageRepo(), m.RemoteName) + if err == nil && u != nil { err := performPush(m.Repo, true) if err != nil { return err