sascha
/
gitea
mirror of https://github.com/go-gitea/gitea.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Performance optimization for tags synchronization (#34355) 

The tags synchronization is very slow for a non-mirror repository with
many tags especially forking. This PR make all repositories' tags
synchronization use the same function and remove the low performance
synchronization function. The commit count of tag now will not be stored
into database when syncing. Since the commits count will always be read
from cache or git data, the `NumCommits` in the release table will be
updated for the first read from git data.
pull/34514/head^2
Lunny Xiao 2025-05-22 13:54:42 +07:00 committed by GitHub
parent 06ccda06c4
commit b595f81b79
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 31 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/hook.go
Unescape Escape View File

@ -24,7 +24,7 @@ import (
)
const (
hookBatchSize = 30
hookBatchSize = 500
)
var (

61
models/repo/release.go
Unescape Escape View File

@ -161,6 +161,11 @@ func UpdateRelease(ctx context.Context, rel *Release) error {
return err
}
func UpdateReleaseNumCommits(ctx context.Context, rel *Release) error {
_, err := db.GetEngine(ctx).ID(rel.ID).Cols("num_commits").Update(rel)
return err
}
// AddReleaseAttachments adds a release attachments
func AddReleaseAttachments(ctx context.Context, releaseID int64, attachmentUUIDs []string) (err error) {
// Check attachments
@ -418,8 +423,8 @@ func UpdateReleasesMigrationsByType(ctx context.Context, gitServiceType structs.
return err
}
// PushUpdateDeleteTagsContext updates a number of delete tags with context
func PushUpdateDeleteTagsContext(ctx context.Context, repo *Repository, tags []string) error {
// PushUpdateDeleteTags updates a number of delete tags with context
func PushUpdateDeleteTags(ctx context.Context, repo *Repository, tags []string) error {
if len(tags) == 0 {
return nil
}
@ -448,58 +453,6 @@ func PushUpdateDeleteTagsContext(ctx context.Context, repo *Repository, tags []s
return nil
}
// PushUpdateDeleteTag must be called for any push actions to delete tag
func PushUpdateDeleteTag(ctx context.Context, repo *Repository, tagName string) error {
rel, err := GetRelease(ctx, repo.ID, tagName)
if err != nil {
if IsErrReleaseNotExist(err) {
return nil
}
return fmt.Errorf("GetRelease: %w", err)
}
if rel.IsTag {
if _, err = db.DeleteByID[Release](ctx, rel.ID); err != nil {
return fmt.Errorf("Delete: %w", err)
}
} else {
rel.IsDraft = true
rel.NumCommits = 0
rel.Sha1 = ""
if _, err = db.GetEngine(ctx).ID(rel.ID).AllCols().Update(rel); err != nil {
return fmt.Errorf("Update: %w", err)
}
}
return nil
}
// SaveOrUpdateTag must be called for any push actions to add tag
func SaveOrUpdateTag(ctx context.Context, repo *Repository, newRel *Release) error {
rel, err := GetRelease(ctx, repo.ID, newRel.TagName)
if err != nil && !IsErrReleaseNotExist(err) {
return fmt.Errorf("GetRelease: %w", err)
}
if rel == nil {
rel = newRel
if _, err = db.GetEngine(ctx).Insert(rel); err != nil {
return fmt.Errorf("InsertOne: %w", err)
}
} else {
rel.Sha1 = newRel.Sha1
rel.CreatedUnix = newRel.CreatedUnix
rel.NumCommits = newRel.NumCommits
rel.IsDraft = false
if rel.IsTag && newRel.PublisherID > 0 {
rel.PublisherID = newRel.PublisherID
}
if _, err = db.GetEngine(ctx).ID(rel.ID).AllCols().Update(rel); err != nil {
return fmt.Errorf("Update: %w", err)
}
}
return nil
}
// RemapExternalUser ExternalUserRemappable interface
func (r *Release) RemapExternalUser(externalName string, externalID, userID int64) error {
r.OriginalAuthor = externalName

133
modules/repository/repo.go
Unescape Escape View File

@ -9,13 +9,10 @@ import (
"fmt"
"io"
"strings"
"time"
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/lfs"
@ -59,118 +56,6 @@ func SyncRepoTags(ctx context.Context, repoID int64) error {
return SyncReleasesWithTags(ctx, repo, gitRepo)
}
// SyncReleasesWithTags synchronizes release table with repository tags
func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error {
log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
// optimized procedure for pull-mirrors which saves a lot of time (in
// particular for repos with many tags).
if repo.IsMirror {
return pullMirrorReleaseSync(ctx, repo, gitRepo)
}
existingRelTags := make(container.Set[string])
opts := repo_model.FindReleasesOptions{
IncludeDrafts: true,
IncludeTags: true,
ListOptions: db.ListOptions{PageSize: 50},
RepoID: repo.ID,
}
for page := 1; ; page++ {
opts.Page = page
rels, err := db.Find[repo_model.Release](gitRepo.Ctx, opts)
if err != nil {
return fmt.Errorf("unable to GetReleasesByRepoID in Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
}
if len(rels) == 0 {
break
}
for _, rel := range rels {
if rel.IsDraft {
continue
}
commitID, err := gitRepo.GetTagCommitID(rel.TagName)
if err != nil && !git.IsErrNotExist(err) {
return fmt.Errorf("unable to GetTagCommitID for %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err)
}
if git.IsErrNotExist(err) || commitID != rel.Sha1 {
if err := repo_model.PushUpdateDeleteTag(ctx, repo, rel.TagName); err != nil {
return fmt.Errorf("unable to PushUpdateDeleteTag: %q in Repo[%d:%s/%s]: %w", rel.TagName, repo.ID, repo.OwnerName, repo.Name, err)
}
} else {
existingRelTags.Add(strings.ToLower(rel.TagName))
}
}
}
_, err := gitRepo.WalkReferences(git.ObjectTag, 0, 0, func(sha1, refname string) error {
tagName := strings.TrimPrefix(refname, git.TagPrefix)
if existingRelTags.Contains(strings.ToLower(tagName)) {
return nil
}
if err := PushUpdateAddTag(ctx, repo, gitRepo, tagName, sha1, refname); err != nil {
// sometimes, some tags will be sync failed. i.e. https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tag/?h=v2.6.11
// this is a tree object, not a tag object which created before git
log.Error("unable to PushUpdateAddTag: %q to Repo[%d:%s/%s]: %v", tagName, repo.ID, repo.OwnerName, repo.Name, err)
}
return nil
})
return err
}
// PushUpdateAddTag must be called for any push actions to add tag
func PushUpdateAddTag(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tagName, sha1, refname string) error {
tag, err := gitRepo.GetTagWithID(sha1, tagName)
if err != nil {
return fmt.Errorf("unable to GetTag: %w", err)
}
commit, err := gitRepo.GetTagCommit(tag.Name)
if err != nil {
return fmt.Errorf("unable to get tag Commit: %w", err)
}
sig := tag.Tagger
if sig == nil {
sig = commit.Author
}
if sig == nil {
sig = commit.Committer
}
var author *user_model.User
createdAt := time.Unix(1, 0)
if sig != nil {
author, err = user_model.GetUserByEmail(ctx, sig.Email)
if err != nil && !user_model.IsErrUserNotExist(err) {
return fmt.Errorf("unable to GetUserByEmail for %q: %w", sig.Email, err)
}
createdAt = sig.When
}
commitsCount, err := commit.CommitsCount()
if err != nil {
return fmt.Errorf("unable to get CommitsCount: %w", err)
}
rel := repo_model.Release{
RepoID: repo.ID,
TagName: tagName,
LowerTagName: strings.ToLower(tagName),
Sha1: commit.ID.String(),
NumCommits: commitsCount,
CreatedUnix: timeutil.TimeStamp(createdAt.Unix()),
IsTag: true,
}
if author != nil {
rel.PublisherID = author.ID
}
return repo_model.SaveOrUpdateTag(ctx, repo, &rel)
}
// StoreMissingLfsObjectsInRepository downloads missing LFS objects
func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, lfsClient lfs.Client) error {
contentStore := lfs.NewContentStore()
@ -286,18 +171,19 @@ func (shortRelease) TableName() string {
return "release"
}
// pullMirrorReleaseSync is a pull-mirror specific tag<->release table
// SyncReleasesWithTags is a tag<->release table
// synchronization which overwrites all Releases from the repository tags. This
// can be relied on since a pull-mirror is always identical to its
// upstream. Hence, after each sync we want the pull-mirror release set to be
// upstream. Hence, after each sync we want the release set to be
// identical to the upstream tag set. This is much more efficient for
// repositories like https://github.com/vim/vim (with over 13000 tags).
func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error {
log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
tags, numTags, err := gitRepo.GetTagInfos(0, 0)
func SyncReleasesWithTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository) error {
log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name)
tags, _, err := gitRepo.GetTagInfos(0, 0)
if err != nil {
return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
}
var added, deleted, updated int
err = db.WithTx(ctx, func(ctx context.Context) error {
dbReleases, err := db.Find[shortRelease](ctx, repo_model.FindReleasesOptions{
RepoID: repo.ID,
@ -318,9 +204,7 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git
TagName: tag.Name,
LowerTagName: strings.ToLower(tag.Name),
Sha1: tag.Object.String(),
// NOTE: ignored, since NumCommits are unused
// for pull-mirrors (only relevant when
// displaying releases, IsTag: false)
// NOTE: ignored, The NumCommits value is calculated and cached on demand when the UI requires it.
NumCommits: -1,
CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()),
IsTag: true,
@ -349,13 +233,14 @@ func pullMirrorReleaseSync(ctx context.Context, repo *repo_model.Repository, git
return fmt.Errorf("unable to update tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err)
}
}
added, deleted, updated = len(deletes), len(updates), len(inserts)
return nil
})
if err != nil {
return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err)
}
log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags)
log.Trace("SyncReleasesWithTags: %d tags added, %d tags deleted, %d tags updated", added, deleted, updated)
return nil
}

9
services/context/repo.go
Unescape Escape View File

@ -936,6 +936,15 @@ func RepoRefByType(detectRefType git.RefType) func(*Context) {
ctx.ServerError("GetCommitsCount", err)
return
}
if ctx.Repo.RefFullName.IsTag() {
rel, err := repo_model.GetRelease(ctx, ctx.Repo.Repository.ID, ctx.Repo.RefFullName.TagName())
if err == nil && rel.NumCommits <= 0 {
rel.NumCommits = ctx.Repo.CommitsCount
if err := repo_model.UpdateReleaseNumCommits(ctx, rel); err != nil {
log.Error("UpdateReleaseNumCommits", err)
}
}
}
ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount
ctx.Repo.GitRepo.LastCommitCache = git.NewLastCommitCache(ctx.Repo.CommitsCount, ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, cache.GetCache())
}

4
services/repository/migrate.go
Unescape Escape View File

@ -149,9 +149,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
return repo, fmt.Errorf("SyncRepoBranchesWithRepo: %v", err)
}
// if releases migration are not requested, we will sync all tags here
// otherwise, the releases sync will be done out of this function
if !opts.Releases {
// note: this will greatly improve release (tag) sync
// for pull-mirrors with many tags
repo.IsMirror = opts.Mirror
if err = repo_module.SyncReleasesWithTags(ctx, repo, gitRepo); err != nil {
log.Error("Failed to synchronize tags to releases for repository: %v", err)

10
services/repository/push.go
Unescape Escape View File

@ -344,7 +344,7 @@ func pushDeleteBranch(ctx context.Context, repo *repo_model.Repository, pusher *
// PushUpdateAddDeleteTags updates a number of added and delete tags
func PushUpdateAddDeleteTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, addTags, delTags []string) error {
return db.WithTx(ctx, func(ctx context.Context) error {
if err := repo_model.PushUpdateDeleteTagsContext(ctx, repo, delTags); err != nil {
if err := repo_model.PushUpdateDeleteTags(ctx, repo, delTags); err != nil {
return err
}
return pushUpdateAddTags(ctx, repo, gitRepo, addTags)
@ -415,11 +415,6 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo
createdAt = sig.When
}
commitsCount, err := commit.CommitsCount()
if err != nil {
return fmt.Errorf("CommitsCount: %w", err)
}
rel, has := relMap[lowerTag]
parts := strings.SplitN(tag.Message, "\n", 2)
@ -435,7 +430,7 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo
LowerTagName: lowerTag,
Target: "",
Sha1: commit.ID.String(),
NumCommits: commitsCount,
NumCommits: -1, // the commits count will be updated when the UI needs it
Note: note,
IsDraft: false,
IsPrerelease: false,
@ -450,7 +445,6 @@ func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo
} else {
rel.Sha1 = commit.ID.String()
rel.CreatedUnix = timeutil.TimeStamp(createdAt.Unix())
rel.NumCommits = commitsCount
if rel.IsTag {
rel.Title = parts[0]
rel.Note = note

2
templates/repo/release/list.tmpl
Unescape Escape View File

@ -61,7 +61,7 @@
{{if $release.CreatedUnix}}
<span class="time">{{DateUtils.TimeSince $release.CreatedUnix}}</span>
{{end}}
{{if and (not $release.IsDraft) ($.Permission.CanRead ctx.Consts.RepoUnitTypeCode)}}
{{if and (gt $release.NumCommits 0) (not $release.IsDraft) ($.Permission.CanRead ctx.Consts.RepoUnitTypeCode)}}
| <span class="ahead"><a href="{{$.RepoLink}}/compare/{{$release.TagName | PathEscapeSegments}}...{{$release.TargetBehind | PathEscapeSegments}}">{{ctx.Locale.Tr "repo.release.ahead.commits" $release.NumCommitsBehind}}</a> {{ctx.Locale.Tr "repo.release.ahead.target" $release.TargetBehind}}</span>
{{end}}
</p>