sascha
/
gitea
mirror of https://github.com/go-gitea/gitea.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cache GPG keys, emails and users when list commits (#34086) 

When list commits, some of the commits authors are the same at many
situations. But current logic will always fetch the same GPG keys from
database. This PR will cache the GPG keys, emails and users for the
context so that reducing the database queries.

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
pull/34146/head^2
Lunny Xiao 2025-04-09 09:34:38 +07:00 committed by GitHub
parent f8edc29f5d
commit 4a5af4edca
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 65 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
models/asymkey/gpg_key.go
Unescape Escape View File

@ -240,3 +240,10 @@ func DeleteGPGKey(ctx context.Context, doer *user_model.User, id int64) (err err
return committer.Commit()
}
func FindGPGKeyWithSubKeys(ctx context.Context, keyID string) ([]*GPGKey, error) {
return db.Find[GPGKey](ctx, FindGPGKeyOptions{
KeyID: keyID,
IncludeSubKeys: true,
})
}

27
models/user/user.go
Unescape Escape View File

@ -1187,29 +1187,28 @@ func GetUsersByEmails(ctx context.Context, emails []string) (map[string]*User, e
for _, email := range emailAddresses {
userIDs.Add(email.UID)
}
users, err := GetUsersMapByIDs(ctx, userIDs.Values())
if err != nil {
return nil, err
}
results := make(map[string]*User, len(emails))
for _, email := range emailAddresses {
user := users[email.UID]
if user != nil {
if user.KeepEmailPrivate {
results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user
} else {
results[email.Email] = user
if len(userIDs) > 0 {
users, err := GetUsersMapByIDs(ctx, userIDs.Values())
if err != nil {
return nil, err
}
for _, email := range emailAddresses {
user := users[email.UID]
if user != nil {
results[user.GetEmail()] = user
}
}
}
users = make(map[int64]*User, len(needCheckUserNames))
users := make(map[int64]*User, len(needCheckUserNames))
if err := db.GetEngine(ctx).In("lower_name", needCheckUserNames.Values()).Find(&users); err != nil {
return nil, err
}
for _, user := range users {
results[user.LowerName+"@"+setting.Service.NoReplyAddress] = user
results[user.GetPlaceholderEmail()] = user
}
return results, nil
}

8
modules/cache/context.go vendored
Unescape Escape View File

@ -166,15 +166,15 @@ func RemoveContextData(ctx context.Context, tp, key any) {
}
// GetWithContextCache returns the cache value of the given key in the given context.
func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error) {
v := GetContextData(ctx, cacheGroupKey, cacheTargetID)
func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) {
v := GetContextData(ctx, groupKey, targetKey)
if vv, ok := v.(T); ok {
return vv, nil
}
t, err := f()
t, err := f(ctx, targetKey)
if err != nil {
return t, err
}
SetContextData(ctx, cacheGroupKey, cacheTargetID, t)
SetContextData(ctx, groupKey, targetKey, t)
return t, nil
}

3
modules/cache/context_test.go vendored
Unescape Escape View File

@ -4,6 +4,7 @@
package cache
import (
"context"
"testing"
"time"
@ -30,7 +31,7 @@ func TestWithCacheContext(t *testing.T) {
v = GetContextData(ctx, field, "my_config1")
assert.Nil(t, v)
vInt, err := GetWithContextCache(ctx, field, "my_config1", func() (int, error) {
vInt, err := GetWithContextCache(ctx, field, "my_config1", func(context.Context, string) (int, error) {
return 1, nil
})
assert.NoError(t, err)

12
modules/cachegroup/cachegroup.go
Unescape Escape View File

@ -0,0 +1,12 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cachegroup
const (
User = "user"
EmailAvatarLink = "email_avatar_link"
UserEmailAddresses = "user_email_addresses"
GPGKeyWithSubKeys = "gpg_key_with_subkeys"
RepoUserPermission = "repo_user_permission"
)

3
modules/repository/commits.go
Unescape Escape View File

@ -13,6 +13,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cachegroup"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@ -131,7 +132,7 @@ func (pc *PushCommits) ToAPIPayloadCommits(ctx context.Context, repo *repo_model
func (pc *PushCommits) AvatarLink(ctx context.Context, email string) string {
size := avatars.DefaultAvatarPixelSize * setting.Avatar.RenderedSizeFactor
v, _ := cache.GetWithContextCache(ctx, "push_commits", email, func() (string, error) {
v, _ := cache.GetWithContextCache(ctx, cachegroup.EmailAvatarLink, email, func(ctx context.Context, email string) (string, error) {
u, err := user_model.GetUserByEmail(ctx, email)
if err != nil {
if !user_model.IsErrUserNotExist(err) {

5
routers/private/hook_post_receive.go
Unescape Escape View File

@ -14,6 +14,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cachegroup"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/log"
@ -326,9 +327,7 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) {
}
func loadContextCacheUser(ctx context.Context, id int64) (*user_model.User, error) {
return cache.GetWithContextCache(ctx, "hook_post_receive_user", id, func() (*user_model.User, error) {
return user_model.GetUserByID(ctx, id)
})
return cache.GetWithContextCache(ctx, cachegroup.User, id, user_model.GetUserByID)
}
// handlePullRequestMerging handle pull request merging, a pull request action should push at least 1 commit

25
services/asymkey/commit.go
Unescape Escape View File

@ -11,6 +11,8 @@ import (
asymkey_model "code.gitea.io/gitea/models/asymkey"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cachegroup"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@ -115,7 +117,7 @@ func ParseCommitWithSignatureCommitter(ctx context.Context, c *git.Commit, commi
}
}
committerEmailAddresses, _ := user_model.GetEmailAddresses(ctx, committer.ID)
committerEmailAddresses, _ := cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, committer.ID, user_model.GetEmailAddresses)
activated := false
for _, e := range committerEmailAddresses {
if e.IsActivated && strings.EqualFold(e.Email, c.Committer.Email) {
@ -209,10 +211,9 @@ func checkKeyEmails(ctx context.Context, email string, keys ...*asymkey_model.GP
}
if key.Verified && key.OwnerID != 0 {
if uid != key.OwnerID {
userEmails, _ = user_model.GetEmailAddresses(ctx, key.OwnerID)
userEmails, _ = cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, key.OwnerID, user_model.GetEmailAddresses)
uid = key.OwnerID
user = &user_model.User{ID: uid}
_, _ = user_model.GetUser(ctx, user)
user, _ = cache.GetWithContextCache(ctx, cachegroup.User, uid, user_model.GetUserByID)
}
for _, e := range userEmails {
if e.IsActivated && (email == "" || strings.EqualFold(e.Email, email)) {
@ -231,10 +232,7 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s
if keyID == "" {
return nil
}
keys, err := db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{
KeyID: keyID,
IncludeSubKeys: true,
})
keys, err := cache.GetWithContextCache(ctx, cachegroup.GPGKeyWithSubKeys, keyID, asymkey_model.FindGPGKeyWithSubKeys)
if err != nil {
log.Error("GetGPGKeysByKeyID: %v", err)
return &asymkey_model.CommitVerification{
@ -249,10 +247,7 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s
for _, key := range keys {
var primaryKeys []*asymkey_model.GPGKey
if key.PrimaryKeyID != "" {
primaryKeys, err = db.Find[asymkey_model.GPGKey](ctx, asymkey_model.FindGPGKeyOptions{
KeyID: key.PrimaryKeyID,
IncludeSubKeys: true,
})
primaryKeys, err = cache.GetWithContextCache(ctx, cachegroup.GPGKeyWithSubKeys, key.PrimaryKeyID, asymkey_model.FindGPGKeyWithSubKeys)
if err != nil {
log.Error("GetGPGKeysByKeyID: %v", err)
return &asymkey_model.CommitVerification{
@ -272,8 +267,8 @@ func HashAndVerifyForKeyID(ctx context.Context, sig *packet.Signature, payload s
Name: name,
Email: email,
}
if key.OwnerID != 0 {
owner, err := user_model.GetUserByID(ctx, key.OwnerID)
if key.OwnerID > 0 {
owner, err := cache.GetWithContextCache(ctx, cachegroup.User, key.OwnerID, user_model.GetUserByID)
if err == nil {
signer = owner
} else if !user_model.IsErrUserNotExist(err) {
@ -381,7 +376,7 @@ func ParseCommitWithSSHSignature(ctx context.Context, c *git.Commit, committer *
}
}
committerEmailAddresses, err := user_model.GetEmailAddresses(ctx, committer.ID)
committerEmailAddresses, err := cache.GetWithContextCache(ctx, cachegroup.UserEmailAddresses, committer.ID, user_model.GetEmailAddresses)
if err != nil {
log.Error("GetEmailAddresses: %v", err)
}

13
services/convert/pull.go
Unescape Escape View File

@ -14,6 +14,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cachegroup"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/log"
@ -60,14 +61,14 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u
doerID = doer.ID
}
const repoDoerPermCacheKey = "repo_doer_perm_cache"
p, err := cache.GetWithContextCache(ctx, repoDoerPermCacheKey, fmt.Sprintf("%d_%d", pr.BaseRepoID, doerID),
func() (access_model.Permission, error) {
repoUserPerm, err := cache.GetWithContextCache(ctx, cachegroup.RepoUserPermission, fmt.Sprintf("%d-%d", pr.BaseRepoID, doerID),
func(ctx context.Context, _ string) (access_model.Permission, error) {
return access_model.GetUserRepoPermission(ctx, pr.BaseRepo, doer)
})
},
)
if err != nil {
log.Error("GetUserRepoPermission[%d]: %v", pr.BaseRepoID, err)
p.AccessMode = perm.AccessModeNone
repoUserPerm.AccessMode = perm.AccessModeNone
}
apiPullRequest := &api.PullRequest{
@ -107,7 +108,7 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u
Name: pr.BaseBranch,
Ref: pr.BaseBranch,
RepoID: pr.BaseRepoID,
Repository: ToRepo(ctx, pr.BaseRepo, p),
Repository: ToRepo(ctx, pr.BaseRepo, repoUserPerm),
},
Head: &api.PRBranchInfo{
Name: pr.HeadBranch,

10
services/git/commit.go
Unescape Escape View File

@ -17,7 +17,7 @@ import (
)
// ParseCommitsWithSignature checks if signaute of commits are corresponding to users gpg keys.
func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType, isOwnerMemberCollaborator func(*user_model.User) (bool, error)) ([]*asymkey_model.SignCommit, error) {
func ParseCommitsWithSignature(ctx context.Context, repo *repo_model.Repository, oldCommits []*user_model.UserCommit, repoTrustModel repo_model.TrustModelType) ([]*asymkey_model.SignCommit, error) {
newCommits := make([]*asymkey_model.SignCommit, 0, len(oldCommits))
keyMap := map[string]bool{}
@ -47,6 +47,10 @@ func ParseCommitsWithSignature(ctx context.Context, oldCommits []*user_model.Use
Verification: asymkey_service.ParseCommitWithSignatureCommitter(ctx, c.Commit, committer),
}
isOwnerMemberCollaborator := func(user *user_model.User) (bool, error) {
return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID)
}
_ = asymkey_model.CalculateTrustStatus(signCommit.Verification, repoTrustModel, isOwnerMemberCollaborator, &keyMap)
newCommits = append(newCommits, signCommit)
@ -62,11 +66,9 @@ func ConvertFromGitCommit(ctx context.Context, commits []*git.Commit, repo *repo
}
signedCommits, err := ParseCommitsWithSignature(
ctx,
repo,
validatedCommits,
repo.GetTrustModel(),
func(user *user_model.User) (bool, error) {
return repo_model.IsOwnerMemberCollaborator(ctx, repo, user.ID)
},
)
if err != nil {
return nil, err