Merge 1f3f691427 into a440116a16

models/repo/repo_list.go

@@ -778,3 +778,29 @@ func GetUserRepositories(ctx context.Context, opts SearchRepoOptions) (Repositor
 	repos := make(RepositoryList, 0, opts.PageSize)
 	return repos, count, db.SetSessionPagination(sess, &opts).Find(&repos)
 }
+
+// GetRepositoriesIDsByFullNames returns repository IDs by their full names.
+func GetRepositoriesIDsByFullNames(ctx context.Context, fullRepoNames []string) ([]int64, error) {
+	if len(fullRepoNames) == 0 {
+		return nil, nil
+	}
+
+	cond := builder.NewCond()
+	for _, name := range fullRepoNames {
+		ownerName, repoName, ok := strings.Cut(name, "/")
+		if !ok {
+			continue
+		}
+		cond = cond.Or(builder.Eq{"name": repoName, "owner_name": ownerName})
+	}
+
+	repoIDs := make([]int64, 0, len(fullRepoNames))
+	if err := db.GetEngine(ctx).
+		Where(cond).
+		Cols("id").
+		Table("repository").
+		Find(&repoIDs); err != nil {
+		return nil, fmt.Errorf("Find: %w", err)
+	}
+	return repoIDs, nil
+}

modules/indexer/code/internal/indexer.go

@@ -29,6 +29,8 @@ type SearchOptions struct {
 
 	SearchMode indexer.SearchModeType
 
+	NoHighlight bool // If true, return raw content, else highlight the search results
+
 	db.Paginator
 }
 

modules/indexer/code/search.go

@@ -26,7 +26,9 @@ type Result struct {
 }
 
 type ResultLine struct {
-	Num              int
+	Num        int
+	RawContent string // Raw content of the line
+	// FormattedContent is the HTML formatted content of the line, it will only be set if Hightlight is true
 	FormattedContent template.HTML
 }
 
@@ -86,7 +88,7 @@ func HighlightSearchResultCode(filename, language string, lineNums []int, code s
 	return lines
 }
 
-func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Result, error) {
+func searchResult(result *internal.SearchResult, startIndex, endIndex int, noHighlight bool) (*Result, error) {
 	startLineNum := 1 + strings.Count(result.Content[:startIndex], "\n")
 
 	var formattedLinesBuffer bytes.Buffer
@@ -117,6 +119,19 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res
 		index += len(line)
 	}
 
+	var lines []*ResultLine
+	if noHighlight {
+		lines = make([]*ResultLine, len(lineNums))
+		for i, lineNum := range lineNums {
+			lines[i] = &ResultLine{
+				Num:        lineNum,
+				RawContent: contentLines[i],
+			}
+		}
+	} else {
+		lines = HighlightSearchResultCode(result.Filename, result.Language, lineNums, formattedLinesBuffer.String())
+	}
+
 	return &Result{
 		RepoID:      result.RepoID,
 		Filename:    result.Filename,
@@ -124,7 +139,7 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res
 		UpdatedUnix: result.UpdatedUnix,
 		Language:    result.Language,
 		Color:       result.Color,
-		Lines:       HighlightSearchResultCode(result.Filename, result.Language, lineNums, formattedLinesBuffer.String()),
+		Lines:       lines,
 	}, nil
 }
 
@@ -143,7 +158,7 @@ func PerformSearch(ctx context.Context, opts *SearchOptions) (int, []*Result, []
 
 	for i, result := range results {
 		startIndex, endIndex := indices(result.Content, result.StartIndex, result.EndIndex)
-		displayResults[i], err = searchResult(result, startIndex, endIndex)
+		displayResults[i], err = searchResult(result, startIndex, endIndex, opts.NoHighlight)
 		if err != nil {
 			return 0, nil, nil, err
 		}

modules/structs/repo_search.go

@@ -0,0 +1,34 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package structs
+
+// CodeSearchResultLanguage result of top languages count in search results
+type CodeSearchResultLanguage struct {
+	Language string
+	Color    string
+	Count    int
+}
+
+type CodeSearchResultLine struct {
+	LineNumber int    `json:"line_number"`
+	RawContent string `json:"raw_content"`
+}
+
+type CodeSearchResult struct {
+	Name       string `json:"name"`
+	Path       string `json:"path"`
+	Language   string `json:"language"`
+	Color      string
+	Lines      []CodeSearchResultLine
+	Sha        string      `json:"sha"`
+	URL        string      `json:"url"`
+	HTMLURL    string      `json:"html_url"`
+	Repository *Repository `json:"repository"`
+}
+
+type CodeSearchResults struct {
+	TotalCount int64                      `json:"total_count"`
+	Items      []CodeSearchResult         `json:"items"`
+	Languages  []CodeSearchResultLanguage `json:"languages,omitempty"`
+}

routers/api/v1/api.go

@@ -90,6 +90,7 @@ import (
 	"code.gitea.io/gitea/routers/api/v1/org"
 	"code.gitea.io/gitea/routers/api/v1/packages"
 	"code.gitea.io/gitea/routers/api/v1/repo"
+	"code.gitea.io/gitea/routers/api/v1/repo/code"
 	"code.gitea.io/gitea/routers/api/v1/settings"
 	"code.gitea.io/gitea/routers/api/v1/user"
 	"code.gitea.io/gitea/routers/common"
@@ -1755,6 +1756,10 @@ func Routes() *web.Router {
 		m.Group("/topics", func() {
 			m.Get("/search", repo.TopicSearch)
 		}, tokenRequiresScopes(auth_model.AccessTokenScopeCategoryRepository))
+
+		m.Group("/search", func() {
+			m.Get("/code", code.GlobalSearch)
+		}, tokenRequiresScopes(auth_model.AccessTokenScopeCategoryRepository))
 	}, sudo())
 
 	return m

routers/api/v1/repo/code/search.go

@@ -0,0 +1,197 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package code
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+	"slices"
+
+	access_model "code.gitea.io/gitea/models/perm/access"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/container"
+	"code.gitea.io/gitea/modules/indexer"
+	"code.gitea.io/gitea/modules/indexer/code"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/routers/api/v1/utils"
+	"code.gitea.io/gitea/services/context"
+	"code.gitea.io/gitea/services/convert"
+)
+
+// GlobalSearch search codes in all accessible repositories with the given keyword.
+func GlobalSearch(ctx *context.APIContext) {
+	// swagger:operation GET /search/code search GlobalSearch
+	// ---
+	// summary: Search for repositories
+	// produces:
+	// - application/json
+	// parameters:
+	// - name: q
+	//   in: query
+	//   description: keyword
+	//   type: string
+	// - name: repo
+	//   in: query
+	//   description: multiple repository names to search in
+	//   type: string
+	//   collectionFormat: multi
+	// - name: mode
+	//   in: query
+	//   description: include search of keyword within repository description
+	//   type: string
+	//   enum: [exact, words, fuzzy, regexp]
+	// - name: language
+	//   in: query
+	//   description: filter by programming language
+	//   type: integer
+	//   format: int64
+	// - name: page
+	//   in: query
+	//   description: page number of results to return (1-based)
+	//   type: integer
+	// - name: limit
+	//   in: query
+	//   description: page size of results
+	//   type: integer
+	// responses:
+	//   "200":
+	//     "$ref": "#/responses/CodeSearchResults"
+	//   "422":
+	//     "$ref": "#/responses/validationError"
+
+	if !setting.Indexer.RepoIndexerEnabled {
+		ctx.APIError(http.StatusBadRequest, "Repository indexing is disabled")
+		return
+	}
+
+	q := ctx.FormTrim("q")
+	if q == "" {
+		ctx.APIError(http.StatusUnprocessableEntity, "Query cannot be empty")
+		return
+	}
+
+	var (
+		accessibleRepoIDs []int64
+		err               error
+		isAdmin           bool
+	)
+	if ctx.Doer != nil {
+		isAdmin = ctx.Doer.IsAdmin
+	}
+
+	// guest user or non-admin user
+	if ctx.Doer == nil || !isAdmin {
+		accessibleRepoIDs, err = repo_model.FindUserCodeAccessibleRepoIDs(ctx, ctx.Doer)
+		if err != nil {
+			ctx.APIErrorInternal(err)
+			return
+		}
+	}
+
+	repoNames := ctx.FormStrings("repo")
+	searchRepoIDs := make([]int64, 0, len(repoNames))
+	if len(repoNames) > 0 {
+		var err error
+		searchRepoIDs, err = repo_model.GetRepositoriesIDsByFullNames(ctx, repoNames)
+		if err != nil {
+			ctx.APIErrorInternal(err)
+			return
+		}
+	}
+	if len(searchRepoIDs) > 0 {
+		for i := 0; i < len(searchRepoIDs); i++ {
+			if !slices.Contains(accessibleRepoIDs, searchRepoIDs[i]) {
+				searchRepoIDs = append(searchRepoIDs[:i], searchRepoIDs[i+1:]...)
+				i--
+			}
+		}
+	}
+	if len(searchRepoIDs) > 0 {
+		accessibleRepoIDs = searchRepoIDs
+	}
+
+	searchMode := indexer.SearchModeType(ctx.FormString("mode"))
+	listOpts := utils.GetListOptions(ctx)
+
+	total, results, languages, err := code.PerformSearch(ctx, &code.SearchOptions{
+		Keyword:     q,
+		RepoIDs:     accessibleRepoIDs,
+		Language:    ctx.FormString("language"),
+		SearchMode:  searchMode,
+		Paginator:   &listOpts,
+		NoHighlight: true, // Default to no highlighting for performance, we don't need to highlight in the API search results
+	})
+	if err != nil {
+		ctx.APIErrorInternal(err)
+		return
+	}
+
+	ctx.SetTotalCountHeader(int64(total))
+	searchResults := structs.CodeSearchResults{
+		TotalCount: int64(total),
+	}
+
+	for _, lang := range languages {
+		searchResults.Languages = append(searchResults.Languages, structs.CodeSearchResultLanguage{
+			Language: lang.Language,
+			Color:    lang.Color,
+			Count:    lang.Count,
+		})
+	}
+
+	repoIDs := make(container.Set[int64], len(results))
+	for _, result := range results {
+		repoIDs.Add(result.RepoID)
+	}
+
+	repos, err := repo_model.GetRepositoriesMapByIDs(ctx, repoIDs.Values())
+	if err != nil {
+		ctx.APIErrorInternal(err)
+		return
+	}
+
+	permissions := make(map[int64]access_model.Permission)
+	for _, repo := range repos {
+		permission, err := access_model.GetUserRepoPermission(ctx, repo, ctx.Doer)
+		if err != nil {
+			ctx.APIErrorInternal(err)
+			return
+		}
+		permissions[repo.ID] = permission
+	}
+
+	for _, result := range results {
+		repo, ok := repos[result.RepoID]
+		if !ok {
+			log.Error("Repository with ID %d not found for search result: %v", result.RepoID, result)
+			continue
+		}
+
+		apiURL := fmt.Sprintf("%s/contents/%s?ref=%s", repo.APIURL(), util.PathEscapeSegments(result.Filename), url.PathEscape(result.CommitID))
+		htmlURL := fmt.Sprintf("%s/blob/%s/%s", repo.HTMLURL(), url.PathEscape(result.CommitID), util.PathEscapeSegments(result.Filename))
+		ret := structs.CodeSearchResult{
+			Name:       path.Base(result.Filename),
+			Path:       result.Filename,
+			Sha:        result.CommitID,
+			URL:        apiURL,
+			HTMLURL:    htmlURL,
+			Language:   result.Language,
+			Repository: convert.ToRepo(ctx, repo, permissions[repo.ID]),
+		}
+		for _, line := range result.Lines {
+			ret.Lines = append(ret.Lines, structs.CodeSearchResultLine{
+				LineNumber: line.Num,
+				RawContent: line.RawContent,
+			})
+		}
+		searchResults.Items = append(searchResults.Items, ret)
+	}
+
+	ctx.JSON(200, searchResults)
+}

routers/api/v1/swagger/search.go

@@ -0,0 +1,15 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package swagger
+
+import (
+	api "code.gitea.io/gitea/modules/structs"
+)
+
+// CodeSearchResults
+// swagger:response CodeSearchResults
+type swaggerResponseCodeSearchResults struct {
+	// in:body
+	Body api.CodeSearchResults `json:"body"`
+}

templates/swagger/v1_json.tmpl

@@ -17604,6 +17604,72 @@
         }
       }
     },
+    "/search/code": {
+      "get": {
+        "produces": [
+          "application/json"
+        ],
+        "tags": [
+          "search"
+        ],
+        "summary": "Search for repositories",
+        "operationId": "GlobalSearch",
+        "parameters": [
+          {
+            "type": "string",
+            "description": "keyword",
+            "name": "q",
+            "in": "query"
+          },
+          {
+            "type": "string",
+            "collectionFormat": "multi",
+            "description": "multiple repository names to search in",
+            "name": "repo",
+            "in": "query"
+          },
+          {
+            "enum": [
+              "exact",
+              "words",
+              "fuzzy",
+              "regexp"
+            ],
+            "type": "string",
+            "description": "include search of keyword within repository description",
+            "name": "mode",
+            "in": "query"
+          },
+          {
+            "type": "integer",
+            "format": "int64",
+            "description": "filter by programming language",
+            "name": "language",
+            "in": "query"
+          },
+          {
+            "type": "integer",
+            "description": "page number of results to return (1-based)",
+            "name": "page",
+            "in": "query"
+          },
+          {
+            "type": "integer",
+            "description": "page size of results",
+            "name": "limit",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "$ref": "#/responses/CodeSearchResults"
+          },
+          "422": {
+            "$ref": "#/responses/validationError"
+          }
+        }
+      }
+    },
     "/settings/api": {
       "get": {
         "produces": [
@@ -22211,6 +22277,105 @@
       },
       "x-go-package": "code.gitea.io/gitea/modules/structs"
     },
+    "CodeSearchResult": {
+      "type": "object",
+      "properties": {
+        "Color": {
+          "type": "string"
+        },
+        "Lines": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/CodeSearchResultLine"
+          }
+        },
+        "html_url": {
+          "type": "string",
+          "x-go-name": "HTMLURL"
+        },
+        "language": {
+          "type": "string",
+          "x-go-name": "Language"
+        },
+        "name": {
+          "type": "string",
+          "x-go-name": "Name"
+        },
+        "path": {
+          "type": "string",
+          "x-go-name": "Path"
+        },
+        "repository": {
+          "$ref": "#/definitions/Repository"
+        },
+        "sha": {
+          "type": "string",
+          "x-go-name": "Sha"
+        },
+        "url": {
+          "type": "string",
+          "x-go-name": "URL"
+        }
+      },
+      "x-go-package": "code.gitea.io/gitea/modules/structs"
+    },
+    "CodeSearchResultLanguage": {
+      "description": "CodeSearchResultLanguage result of top languages count in search results",
+      "type": "object",
+      "properties": {
+        "Color": {
+          "type": "string"
+        },
+        "Count": {
+          "type": "integer",
+          "format": "int64"
+        },
+        "Language": {
+          "type": "string"
+        }
+      },
+      "x-go-package": "code.gitea.io/gitea/modules/structs"
+    },
+    "CodeSearchResultLine": {
+      "type": "object",
+      "properties": {
+        "line_number": {
+          "type": "integer",
+          "format": "int64",
+          "x-go-name": "LineNumber"
+        },
+        "raw_content": {
+          "type": "string",
+          "x-go-name": "RawContent"
+        }
+      },
+      "x-go-package": "code.gitea.io/gitea/modules/structs"
+    },
+    "CodeSearchResults": {
+      "type": "object",
+      "properties": {
+        "items": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/CodeSearchResult"
+          },
+          "x-go-name": "Items"
+        },
+        "languages": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/CodeSearchResultLanguage"
+          },
+          "x-go-name": "Languages"
+        },
+        "total_count": {
+          "type": "integer",
+          "format": "int64",
+          "x-go-name": "TotalCount"
+        }
+      },
+      "x-go-package": "code.gitea.io/gitea/modules/structs"
+    },
     "CombinedStatus": {
       "description": "CombinedStatus holds the combined state of several statuses for a single commit",
       "type": "object",
@@ -29446,6 +29611,12 @@
         }
       }
     },
+    "CodeSearchResults": {
+      "description": "CodeSearchResults",
+      "schema": {
+        "$ref": "#/definitions/CodeSearchResults"
+      }
+    },
     "CombinedStatus": {
       "description": "CombinedStatus",
       "schema": {

tests/integration/api_search_code_test.go

@@ -0,0 +1,57 @@
+// Copyright 2025 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package integration
+
+import (
+	"net/http"
+	"testing"
+
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/unittest"
+	"code.gitea.io/gitea/modules/gitrepo"
+	"code.gitea.io/gitea/modules/setting"
+	api "code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/tests"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAPISearchCodeNotLogin(t *testing.T) {
+	defer tests.PrepareTestEnv(t)()
+
+	// test with no keyword
+	req := NewRequest(t, "GET", "/api/v1/search/code")
+	MakeRequest(t, req, http.StatusUnprocessableEntity)
+
+	req = NewRequest(t, "GET", "/api/v1/search/code?q=Description")
+	resp := MakeRequest(t, req, http.StatusOK)
+
+	var apiCodeSearchResults api.CodeSearchResults
+	DecodeJSON(t, resp, &apiCodeSearchResults)
+	assert.Equal(t, int64(1), apiCodeSearchResults.TotalCount)
+	assert.Len(t, apiCodeSearchResults.Items, 1)
+	assert.Equal(t, "README.md", apiCodeSearchResults.Items[0].Name)
+	assert.Equal(t, "README.md", apiCodeSearchResults.Items[0].Path)
+	assert.Equal(t, "Markdown", apiCodeSearchResults.Items[0].Language)
+	assert.Len(t, apiCodeSearchResults.Items[0].Lines, 2)
+	assert.Equal(t, "\n", apiCodeSearchResults.Items[0].Lines[0].RawContent)
+	assert.Equal(t, "Description for repo1", apiCodeSearchResults.Items[0].Lines[1].RawContent)
+
+	repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
+	gitRepo1, err := gitrepo.OpenRepository(t.Context(), repo1)
+	assert.NoError(t, err)
+	defer gitRepo1.Close()
+
+	commitID, err := gitRepo1.GetBranchCommitID(repo1.DefaultBranch)
+	assert.NoError(t, err)
+
+	assert.Equal(t, setting.AppURL+"api/v1/repos/user2/repo1/contents/README.md?ref="+commitID, apiCodeSearchResults.Items[0].URL)
+	assert.Equal(t, setting.AppURL+"user2/repo1/blob/"+commitID+"/README.md", apiCodeSearchResults.Items[0].HTMLURL)
+
+	assert.Equal(t, int64(1), apiCodeSearchResults.Items[0].Repository.ID)
+
+	assert.Len(t, apiCodeSearchResults.Languages, 1)
+	assert.Equal(t, "Markdown", apiCodeSearchResults.Languages[0].Language)
+	assert.Equal(t, 1, apiCodeSearchResults.Languages[0].Count)
+}