diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 53e25a8c3b..b4b6b830b2 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2439,6 +2439,8 @@ LEVEL = Info ;DEFAULT_GIT_TREES_PER_PAGE = 1000 ;; Default max size of a blob returned by the blobs API (default is 10MiB) ;DEFAULT_MAX_BLOB_SIZE = 10485760 +;; Default max combined size of all blobs returned by the files API (default is 100MiB) +;DEFAULT_MAX_RESPONSE_SIZE = 104857600 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/modules/git/repo_object.go b/modules/git/repo_object.go index 1d34305270..08e0413311 100644 --- a/modules/git/repo_object.go +++ b/modules/git/repo_object.go @@ -85,17 +85,3 @@ func (repo *Repository) hashObject(reader io.Reader, save bool) (string, error) } return strings.TrimSpace(stdout.String()), nil } - -// GetRefType gets the type of the ref based on the string -func (repo *Repository) GetRefType(ref string) ObjectType { - if repo.IsTagExist(ref) { - return ObjectTag - } else if repo.IsBranchExist(ref) { - return ObjectBranch - } else if repo.IsCommitExist(ref) { - return ObjectCommit - } else if _, err := repo.GetBlob(ref); err == nil { - return ObjectBlob - } - return ObjectType("invalid") -} diff --git a/modules/setting/api.go b/modules/setting/api.go index c36f05cfd1..cdad474cb9 100644 --- a/modules/setting/api.go +++ b/modules/setting/api.go @@ -18,6 +18,7 @@ var API = struct { DefaultPagingNum int DefaultGitTreesPerPage int DefaultMaxBlobSize int64 + DefaultMaxResponseSize int64 }{ EnableSwagger: true, SwaggerURL: "", @@ -25,6 +26,7 @@ var API = struct { DefaultPagingNum: 30, DefaultGitTreesPerPage: 1000, DefaultMaxBlobSize: 10485760, + DefaultMaxResponseSize: 104857600, } func loadAPIFrom(rootCfg ConfigProvider) { diff --git a/modules/structs/git_blob.go b/modules/structs/git_blob.go index 96c7a271a9..96770cc62e 100644 --- a/modules/structs/git_blob.go +++ b/modules/structs/git_blob.go @@ -5,9 +5,9 @@ package structs // GitBlobResponse represents a git blob type GitBlobResponse struct { - Content string `json:"content"` - Encoding string `json:"encoding"` - URL string `json:"url"` - SHA string `json:"sha"` - Size int64 `json:"size"` + Content *string `json:"content"` + Encoding *string `json:"encoding"` + URL string `json:"url"` + SHA string `json:"sha"` + Size int64 `json:"size"` } diff --git a/modules/structs/repo_file.go b/modules/structs/repo_file.go index 0cd88b3703..b0e0bd979e 100644 --- a/modules/structs/repo_file.go +++ b/modules/structs/repo_file.go @@ -176,3 +176,8 @@ type FileDeleteResponse struct { Commit *FileCommitResponse `json:"commit"` Verification *PayloadCommitVerification `json:"verification"` } + +// GetFilesOptions options for retrieving metadate and content of multiple files +type GetFilesOptions struct { + Files []string `json:"files" binding:"Required"` +} diff --git a/modules/structs/settings.go b/modules/structs/settings.go index e48b1a493d..59176210e6 100644 --- a/modules/structs/settings.go +++ b/modules/structs/settings.go @@ -26,6 +26,7 @@ type GeneralAPISettings struct { DefaultPagingNum int `json:"default_paging_num"` DefaultGitTreesPerPage int `json:"default_git_trees_per_page"` DefaultMaxBlobSize int64 `json:"default_max_blob_size"` + DefaultMaxResponseSize int64 `json:"default_max_response_size"` } // GeneralAttachmentSettings contains global Attachment settings exposed by API diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 58d0891ea5..58ae8ec90a 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -1389,14 +1389,17 @@ func Routes() *web.Router { m.Post("/diffpatch", reqRepoWriter(unit.TypeCode), reqToken(), bind(api.ApplyDiffPatchFileOptions{}), mustNotBeArchived, repo.ApplyDiffPatch) m.Group("/contents", func() { m.Get("", repo.GetContentsList) - m.Post("", reqToken(), bind(api.ChangeFilesOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.ChangeFiles) m.Get("/*", repo.GetContents) + m.Post("", reqToken(), bind(api.ChangeFilesOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.ChangeFiles) m.Group("/*", func() { m.Post("", bind(api.CreateFileOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.CreateFile) m.Put("", bind(api.UpdateFileOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.UpdateFile) m.Delete("", bind(api.DeleteFileOptions{}), reqRepoBranchWriter, mustNotBeArchived, repo.DeleteFile) }, reqToken()) - }, reqRepoReader(unit.TypeCode)) + }, reqRepoReader(unit.TypeCode), context.ReferencesGitRepo()) + m.Combo("/file-contents", reqRepoReader(unit.TypeCode), context.ReferencesGitRepo()). + Get(repo.GetFileContentsGet). + Post(bind(api.GetFilesOptions{}), repo.GetFileContentsPost) // POST method requires "write" permission, so we also support "GET" method above m.Get("/signing-key.gpg", misc.SigningKey) m.Group("/topics", func() { m.Combo("").Get(repo.ListTopics). diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index d8e9fde2c0..f40d39a251 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -16,16 +16,18 @@ import ( git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/httpcache" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/api/v1/utils" "code.gitea.io/gitea/routers/common" "code.gitea.io/gitea/services/context" pull_service "code.gitea.io/gitea/services/pull" @@ -375,7 +377,7 @@ func GetEditorconfig(ctx *context.APIContext) { // required: true // - name: ref // in: query - // description: "The name of the commit/branch/tag. Default the repository’s default branch (usually master)" + // description: "The name of the commit/branch/tag. Default to the repository’s default branch." // type: string // required: false // responses: @@ -410,11 +412,6 @@ func canWriteFiles(ctx *context.APIContext, branch string) bool { !ctx.Repo.Repository.IsArchived } -// canReadFiles returns true if repository is readable and user has proper access level. -func canReadFiles(r *context.Repository) bool { - return r.Permission.CanRead(unit.TypeCode) -} - func base64Reader(s string) (io.ReadSeeker, error) { b, err := base64.StdEncoding.DecodeString(s) if err != nil { @@ -894,6 +891,17 @@ func DeleteFile(ctx *context.APIContext) { } } +func resolveRefCommit(ctx *context.APIContext, ref string, minCommitIDLen ...int) *utils.RefCommit { + ref = util.IfZero(ref, ctx.Repo.Repository.DefaultBranch) + refCommit, err := utils.ResolveRefCommit(ctx, ctx.Repo.Repository, ref, minCommitIDLen...) + if errors.Is(err, util.ErrNotExist) { + ctx.APIErrorNotFound(err) + } else if err != nil { + ctx.APIErrorInternal(err) + } + return refCommit +} + // GetContents Get the metadata and contents (if a file) of an entry in a repository, or a list of entries if a dir func GetContents(ctx *context.APIContext) { // swagger:operation GET /repos/{owner}/{repo}/contents/{filepath} repository repoGetContents @@ -919,7 +927,7 @@ func GetContents(ctx *context.APIContext) { // required: true // - name: ref // in: query - // description: "The name of the commit/branch/tag. Default the repository’s default branch (usually master)" + // description: "The name of the commit/branch/tag. Default to the repository’s default branch." // type: string // required: false // responses: @@ -928,18 +936,13 @@ func GetContents(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - if !canReadFiles(ctx.Repo) { - ctx.APIErrorInternal(repo_model.ErrUserDoesNotHaveAccessToRepo{ - UserID: ctx.Doer.ID, - RepoName: ctx.Repo.Repository.LowerName, - }) + treePath := ctx.PathParam("*") + refCommit := resolveRefCommit(ctx, ctx.FormTrim("ref")) + if ctx.Written() { return } - treePath := ctx.PathParam("*") - ref := ctx.FormTrim("ref") - - if fileList, err := files_service.GetContentsOrList(ctx, ctx.Repo.Repository, treePath, ref); err != nil { + if fileList, err := files_service.GetContentsOrList(ctx, ctx.Repo.Repository, refCommit, treePath); err != nil { if git.IsErrNotExist(err) { ctx.APIErrorNotFound("GetContentsOrList", err) return @@ -970,7 +973,7 @@ func GetContentsList(ctx *context.APIContext) { // required: true // - name: ref // in: query - // description: "The name of the commit/branch/tag. Default the repository’s default branch (usually master)" + // description: "The name of the commit/branch/tag. Default to the repository’s default branch." // type: string // required: false // responses: @@ -982,3 +985,102 @@ func GetContentsList(ctx *context.APIContext) { // same as GetContents(), this function is here because swagger fails if path is empty in GetContents() interface GetContents(ctx) } + +func GetFileContentsGet(ctx *context.APIContext) { + // swagger:operation GET /repos/{owner}/{repo}/file-contents repository repoGetFileContents + // --- + // summary: Get the metadata and contents of requested files + // description: See the POST method. This GET method supports to use JSON encoded request body in query parameter. + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: ref + // in: query + // description: "The name of the commit/branch/tag. Default to the repository’s default branch." + // type: string + // required: false + // - name: body + // in: query + // description: "The JSON encoded body (see the POST request): {\"files\": [\"filename1\", \"filename2\"]}" + // type: string + // required: true + // responses: + // "200": + // "$ref": "#/responses/ContentsListResponse" + // "404": + // "$ref": "#/responses/notFound" + + // POST method requires "write" permission, so we also support this "GET" method + handleGetFileContents(ctx) +} + +func GetFileContentsPost(ctx *context.APIContext) { + // swagger:operation POST /repos/{owner}/{repo}/file-contents repository repoGetFileContentsPost + // --- + // summary: Get the metadata and contents of requested files + // description: Uses automatic pagination based on default page size and + // max response size and returns the maximum allowed number of files. + // Files which could not be retrieved are null. Files which are too large + // are being returned with `encoding == null`, `content == null` and `size > 0`, + // they can be requested separately by using the `download_url`. + // produces: + // - application/json + // parameters: + // - name: owner + // in: path + // description: owner of the repo + // type: string + // required: true + // - name: repo + // in: path + // description: name of the repo + // type: string + // required: true + // - name: ref + // in: query + // description: "The name of the commit/branch/tag. Default to the repository’s default branch." + // type: string + // required: false + // - name: body + // in: body + // required: true + // schema: + // "$ref": "#/definitions/GetFilesOptions" + // responses: + // "200": + // "$ref": "#/responses/ContentsListResponse" + // "404": + // "$ref": "#/responses/notFound" + + // This is actually a "read" request, but we need to accept a "files" list, then POST method seems easy to use. + // But the permission system requires that the caller must have "write" permission to use POST method. + // At the moment there is no other way to get around the permission check, so there is a "GET" workaround method above. + handleGetFileContents(ctx) +} + +func handleGetFileContents(ctx *context.APIContext) { + opts, ok := web.GetForm(ctx).(*api.GetFilesOptions) + if !ok { + err := json.Unmarshal(util.UnsafeStringToBytes(ctx.FormString("body")), &opts) + if err != nil { + ctx.APIError(http.StatusBadRequest, "invalid body parameter") + return + } + } + refCommit := resolveRefCommit(ctx, ctx.FormTrim("ref")) + if ctx.Written() { + return + } + filesResponse := files_service.GetContentsListFromTreePaths(ctx, ctx.Repo.Repository, refCommit, opts.Files) + ctx.JSON(http.StatusOK, util.SliceNilAsEmpty(filesResponse)) +} diff --git a/routers/api/v1/repo/status.go b/routers/api/v1/repo/status.go index e1dbb25865..756adcf3a3 100644 --- a/routers/api/v1/repo/status.go +++ b/routers/api/v1/repo/status.go @@ -177,20 +177,14 @@ func GetCommitStatusesByRef(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - filter := utils.ResolveRefOrSha(ctx, ctx.PathParam("ref")) + refCommit := resolveRefCommit(ctx, ctx.PathParam("ref"), 7) if ctx.Written() { return } - - getCommitStatuses(ctx, filter) // By default filter is maybe the raw SHA + getCommitStatuses(ctx, refCommit.CommitID) } -func getCommitStatuses(ctx *context.APIContext, sha string) { - if len(sha) == 0 { - ctx.APIError(http.StatusBadRequest, nil) - return - } - sha = utils.MustConvertToSHA1(ctx.Base, ctx.Repo, sha) +func getCommitStatuses(ctx *context.APIContext, commitID string) { repo := ctx.Repo.Repository listOptions := utils.GetListOptions(ctx) @@ -198,12 +192,12 @@ func getCommitStatuses(ctx *context.APIContext, sha string) { statuses, maxResults, err := db.FindAndCount[git_model.CommitStatus](ctx, &git_model.CommitStatusOptions{ ListOptions: listOptions, RepoID: repo.ID, - SHA: sha, + SHA: commitID, SortType: ctx.FormTrim("sort"), State: ctx.FormTrim("state"), }) if err != nil { - ctx.APIErrorInternal(fmt.Errorf("GetCommitStatuses[%s, %s, %d]: %w", repo.FullName(), sha, ctx.FormInt("page"), err)) + ctx.APIErrorInternal(fmt.Errorf("GetCommitStatuses[%s, %s, %d]: %w", repo.FullName(), commitID, ctx.FormInt("page"), err)) return } @@ -257,16 +251,16 @@ func GetCombinedCommitStatusByRef(ctx *context.APIContext) { // "404": // "$ref": "#/responses/notFound" - sha := utils.ResolveRefOrSha(ctx, ctx.PathParam("ref")) + refCommit := resolveRefCommit(ctx, ctx.PathParam("ref"), 7) if ctx.Written() { return } repo := ctx.Repo.Repository - statuses, count, err := git_model.GetLatestCommitStatus(ctx, repo.ID, sha, utils.GetListOptions(ctx)) + statuses, count, err := git_model.GetLatestCommitStatus(ctx, repo.ID, refCommit.Commit.ID.String(), utils.GetListOptions(ctx)) if err != nil { - ctx.APIErrorInternal(fmt.Errorf("GetLatestCommitStatus[%s, %s]: %w", repo.FullName(), sha, err)) + ctx.APIErrorInternal(fmt.Errorf("GetLatestCommitStatus[%s, %s]: %w", repo.FullName(), refCommit.CommitID, err)) return } diff --git a/routers/api/v1/settings/settings.go b/routers/api/v1/settings/settings.go index 0ee81b96d5..94fbadeab0 100644 --- a/routers/api/v1/settings/settings.go +++ b/routers/api/v1/settings/settings.go @@ -43,6 +43,7 @@ func GetGeneralAPISettings(ctx *context.APIContext) { DefaultPagingNum: setting.API.DefaultPagingNum, DefaultGitTreesPerPage: setting.API.DefaultGitTreesPerPage, DefaultMaxBlobSize: setting.API.DefaultMaxBlobSize, + DefaultMaxResponseSize: setting.API.DefaultMaxResponseSize, }) } diff --git a/routers/api/v1/swagger/options.go b/routers/api/v1/swagger/options.go index d5e042f8fa..bafd5e04a2 100644 --- a/routers/api/v1/swagger/options.go +++ b/routers/api/v1/swagger/options.go @@ -118,6 +118,9 @@ type swaggerParameterBodies struct { // in:body EditAttachmentOptions api.EditAttachmentOptions + // in:body + GetFilesOptions api.GetFilesOptions + // in:body ChangeFilesOptions api.ChangeFilesOptions diff --git a/routers/api/v1/utils/git.go b/routers/api/v1/utils/git.go index 7d276d9d98..1cfe01a639 100644 --- a/routers/api/v1/utils/git.go +++ b/routers/api/v1/utils/git.go @@ -4,48 +4,48 @@ package utils import ( - gocontext "context" "errors" - "fmt" - "net/http" + repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" - "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/reqctx" "code.gitea.io/gitea/services/context" ) -// ResolveRefOrSha resolve ref to sha if exist -func ResolveRefOrSha(ctx *context.APIContext, ref string) string { - if len(ref) == 0 { - ctx.APIError(http.StatusBadRequest, nil) - return "" +type RefCommit struct { + InputRef string + RefName git.RefName + Commit *git.Commit + CommitID string +} + +// ResolveRefCommit resolve ref to a commit if exist +func ResolveRefCommit(ctx reqctx.RequestContext, repo *repo_model.Repository, inputRef string, minCommitIDLen ...int) (_ *RefCommit, err error) { + gitRepo, err := gitrepo.RepositoryFromRequestContextOrOpen(ctx, repo) + if err != nil { + return nil, err } - - sha := ref - // Search branches and tags - for _, refType := range []string{"heads", "tags"} { - refSHA, lastMethodName, err := searchRefCommitByType(ctx, refType, ref) - if err != nil { - ctx.APIErrorInternal(fmt.Errorf("%s: %w", lastMethodName, err)) - return "" - } - if refSHA != "" { - sha = refSHA - break - } + refCommit := RefCommit{InputRef: inputRef} + if gitrepo.IsBranchExist(ctx, repo, inputRef) { + refCommit.RefName = git.RefNameFromBranch(inputRef) + } else if gitrepo.IsTagExist(ctx, repo, inputRef) { + refCommit.RefName = git.RefNameFromTag(inputRef) + } else if git.IsStringLikelyCommitID(git.ObjectFormatFromName(repo.ObjectFormatName), inputRef, minCommitIDLen...) { + refCommit.RefName = git.RefNameFromCommit(inputRef) } - - sha = MustConvertToSHA1(ctx, ctx.Repo, sha) - - if ctx.Repo.GitRepo != nil { - err := ctx.Repo.GitRepo.AddLastCommitCache(ctx.Repo.Repository.GetCommitsCountCacheKey(ref, ref != sha), ctx.Repo.Repository.FullName(), sha) - if err != nil { - log.Error("Unable to get commits count for %s in %s. Error: %v", sha, ctx.Repo.Repository.FullName(), err) - } + if refCommit.RefName == "" { + return nil, git.ErrNotExist{ID: inputRef} } + if refCommit.Commit, err = gitRepo.GetCommit(refCommit.RefName.String()); err != nil { + return nil, err + } + refCommit.CommitID = refCommit.Commit.ID.String() + return &refCommit, nil +} - return sha +func NewRefCommit(refName git.RefName, commit *git.Commit) *RefCommit { + return &RefCommit{InputRef: refName.ShortName(), RefName: refName, Commit: commit, CommitID: commit.ID.String()} } // GetGitRefs return git references based on filter @@ -59,42 +59,3 @@ func GetGitRefs(ctx *context.APIContext, filter string) ([]*git.Reference, strin refs, err := ctx.Repo.GitRepo.GetRefsFiltered(filter) return refs, "GetRefsFiltered", err } - -func searchRefCommitByType(ctx *context.APIContext, refType, filter string) (string, string, error) { - refs, lastMethodName, err := GetGitRefs(ctx, refType+"/"+filter) // Search by type - if err != nil { - return "", lastMethodName, err - } - if len(refs) > 0 { - return refs[0].Object.String(), "", nil // Return found SHA - } - return "", "", nil -} - -// ConvertToObjectID returns a full-length SHA1 from a potential ID string -func ConvertToObjectID(ctx gocontext.Context, repo *context.Repository, commitID string) (git.ObjectID, error) { - objectFormat := repo.GetObjectFormat() - if len(commitID) == objectFormat.FullLength() && objectFormat.IsValid(commitID) { - sha, err := git.NewIDFromString(commitID) - if err == nil { - return sha, nil - } - } - - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo.Repository) - if err != nil { - return objectFormat.EmptyObjectID(), fmt.Errorf("RepositoryFromContextOrOpen: %w", err) - } - defer closer.Close() - - return gitRepo.ConvertToGitID(commitID) -} - -// MustConvertToSHA1 returns a full-length SHA1 string from a potential ID string, or returns origin input if it can't convert to SHA1 -func MustConvertToSHA1(ctx gocontext.Context, repo *context.Repository, commitID string) string { - sha, err := ConvertToObjectID(ctx, repo, commitID) - if err != nil { - return commitID - } - return sha.String() -} diff --git a/services/context/api.go b/services/context/api.go index 6e3b635ce3..041e69ee64 100644 --- a/services/context/api.go +++ b/services/context/api.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/log" @@ -245,7 +246,7 @@ func APIContexter() func(http.Handler) http.Handler { // String will replace message, errors will be added to a slice func (ctx *APIContext) APIErrorNotFound(objs ...any) { message := ctx.Locale.TrString("error.not_found") - var errors []string + var errs []string for _, obj := range objs { // Ignore nil if obj == nil { @@ -253,7 +254,7 @@ func (ctx *APIContext) APIErrorNotFound(objs ...any) { } if err, ok := obj.(error); ok { - errors = append(errors, err.Error()) + errs = append(errs, err.Error()) } else { message = obj.(string) } @@ -262,7 +263,7 @@ func (ctx *APIContext) APIErrorNotFound(objs ...any) { ctx.JSON(http.StatusNotFound, map[string]any{ "message": message, "url": setting.API.SwaggerURL, - "errors": errors, + "errors": errs, }) } @@ -298,39 +299,27 @@ func RepoRefForAPI(next http.Handler) http.Handler { } if ctx.Repo.GitRepo == nil { - ctx.APIErrorInternal(errors.New("no open git repo")) - return + panic("no GitRepo, forgot to call the middleware?") // it is a programming error } - refName, _, _ := getRefNameLegacy(ctx.Base, ctx.Repo, ctx.PathParam("*"), ctx.FormTrim("ref")) + refName, refType, _ := getRefNameLegacy(ctx.Base, ctx.Repo, ctx.PathParam("*"), ctx.FormTrim("ref")) var err error - - if gitrepo.IsBranchExist(ctx, ctx.Repo.Repository, refName) { + switch refType { + case git.RefTypeBranch: ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(refName) - if err != nil { - ctx.APIErrorInternal(err) - return - } - ctx.Repo.CommitID = ctx.Repo.Commit.ID.String() - } else if gitrepo.IsTagExist(ctx, ctx.Repo.Repository, refName) { + case git.RefTypeTag: ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetTagCommit(refName) - if err != nil { - ctx.APIErrorInternal(err) - return - } - ctx.Repo.CommitID = ctx.Repo.Commit.ID.String() - } else if len(refName) == ctx.Repo.GetObjectFormat().FullLength() { - ctx.Repo.CommitID = refName + case git.RefTypeCommit: ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetCommit(refName) - if err != nil { - ctx.APIErrorNotFound("GetCommit", err) - return - } - } else { + } + if ctx.Repo.Commit == nil || errors.Is(err, util.ErrNotExist) { ctx.APIErrorNotFound(fmt.Errorf("not exist: '%s'", ctx.PathParam("*"))) return + } else if err != nil { + ctx.APIErrorInternal(err) + return } - + ctx.Repo.CommitID = ctx.Repo.Commit.ID.String() next.ServeHTTP(w, req) }) } diff --git a/services/contexttest/context_tests.go b/services/contexttest/context_tests.go index c895de3569..b54023897b 100644 --- a/services/contexttest/context_tests.go +++ b/services/contexttest/context_tests.go @@ -170,10 +170,19 @@ func LoadUser(t *testing.T, ctx gocontext.Context, userID int64) { // LoadGitRepo load a git repo into a test context. Requires that ctx.Repo has // already been populated. -func LoadGitRepo(t *testing.T, ctx *context.Context) { - assert.NoError(t, ctx.Repo.Repository.LoadOwner(ctx)) +func LoadGitRepo(t *testing.T, ctx gocontext.Context) { + var repo *context.Repository + switch ctx := any(ctx).(type) { + case *context.Context: + repo = ctx.Repo + case *context.APIContext: + repo = ctx.Repo + default: + assert.FailNow(t, "context is not *context.Context or *context.APIContext") + } + assert.NoError(t, repo.Repository.LoadOwner(ctx)) var err error - ctx.Repo.GitRepo, err = gitrepo.OpenRepository(ctx, ctx.Repo.Repository) + repo.GitRepo, err = gitrepo.OpenRepository(ctx, repo.Repository) assert.NoError(t, err) } diff --git a/services/repository/files/content.go b/services/repository/files/content.go index 0327e7f2ce..7a07a0ddca 100644 --- a/services/repository/files/content.go +++ b/services/repository/files/content.go @@ -8,7 +8,6 @@ import ( "fmt" "net/url" "path" - "strings" repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" @@ -16,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" ) // ContentType repo content type @@ -23,14 +23,10 @@ type ContentType string // The string representations of different content types const ( - // ContentTypeRegular regular content type (file) - ContentTypeRegular ContentType = "file" - // ContentTypeDir dir content type (dir) - ContentTypeDir ContentType = "dir" - // ContentLink link content type (symlink) - ContentTypeLink ContentType = "symlink" - // ContentTag submodule content type (submodule) - ContentTypeSubmodule ContentType = "submodule" + ContentTypeRegular ContentType = "file" // regular content type (file) + ContentTypeDir ContentType = "dir" // dir content type (dir) + ContentTypeLink ContentType = "symlink" // link content type (symlink) + ContentTypeSubmodule ContentType = "submodule" // submodule content type (submodule) ) // String gets the string of ContentType @@ -38,16 +34,12 @@ func (ct *ContentType) String() string { return string(*ct) } -// GetContentsOrList gets the meta data of a file's contents (*ContentsResponse) if treePath not a tree +// GetContentsOrList gets the metadata of a file's contents (*ContentsResponse) if treePath not a tree // directory, otherwise a listing of file contents ([]*ContentsResponse). Ref can be a branch, commit or tag -func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, treePath, ref string) (any, error) { +func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, refCommit *utils.RefCommit, treePath string) (any, error) { if repo.IsEmpty { return make([]any, 0), nil } - if ref == "" { - ref = repo.DefaultBranch - } - origRef := ref // Check that the path given in opts.treePath is valid (not a git path) cleanTreePath := CleanUploadFileName(treePath) @@ -58,17 +50,8 @@ func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, treePat } treePath = cleanTreePath - gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo) - if err != nil { - return nil, err - } - defer closer.Close() - // Get the commit object for the ref - commit, err := gitRepo.GetCommit(ref) - if err != nil { - return nil, err - } + commit := refCommit.Commit entry, err := commit.GetTreeEntryByPath(treePath) if err != nil { @@ -76,7 +59,7 @@ func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, treePat } if entry.Type() != "tree" { - return GetContents(ctx, repo, treePath, origRef, false) + return GetContents(ctx, repo, refCommit, treePath, false) } // We are in a directory, so we return a list of FileContentResponse objects @@ -92,7 +75,7 @@ func GetContentsOrList(ctx context.Context, repo *repo_model.Repository, treePat } for _, e := range entries { subTreePath := path.Join(treePath, e.Name()) - fileContentResponse, err := GetContents(ctx, repo, subTreePath, origRef, true) + fileContentResponse, err := GetContents(ctx, repo, refCommit, subTreePath, true) if err != nil { return nil, err } @@ -117,13 +100,8 @@ func GetObjectTypeFromTreeEntry(entry *git.TreeEntry) ContentType { } } -// GetContents gets the meta data on a file's contents. Ref can be a branch, commit or tag -func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref string, forList bool) (*api.ContentsResponse, error) { - if ref == "" { - ref = repo.DefaultBranch - } - origRef := ref - +// GetContents gets the metadata on a file's contents. Ref can be a branch, commit or tag +func GetContents(ctx context.Context, repo *repo_model.Repository, refCommit *utils.RefCommit, treePath string, forList bool) (*api.ContentsResponse, error) { // Check that the path given in opts.treePath is valid (not a git path) cleanTreePath := CleanUploadFileName(treePath) if cleanTreePath == "" && treePath != "" { @@ -139,33 +117,24 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref } defer closer.Close() - // Get the commit object for the ref - commit, err := gitRepo.GetCommit(ref) - if err != nil { - return nil, err - } - commitID := commit.ID.String() - if len(ref) >= 4 && strings.HasPrefix(commitID, ref) { - ref = commit.ID.String() - } - + commit := refCommit.Commit entry, err := commit.GetTreeEntryByPath(treePath) if err != nil { return nil, err } - refType := gitRepo.GetRefType(ref) - if refType == "invalid" { - return nil, fmt.Errorf("no commit found for the ref [ref: %s]", ref) + refType := refCommit.RefName.RefType() + if refType != git.RefTypeBranch && refType != git.RefTypeTag && refType != git.RefTypeCommit { + return nil, fmt.Errorf("no commit found for the ref [ref: %s]", refCommit.RefName) } - selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(treePath) + "?ref=" + url.QueryEscape(origRef)) + selfURL, err := url.Parse(repo.APIURL() + "/contents/" + util.PathEscapeSegments(treePath) + "?ref=" + url.QueryEscape(refCommit.InputRef)) if err != nil { return nil, err } selfURLString := selfURL.String() - err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(ref, refType != git.ObjectCommit), repo.FullName(), commitID) + err = gitRepo.AddLastCommitCache(repo.GetCommitsCountCacheKey(refCommit.InputRef, refType != git.RefTypeCommit), repo.FullName(), refCommit.CommitID) if err != nil { return nil, err } @@ -196,15 +165,18 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref if lastCommit.Author != nil { contentsResponse.LastAuthorDate = lastCommit.Author.When } + // Now populate the rest of the ContentsResponse based on entry type if entry.IsRegular() || entry.IsExecutable() { contentsResponse.Type = string(ContentTypeRegular) - if blobResponse, err := GetBlobBySHA(ctx, repo, gitRepo, entry.ID.String()); err != nil { - return nil, err - } else if !forList { - // We don't show the content if we are getting a list of FileContentResponses - contentsResponse.Encoding = &blobResponse.Encoding - contentsResponse.Content = &blobResponse.Content + // if it is listing the repo root dir, don't waste system resources on reading content + if !forList { + blobResponse, err := GetBlobBySHA(ctx, repo, gitRepo, entry.ID.String()) + if err != nil { + return nil, err + } + contentsResponse.Encoding = blobResponse.Encoding + contentsResponse.Content = blobResponse.Content } } else if entry.IsDir() { contentsResponse.Type = string(ContentTypeDir) @@ -228,7 +200,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref } // Handle links if entry.IsRegular() || entry.IsLink() || entry.IsExecutable() { - downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + downloadURL, err := url.Parse(repo.HTMLURL() + "/raw/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(treePath)) if err != nil { return nil, err } @@ -236,7 +208,7 @@ func GetContents(ctx context.Context, repo *repo_model.Repository, treePath, ref contentsResponse.DownloadURL = &downloadURLString } if !entry.IsSubModule() { - htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + url.PathEscape(string(refType)) + "/" + util.PathEscapeSegments(ref) + "/" + util.PathEscapeSegments(treePath)) + htmlURL, err := url.Parse(repo.HTMLURL() + "/src/" + refCommit.RefName.RefWebLinkPath() + "/" + util.PathEscapeSegments(treePath)) if err != nil { return nil, err } @@ -262,18 +234,17 @@ func GetBlobBySHA(ctx context.Context, repo *repo_model.Repository, gitRepo *git if err != nil { return nil, err } - content := "" + ret := &api.GitBlobResponse{ + SHA: gitBlob.ID.String(), + URL: repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()), + Size: gitBlob.Size(), + } if gitBlob.Size() <= setting.API.DefaultMaxBlobSize { - content, err = gitBlob.GetBlobContentBase64() + content, err := gitBlob.GetBlobContentBase64() if err != nil { return nil, err } + ret.Encoding, ret.Content = util.ToPointer("base64"), &content } - return &api.GitBlobResponse{ - SHA: gitBlob.ID.String(), - URL: repo.APIURL() + "/git/blobs/" + url.PathEscape(gitBlob.ID.String()), - Size: gitBlob.Size(), - Encoding: "base64", - Content: content, - }, nil + return ret, nil } diff --git a/services/repository/files/content_test.go b/services/repository/files/content_test.go index e066f30cba..eb10f5c9b1 100644 --- a/services/repository/files/content_test.go +++ b/services/repository/files/content_test.go @@ -10,11 +10,14 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/gitrepo" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" "code.gitea.io/gitea/services/contexttest" _ "code.gitea.io/gitea/models/actions" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMain(m *testing.M) { @@ -64,18 +67,13 @@ func TestGetContents(t *testing.T) { defer ctx.Repo.GitRepo.Close() treePath := "README.md" - ref := ctx.Repo.Repository.DefaultBranch + refCommit, err := utils.ResolveRefCommit(ctx, ctx.Repo.Repository, ctx.Repo.Repository.DefaultBranch) + require.NoError(t, err) expectedContentsResponse := getExpectedReadmeContentsResponse() t.Run("Get README.md contents with GetContents(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContents(ctx, ctx.Repo.Repository, treePath, ref, false) - assert.Equal(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContents(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContents(ctx, ctx.Repo.Repository, treePath, "", false) + fileContentResponse, err := GetContents(ctx, ctx.Repo.Repository, refCommit, treePath, false) assert.Equal(t, expectedContentsResponse, fileContentResponse) assert.NoError(t, err) }) @@ -92,7 +90,8 @@ func TestGetContentsOrListForDir(t *testing.T) { defer ctx.Repo.GitRepo.Close() treePath := "" // root dir - ref := ctx.Repo.Repository.DefaultBranch + refCommit, err := utils.ResolveRefCommit(ctx, ctx.Repo.Repository, ctx.Repo.Repository.DefaultBranch) + require.NoError(t, err) readmeContentsResponse := getExpectedReadmeContentsResponse() // because will be in a list, doesn't have encoding and content @@ -104,13 +103,7 @@ func TestGetContentsOrListForDir(t *testing.T) { } t.Run("Get root dir contents with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, ref) - assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get root dir contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, "") + fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, refCommit, treePath) assert.EqualValues(t, expectedContentsListResponse, fileContentResponse) assert.NoError(t, err) }) @@ -127,18 +120,13 @@ func TestGetContentsOrListForFile(t *testing.T) { defer ctx.Repo.GitRepo.Close() treePath := "README.md" - ref := ctx.Repo.Repository.DefaultBranch + refCommit, err := utils.ResolveRefCommit(ctx, ctx.Repo.Repository, ctx.Repo.Repository.DefaultBranch) + require.NoError(t, err) expectedContentsResponse := getExpectedReadmeContentsResponse() t.Run("Get README.md contents with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, ref) - assert.EqualValues(t, expectedContentsResponse, fileContentResponse) - assert.NoError(t, err) - }) - - t.Run("Get README.md contents with ref as empty string (should then use the repo's default branch) with GetContentsOrList(ctx, )", func(t *testing.T) { - fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, treePath, "") + fileContentResponse, err := GetContentsOrList(ctx, ctx.Repo.Repository, refCommit, treePath) assert.EqualValues(t, expectedContentsResponse, fileContentResponse) assert.NoError(t, err) }) @@ -155,24 +143,16 @@ func TestGetContentsErrors(t *testing.T) { defer ctx.Repo.GitRepo.Close() repo := ctx.Repo.Repository - treePath := "README.md" - ref := repo.DefaultBranch + refCommit, err := utils.ResolveRefCommit(ctx, ctx.Repo.Repository, ctx.Repo.Repository.DefaultBranch) + require.NoError(t, err) t.Run("bad treePath", func(t *testing.T) { badTreePath := "bad/tree.md" - fileContentResponse, err := GetContents(ctx, repo, badTreePath, ref, false) + fileContentResponse, err := GetContents(ctx, repo, refCommit, badTreePath, false) assert.Error(t, err) assert.EqualError(t, err, "object does not exist [id: , rel_path: bad]") assert.Nil(t, fileContentResponse) }) - - t.Run("bad ref", func(t *testing.T) { - badRef := "bad_ref" - fileContentResponse, err := GetContents(ctx, repo, treePath, badRef, false) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") - assert.Nil(t, fileContentResponse) - }) } func TestGetContentsOrListErrors(t *testing.T) { @@ -186,42 +166,16 @@ func TestGetContentsOrListErrors(t *testing.T) { defer ctx.Repo.GitRepo.Close() repo := ctx.Repo.Repository - treePath := "README.md" - ref := repo.DefaultBranch + refCommit, err := utils.ResolveRefCommit(ctx, ctx.Repo.Repository, ctx.Repo.Repository.DefaultBranch) + require.NoError(t, err) t.Run("bad treePath", func(t *testing.T) { badTreePath := "bad/tree.md" - fileContentResponse, err := GetContentsOrList(ctx, repo, badTreePath, ref) + fileContentResponse, err := GetContentsOrList(ctx, repo, refCommit, badTreePath) assert.Error(t, err) assert.EqualError(t, err, "object does not exist [id: , rel_path: bad]") assert.Nil(t, fileContentResponse) }) - - t.Run("bad ref", func(t *testing.T) { - badRef := "bad_ref" - fileContentResponse, err := GetContentsOrList(ctx, repo, treePath, badRef) - assert.Error(t, err) - assert.EqualError(t, err, "object does not exist [id: "+badRef+", rel_path: ]") - assert.Nil(t, fileContentResponse) - }) -} - -func TestGetContentsOrListOfEmptyRepos(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user30/empty") - ctx.SetPathParam("id", "52") - contexttest.LoadRepo(t, ctx, 52) - contexttest.LoadUser(t, ctx, 30) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - - t.Run("empty repo", func(t *testing.T) { - contents, err := GetContentsOrList(ctx, repo, "", "") - assert.NoError(t, err) - assert.Empty(t, contents) - }) } func TestGetBlobBySHA(t *testing.T) { @@ -244,8 +198,8 @@ func TestGetBlobBySHA(t *testing.T) { gbr, err := GetBlobBySHA(ctx, ctx.Repo.Repository, gitRepo, ctx.PathParam("sha")) expectedGBR := &api.GitBlobResponse{ - Content: "dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK", - Encoding: "base64", + Content: util.ToPointer("dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK"), + Encoding: util.ToPointer("base64"), URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/65f1bf27bc3bf70f64657658635e66094edbcb4d", SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", Size: 180, diff --git a/services/repository/files/file.go b/services/repository/files/file.go index a8ad5889cb..c4991b458d 100644 --- a/services/repository/files/file.go +++ b/services/repository/files/file.go @@ -13,18 +13,35 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" ) -func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch string, treeNames []string) (*api.FilesResponse, error) { - files := []*api.ContentsResponse{} - for _, file := range treeNames { - fileContents, _ := GetContents(ctx, repo, file, branch, false) // ok if fails, then will be nil +func GetContentsListFromTreePaths(ctx context.Context, repo *repo_model.Repository, refCommit *utils.RefCommit, treePaths []string) (files []*api.ContentsResponse) { + var size int64 + for _, treePath := range treePaths { + fileContents, _ := GetContents(ctx, repo, refCommit, treePath, false) // ok if fails, then will be nil + if fileContents != nil && fileContents.Content != nil && *fileContents.Content != "" { + // if content isn't empty (e.g. due to the single blob being too large), add file size to response size + size += int64(len(*fileContents.Content)) + } + if size > setting.API.DefaultMaxResponseSize { + break // stop if max response size would be exceeded + } files = append(files, fileContents) + if len(files) == setting.API.DefaultPagingNum { + break // stop if paging num reached + } } - fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil - verification := GetPayloadCommitVerification(ctx, commit) + return files +} + +func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository, refCommit *utils.RefCommit, treeNames []string) (*api.FilesResponse, error) { + files := GetContentsListFromTreePaths(ctx, repo, refCommit, treeNames) + fileCommitResponse, _ := GetFileCommitResponse(repo, refCommit.Commit) // ok if fails, then will be nil + verification := GetPayloadCommitVerification(ctx, refCommit.Commit) filesResponse := &api.FilesResponse{ Files: files, Commit: fileCommitResponse, @@ -33,19 +50,6 @@ func GetFilesResponseFromCommit(ctx context.Context, repo *repo_model.Repository return filesResponse, nil } -// GetFileResponseFromCommit Constructs a FileResponse from a Commit object -func GetFileResponseFromCommit(ctx context.Context, repo *repo_model.Repository, commit *git.Commit, branch, treeName string) (*api.FileResponse, error) { - fileContents, _ := GetContents(ctx, repo, treeName, branch, false) // ok if fails, then will be nil - fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil - verification := GetPayloadCommitVerification(ctx, commit) - fileResponse := &api.FileResponse{ - Content: fileContents, - Commit: fileCommitResponse, - Verification: verification, - } - return fileResponse, nil -} - // constructs a FileResponse with the file at the index from FilesResponse func GetFileResponseFromFilesResponse(filesResponse *api.FilesResponse, index int) *api.FileResponse { content := &api.ContentsResponse{} diff --git a/services/repository/files/file_test.go b/services/repository/files/file_test.go index 7cb79d7ebf..169cafba0d 100644 --- a/services/repository/files/file_test.go +++ b/services/repository/files/file_test.go @@ -5,13 +5,6 @@ package files import ( "testing" - "time" - - "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/gitrepo" - "code.gitea.io/gitea/modules/setting" - api "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/services/contexttest" "github.com/stretchr/testify/assert" ) @@ -31,93 +24,3 @@ func TestCleanUploadFileName(t *testing.T) { assert.Equal(t, expectedCleanName, cleanName) }) } - -func getExpectedFileResponse() *api.FileResponse { - treePath := "README.md" - sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" - encoding := "base64" - content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" - selfURL := setting.AppURL + "api/v1/repos/user2/repo1/contents/" + treePath + "?ref=master" - htmlURL := setting.AppURL + "user2/repo1/src/branch/master/" + treePath - gitURL := setting.AppURL + "api/v1/repos/user2/repo1/git/blobs/" + sha - downloadURL := setting.AppURL + "user2/repo1/raw/branch/master/" + treePath - return &api.FileResponse{ - Content: &api.ContentsResponse{ - Name: treePath, - Path: treePath, - SHA: sha, - LastCommitSHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - LastCommitterDate: time.Date(2017, time.March, 19, 16, 47, 59, 0, time.FixedZone("", -14400)), - LastAuthorDate: time.Date(2017, time.March, 19, 16, 47, 59, 0, time.FixedZone("", -14400)), - Type: "file", - Size: 30, - Encoding: &encoding, - Content: &content, - URL: &selfURL, - HTMLURL: &htmlURL, - GitURL: &gitURL, - DownloadURL: &downloadURL, - Links: &api.FileLinksResponse{ - Self: &selfURL, - GitURL: &gitURL, - HTMLURL: &htmlURL, - }, - }, - Commit: &api.FileCommitResponse{ - CommitMeta: api.CommitMeta{ - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/commits/65f1bf27bc3bf70f64657658635e66094edbcb4d", - SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d", - }, - HTMLURL: "https://try.gitea.io/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d", - Author: &api.CommitUser{ - Identity: api.Identity{ - Name: "user1", - Email: "address1@example.com", - }, - Date: "2017-03-19T20:47:59Z", - }, - Committer: &api.CommitUser{ - Identity: api.Identity{ - Name: "Ethan Koenig", - Email: "ethantkoenig@gmail.com", - }, - Date: "2017-03-19T20:47:59Z", - }, - Parents: []*api.CommitMeta{}, - Message: "Initial commit\n", - Tree: &api.CommitMeta{ - URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/2a2f1d4670728a2e10049e345bd7a276468beab6", - SHA: "2a2f1d4670728a2e10049e345bd7a276468beab6", - }, - }, - Verification: &api.PayloadCommitVerification{ - Verified: false, - Reason: "gpg.error.not_signed_commit", - Signature: "", - Payload: "", - }, - } -} - -func TestGetFileResponseFromCommit(t *testing.T) { - unittest.PrepareTestEnv(t) - ctx, _ := contexttest.MockContext(t, "user2/repo1") - ctx.SetPathParam("id", "1") - contexttest.LoadRepo(t, ctx, 1) - contexttest.LoadRepoCommit(t, ctx) - contexttest.LoadUser(t, ctx, 2) - contexttest.LoadGitRepo(t, ctx) - defer ctx.Repo.GitRepo.Close() - - repo := ctx.Repo.Repository - branch := repo.DefaultBranch - treePath := "README.md" - gitRepo, _ := gitrepo.OpenRepository(ctx, repo) - defer gitRepo.Close() - commit, _ := gitRepo.GetBranchCommit(branch) - expectedFileResponse := getExpectedFileResponse() - - fileResponse, err := GetFileResponseFromCommit(ctx, repo, commit, branch, treePath) - assert.NoError(t, err) - assert.Equal(t, expectedFileResponse, fileResponse) -} diff --git a/services/repository/files/update.go b/services/repository/files/update.go index 75ede4976f..fbf59c40ed 100644 --- a/services/repository/files/update.go +++ b/services/repository/files/update.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/api/v1/utils" asymkey_service "code.gitea.io/gitea/services/asymkey" pull_service "code.gitea.io/gitea/services/pull" ) @@ -296,7 +297,9 @@ func ChangeRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use return nil, err } - filesResponse, err := GetFilesResponseFromCommit(ctx, repo, commit, opts.NewBranch, treePaths) + // FIXME: this call seems not right, why it needs to read the file content again + // FIXME: why it uses the NewBranch as "ref", it should use the commit ID because the response is only for this commit + filesResponse, err := GetFilesResponseFromCommit(ctx, repo, utils.NewRefCommit(git.RefNameFromBranch(opts.NewBranch), commit), treePaths) if err != nil { return nil, err } diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 99d3c994f9..0a9432fc9d 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -6863,7 +6863,7 @@ }, { "type": "string", - "description": "The name of the commit/branch/tag. Default the repository’s default branch (usually master)", + "description": "The name of the commit/branch/tag. Default to the repository’s default branch.", "name": "ref", "in": "query" } @@ -6966,7 +6966,7 @@ }, { "type": "string", - "description": "The name of the commit/branch/tag. Default the repository’s default branch (usually master)", + "description": "The name of the commit/branch/tag. Default to the repository’s default branch.", "name": "ref", "in": "query" } @@ -7248,7 +7248,7 @@ }, { "type": "string", - "description": "The name of the commit/branch/tag. Default the repository’s default branch (usually master)", + "description": "The name of the commit/branch/tag. Default to the repository’s default branch.", "name": "ref", "in": "query" } @@ -7263,6 +7263,105 @@ } } }, + "/repos/{owner}/{repo}/file-contents": { + "get": { + "description": "See the POST method. This GET method supports to use JSON encoded request body in query parameter.", + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "Get the metadata and contents of requested files", + "operationId": "repoGetFileContents", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "The name of the commit/branch/tag. Default to the repository’s default branch.", + "name": "ref", + "in": "query" + }, + { + "type": "string", + "description": "The JSON encoded body (see the POST request): {\"files\": [\"filename1\", \"filename2\"]}", + "name": "body", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "$ref": "#/responses/ContentsListResponse" + }, + "404": { + "$ref": "#/responses/notFound" + } + } + }, + "post": { + "description": "Uses automatic pagination based on default page size and max response size and returns the maximum allowed number of files. Files which could not be retrieved are null. Files which are too large are being returned with `encoding == null`, `content == null` and `size \u003e 0`, they can be requested separately by using the `download_url`.", + "produces": [ + "application/json" + ], + "tags": [ + "repository" + ], + "summary": "Get the metadata and contents of requested files", + "operationId": "repoGetFileContentsPost", + "parameters": [ + { + "type": "string", + "description": "owner of the repo", + "name": "owner", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name of the repo", + "name": "repo", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "The name of the commit/branch/tag. Default to the repository’s default branch.", + "name": "ref", + "in": "query" + }, + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/GetFilesOptions" + } + } + ], + "responses": { + "200": { + "$ref": "#/responses/ContentsListResponse" + }, + "404": { + "$ref": "#/responses/notFound" + } + } + } + }, "/repos/{owner}/{repo}/forks": { "get": { "produces": [ @@ -23623,6 +23722,11 @@ "format": "int64", "x-go-name": "DefaultMaxBlobSize" }, + "default_max_response_size": { + "type": "integer", + "format": "int64", + "x-go-name": "DefaultMaxResponseSize" + }, "default_paging_num": { "type": "integer", "format": "int64", @@ -23789,6 +23893,20 @@ }, "x-go-package": "code.gitea.io/gitea/modules/structs" }, + "GetFilesOptions": { + "description": "GetFilesOptions options for retrieving metadate and content of multiple files", + "type": "object", + "properties": { + "files": { + "type": "array", + "items": { + "type": "string" + }, + "x-go-name": "Files" + } + }, + "x-go-package": "code.gitea.io/gitea/modules/structs" + }, "GitBlobResponse": { "description": "GitBlobResponse represents a git blob", "type": "object", diff --git a/tests/integration/api_repo_files_get_test.go b/tests/integration/api_repo_files_get_test.go new file mode 100644 index 0000000000..a4ded7da3f --- /dev/null +++ b/tests/integration/api_repo_files_get_test.go @@ -0,0 +1,157 @@ +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "fmt" + "net/http" + "net/url" + "testing" + + auth_model "code.gitea.io/gitea/models/auth" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAPIGetRequestedFiles(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + user2 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) // owner of the repo1 & repo16 + org3 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 3}) // owner of the repo3, is an org + user4 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 4}) // owner of neither repos + repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) // public repo + repo3 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 3}) // public repo + repo16 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 16}) // private repo + + // Get user2's token + session := loginUser(t, user2.Name) + token2 := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository) + // Get user4's token + session = loginUser(t, user4.Name) + token4 := getTokenForLoggedInUser(t, session, auth_model.AccessTokenScopeWriteRepository) + + gitRepo, err := gitrepo.OpenRepository(git.DefaultContext, repo1) + assert.NoError(t, err) + defer gitRepo.Close() + lastCommit, _ := gitRepo.GetCommitByPath("README.md") + + requestFiles := func(t *testing.T, url string, files []string, expectedStatusCode ...int) (ret []*api.ContentsResponse) { + req := NewRequestWithJSON(t, "POST", url, &api.GetFilesOptions{Files: files}) + resp := MakeRequest(t, req, util.OptionalArg(expectedStatusCode, http.StatusOK)) + if resp.Code != http.StatusOK { + return nil + } + DecodeJSON(t, resp, &ret) + return ret + } + + t.Run("User2Get", func(t *testing.T) { + reqBodyOpt := &api.GetFilesOptions{Files: []string{"README.md"}} + reqBodyParam, _ := json.Marshal(reqBodyOpt) + req := NewRequest(t, "GET", "/api/v1/repos/user2/repo1/file-contents?body="+url.QueryEscape(string(reqBodyParam))) + resp := MakeRequest(t, req, http.StatusOK) + var ret []*api.ContentsResponse + DecodeJSON(t, resp, &ret) + expected := []*api.ContentsResponse{getExpectedContentsResponseForContents(repo1.DefaultBranch, "branch", lastCommit.ID.String())} + assert.Equal(t, expected, ret) + }) + t.Run("User2NoRef", func(t *testing.T) { + ret := requestFiles(t, "/api/v1/repos/user2/repo1/file-contents", []string{"README.md"}) + expected := []*api.ContentsResponse{getExpectedContentsResponseForContents(repo1.DefaultBranch, "branch", lastCommit.ID.String())} + assert.Equal(t, expected, ret) + }) + t.Run("User2RefBranch", func(t *testing.T) { + ret := requestFiles(t, "/api/v1/repos/user2/repo1/file-contents?ref=master", []string{"README.md"}) + expected := []*api.ContentsResponse{getExpectedContentsResponseForContents(repo1.DefaultBranch, "branch", lastCommit.ID.String())} + assert.Equal(t, expected, ret) + }) + t.Run("User2RefTag", func(t *testing.T) { + ret := requestFiles(t, "/api/v1/repos/user2/repo1/file-contents?ref=v1.1", []string{"README.md"}) + expected := []*api.ContentsResponse{getExpectedContentsResponseForContents("v1.1", "tag", lastCommit.ID.String())} + assert.Equal(t, expected, ret) + }) + t.Run("User2RefCommit", func(t *testing.T) { + ret := requestFiles(t, "/api/v1/repos/user2/repo1/file-contents?ref=65f1bf27bc3bf70f64657658635e66094edbcb4d", []string{"README.md"}) + expected := []*api.ContentsResponse{getExpectedContentsResponseForContents("65f1bf27bc3bf70f64657658635e66094edbcb4d", "commit", lastCommit.ID.String())} + assert.Equal(t, expected, ret) + }) + t.Run("User2RefNotExist", func(t *testing.T) { + ret := requestFiles(t, "/api/v1/repos/user2/repo1/file-contents?ref=not-exist", []string{"README.md"}, http.StatusNotFound) + assert.Empty(t, ret) + }) + + t.Run("PermissionCheck", func(t *testing.T) { + filesOptions := &api.GetFilesOptions{Files: []string{"README.md"}} + // Test accessing private ref with user token that does not have access - should fail + req := NewRequestWithJSON(t, "POST", fmt.Sprintf("/api/v1/repos/%s/%s/file-contents", user2.Name, repo16.Name), &filesOptions).AddTokenAuth(token4) + MakeRequest(t, req, http.StatusNotFound) + // Test access private ref of owner of token + req = NewRequestWithJSON(t, "POST", fmt.Sprintf("/api/v1/repos/%s/%s/file-contents", user2.Name, repo16.Name), &filesOptions).AddTokenAuth(token2) + MakeRequest(t, req, http.StatusOK) + // Test access of org org3 private repo file by owner user2 + req = NewRequestWithJSON(t, "POST", fmt.Sprintf("/api/v1/repos/%s/%s/file-contents", org3.Name, repo3.Name), &filesOptions).AddTokenAuth(token2) + MakeRequest(t, req, http.StatusOK) + }) + + t.Run("ResponseList", func(t *testing.T) { + defer test.MockVariableValue(&setting.API.DefaultPagingNum)() + defer test.MockVariableValue(&setting.API.DefaultMaxBlobSize)() + defer test.MockVariableValue(&setting.API.DefaultMaxResponseSize)() + + type expected struct { + Name string + HasContent bool + } + assertResponse := func(t *testing.T, expected []*expected, ret []*api.ContentsResponse) { + require.Len(t, ret, len(expected)) + for i, e := range expected { + if e == nil { + assert.Nil(t, ret[i], "item %d", i) + continue + } + assert.Equal(t, e.Name, ret[i].Name, "item %d name", i) + if e.HasContent { + require.NotNil(t, ret[i].Content, "item %d content", i) + assert.NotEmpty(t, *ret[i].Content, "item %d content", i) + } else { + assert.Nil(t, ret[i].Content, "item %d content", i) + } + } + } + + // repo1 "DefaultBranch" has 2 files: LICENSE (1064 bytes), README.md (30 bytes) + ret := requestFiles(t, "/api/v1/repos/user2/repo1/file-contents?ref=DefaultBranch", []string{"no-such.txt", "LICENSE", "README.md"}) + assertResponse(t, []*expected{nil, {"LICENSE", true}, {"README.md", true}}, ret) + + // the returned file list is limited by the DefaultPagingNum + setting.API.DefaultPagingNum = 2 + ret = requestFiles(t, "/api/v1/repos/user2/repo1/file-contents?ref=DefaultBranch", []string{"no-such.txt", "LICENSE", "README.md"}) + assertResponse(t, []*expected{nil, {"LICENSE", true}}, ret) + setting.API.DefaultPagingNum = 100 + + // if a file exceeds the DefaultMaxBlobSize, the content is not returned + setting.API.DefaultMaxBlobSize = 200 + ret = requestFiles(t, "/api/v1/repos/user2/repo1/file-contents?ref=DefaultBranch", []string{"no-such.txt", "LICENSE", "README.md"}) + assertResponse(t, []*expected{nil, {"LICENSE", false}, {"README.md", true}}, ret) + setting.API.DefaultMaxBlobSize = 20000 + + // if the total response size would exceed the DefaultMaxResponseSize, then the list stops + setting.API.DefaultMaxResponseSize = ret[1].Size*4/3 + 10 + ret = requestFiles(t, "/api/v1/repos/user2/repo1/file-contents?ref=DefaultBranch", []string{"no-such.txt", "LICENSE", "README.md"}) + assertResponse(t, []*expected{nil, {"LICENSE", true}}, ret) + setting.API.DefaultMaxBlobSize = 20000 + }) +} diff --git a/tests/integration/api_repo_get_contents_test.go b/tests/integration/api_repo_get_contents_test.go index 1571787886..a7de97c052 100644 --- a/tests/integration/api_repo_get_contents_test.go +++ b/tests/integration/api_repo_get_contents_test.go @@ -18,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" repo_service "code.gitea.io/gitea/services/repository" "code.gitea.io/gitea/tests" @@ -26,28 +27,24 @@ import ( func getExpectedContentsResponseForContents(ref, refType, lastCommitSHA string) *api.ContentsResponse { treePath := "README.md" - sha := "4b4851ad51df6a7d9f25c979345979eaeb5b349f" - encoding := "base64" - content := "IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x" selfURL := setting.AppURL + "api/v1/repos/user2/repo1/contents/" + treePath + "?ref=" + ref htmlURL := setting.AppURL + "user2/repo1/src/" + refType + "/" + ref + "/" + treePath - gitURL := setting.AppURL + "api/v1/repos/user2/repo1/git/blobs/" + sha - downloadURL := setting.AppURL + "user2/repo1/raw/" + refType + "/" + ref + "/" + treePath + gitURL := setting.AppURL + "api/v1/repos/user2/repo1/git/blobs/4b4851ad51df6a7d9f25c979345979eaeb5b349f" return &api.ContentsResponse{ Name: treePath, Path: treePath, - SHA: sha, + SHA: "4b4851ad51df6a7d9f25c979345979eaeb5b349f", LastCommitSHA: lastCommitSHA, LastCommitterDate: time.Date(2017, time.March, 19, 16, 47, 59, 0, time.FixedZone("", -14400)), LastAuthorDate: time.Date(2017, time.March, 19, 16, 47, 59, 0, time.FixedZone("", -14400)), Type: "file", Size: 30, - Encoding: &encoding, - Content: &content, + Encoding: util.ToPointer("base64"), + Content: util.ToPointer("IyByZXBvMQoKRGVzY3JpcHRpb24gZm9yIHJlcG8x"), URL: &selfURL, HTMLURL: &htmlURL, GitURL: &gitURL, - DownloadURL: &downloadURL, + DownloadURL: util.ToPointer(setting.AppURL + "user2/repo1/raw/" + refType + "/" + ref + "/" + treePath), Links: &api.FileLinksResponse{ Self: &selfURL, GitURL: &gitURL, diff --git a/tests/integration/api_repo_git_blobs_test.go b/tests/integration/api_repo_git_blobs_test.go index 9c4be31396..d4274bdb40 100644 --- a/tests/integration/api_repo_git_blobs_test.go +++ b/tests/integration/api_repo_git_blobs_test.go @@ -41,7 +41,7 @@ func TestAPIReposGitBlobs(t *testing.T) { DecodeJSON(t, resp, &gitBlobResponse) assert.NotNil(t, gitBlobResponse) expectedContent := "dHJlZSAyYTJmMWQ0NjcwNzI4YTJlMTAwNDllMzQ1YmQ3YTI3NjQ2OGJlYWI2CmF1dGhvciB1c2VyMSA8YWRkcmVzczFAZXhhbXBsZS5jb20+IDE0ODk5NTY0NzkgLTA0MDAKY29tbWl0dGVyIEV0aGFuIEtvZW5pZyA8ZXRoYW50a29lbmlnQGdtYWlsLmNvbT4gMTQ4OTk1NjQ3OSAtMDQwMAoKSW5pdGlhbCBjb21taXQK" - assert.Equal(t, expectedContent, gitBlobResponse.Content) + assert.Equal(t, expectedContent, *gitBlobResponse.Content) // Tests a private repo with no token so will fail req = NewRequestf(t, "GET", "/api/v1/repos/%s/%s/git/blobs/%s", user2.Name, repo16.Name, repo16ReadmeSHA) diff --git a/tests/integration/api_settings_test.go b/tests/integration/api_settings_test.go index 6f433148f5..743dbb0481 100644 --- a/tests/integration/api_settings_test.go +++ b/tests/integration/api_settings_test.go @@ -35,6 +35,7 @@ func TestAPIExposedSettings(t *testing.T) { DefaultPagingNum: setting.API.DefaultPagingNum, DefaultGitTreesPerPage: setting.API.DefaultGitTreesPerPage, DefaultMaxBlobSize: setting.API.DefaultMaxBlobSize, + DefaultMaxResponseSize: setting.API.DefaultMaxResponseSize, }, apiSettings) repo := new(api.GeneralRepoSettings)