feat: [CODE-2528]: Update raw and get-content APIs to download lfs objects (#3549)

This commit is contained in:
Atefeh Mohseni Ejiyeh 2025-04-02 21:28:41 +00:00 committed by Harness
parent 88d1f60157
commit d3261ebc20
20 changed files with 289 additions and 65 deletions

View File

@ -23,17 +23,38 @@ import (
"github.com/harness/gitness/types/enum" "github.com/harness/gitness/types/enum"
) )
type Content struct {
Data io.ReadCloser
Size int64
}
func (c *Content) Read(p []byte) (n int, err error) {
return c.Data.Read(p)
}
func (c *Content) Close() error {
return c.Data.Close()
}
func (c *Controller) Download(ctx context.Context, func (c *Controller) Download(ctx context.Context,
session *auth.Session, session *auth.Session,
repoRef string, repoRef string,
oid string, oid string,
) (io.ReadCloser, error) { ) (*Content, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView) repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to acquire access to repo: %w", err) return nil, fmt.Errorf("failed to acquire access to repo: %w", err)
} }
_, err = c.lfsStore.Find(ctx, repo.ID, oid) return c.DownloadNoAuth(ctx, repo.ID, oid)
}
func (c *Controller) DownloadNoAuth(
ctx context.Context,
repoID int64,
oid string,
) (*Content, error) {
obj, err := c.lfsStore.Find(ctx, repoID, oid)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to find the oid %q for the repo: %w", oid, err) return nil, fmt.Errorf("failed to find the oid %q for the repo: %w", oid, err)
} }
@ -44,5 +65,8 @@ func (c *Controller) Download(ctx context.Context,
return nil, fmt.Errorf("failed to download file from blobstore: %w", err) return nil, fmt.Errorf("failed to download file from blobstore: %w", err)
} }
return file, nil return &Content{
Data: file,
Size: obj.Size,
}, nil
} }

View File

@ -48,7 +48,7 @@ func (c *Controller) Upload(ctx context.Context,
return nil, usererror.BadRequest("no file or content provided") return nil, usererror.BadRequest("no file or content provided")
} }
bufReader := bufio.NewReader(file) bufReader := bufio.NewReader(io.LimitReader(file, pointer.Size))
objPath := getLFSObjectPath(pointer.OId) objPath := getLFSObjectPath(pointer.OId)
err = c.blobStore.Upload(ctx, bufReader, objPath) err = c.blobStore.Upload(ctx, bufReader, objPath)

View File

@ -24,6 +24,7 @@ import (
"github.com/harness/gitness/app/auth" "github.com/harness/gitness/app/auth"
"github.com/harness/gitness/errors" "github.com/harness/gitness/errors"
"github.com/harness/gitness/git" "github.com/harness/gitness/git"
"github.com/harness/gitness/git/parser"
"github.com/harness/gitness/types" "github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum" "github.com/harness/gitness/types/enum"
@ -33,7 +34,7 @@ import (
const ( const (
// maxGetContentFileSize specifies the maximum number of bytes a file content response contains. // maxGetContentFileSize specifies the maximum number of bytes a file content response contains.
// If a file is any larger, the content is truncated. // If a file is any larger, the content is truncated.
maxGetContentFileSize = 1 << 22 // 4 MB maxGetContentFileSize = 10 * 1024 * 1024 // 10 MB
) )
type ContentType string type ContentType string
@ -64,10 +65,12 @@ type Content interface {
} }
type FileContent struct { type FileContent struct {
Encoding enum.ContentEncodingType `json:"encoding"` Encoding enum.ContentEncodingType `json:"encoding"`
Data string `json:"data"` Data string `json:"data"`
Size int64 `json:"size"` Size int64 `json:"size"`
DataSize int64 `json:"data_size"` DataSize int64 `json:"data_size"`
LFSObjectID string `json:"lfs_object_id,omitempty"`
LFSObjectSize int64 `json:"lfs_object_size,omitempty"`
} }
func (c *FileContent) isContent() {} func (c *FileContent) isContent() {}
@ -200,6 +203,19 @@ func (c *Controller) getFileContent(ctx context.Context,
return nil, fmt.Errorf("failed to read blob content: %w", err) return nil, fmt.Errorf("failed to read blob content: %w", err)
} }
// check if blob is an LFS pointer
lfsInfo, ok := parser.IsLFSPointer(ctx, content, output.Size)
if ok {
return &FileContent{
Size: output.Size,
DataSize: output.ContentSize,
Encoding: enum.ContentEncodingTypeBase64,
Data: base64.StdEncoding.EncodeToString(content),
LFSObjectID: lfsInfo.OID,
LFSObjectSize: lfsInfo.Size,
}, nil
}
return &FileContent{ return &FileContent{
Size: output.Size, Size: output.Size,
DataSize: output.ContentSize, DataSize: output.ContentSize,

View File

@ -22,6 +22,7 @@ import (
"strings" "strings"
apiauth "github.com/harness/gitness/app/api/auth" apiauth "github.com/harness/gitness/app/api/auth"
"github.com/harness/gitness/app/api/controller/lfs"
"github.com/harness/gitness/app/api/controller/limiter" "github.com/harness/gitness/app/api/controller/limiter"
"github.com/harness/gitness/app/api/usererror" "github.com/harness/gitness/app/api/usererror"
"github.com/harness/gitness/app/auth" "github.com/harness/gitness/app/auth"
@ -110,6 +111,7 @@ type Controller struct {
instrumentation instrument.Service instrumentation instrument.Service
rulesSvc *rules.Service rulesSvc *rules.Service
sseStreamer sse.Streamer sseStreamer sse.Streamer
lfsCtrl *lfs.Controller
} }
func NewController( func NewController(
@ -148,6 +150,7 @@ func NewController(
userGroupService usergroup.SearchService, userGroupService usergroup.SearchService,
rulesSvc *rules.Service, rulesSvc *rules.Service,
sseStreamer sse.Streamer, sseStreamer sse.Streamer,
lfsCtrl *lfs.Controller,
) *Controller { ) *Controller {
return &Controller{ return &Controller{
defaultBranch: config.Git.DefaultBranch, defaultBranch: config.Git.DefaultBranch,
@ -185,6 +188,7 @@ func NewController(
userGroupService: userGroupService, userGroupService: userGroupService,
rulesSvc: rulesSvc, rulesSvc: rulesSvc,
sseStreamer: sseStreamer, sseStreamer: sseStreamer,
lfsCtrl: lfsCtrl,
} }
} }

View File

@ -22,10 +22,17 @@ import (
"github.com/harness/gitness/app/api/usererror" "github.com/harness/gitness/app/api/usererror"
"github.com/harness/gitness/app/auth" "github.com/harness/gitness/app/auth"
"github.com/harness/gitness/git" "github.com/harness/gitness/git"
"github.com/harness/gitness/git/parser"
"github.com/harness/gitness/git/sha" "github.com/harness/gitness/git/sha"
"github.com/harness/gitness/types/enum" "github.com/harness/gitness/types/enum"
) )
type RawContent struct {
Data io.ReadCloser
Size int64
SHA sha.SHA
}
// Raw finds the file of the repo at the given path and returns its raw content. // Raw finds the file of the repo at the given path and returns its raw content.
// If no gitRef is provided, the content is retrieved from the default branch. // If no gitRef is provided, the content is retrieved from the default branch.
func (c *Controller) Raw(ctx context.Context, func (c *Controller) Raw(ctx context.Context,
@ -33,10 +40,10 @@ func (c *Controller) Raw(ctx context.Context,
repoRef string, repoRef string,
gitRef string, gitRef string,
path string, path string,
) (io.ReadCloser, int64, sha.SHA, error) { ) (*RawContent, error) {
repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView) repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView)
if err != nil { if err != nil {
return nil, 0, sha.Nil, err return nil, err
} }
// set gitRef to default branch in case an empty reference was provided // set gitRef to default branch in case an empty reference was provided
@ -53,12 +60,12 @@ func (c *Controller) Raw(ctx context.Context,
IncludeLatestCommit: false, IncludeLatestCommit: false,
}) })
if err != nil { if err != nil {
return nil, 0, sha.Nil, fmt.Errorf("failed to read tree node: %w", err) return nil, fmt.Errorf("failed to read tree node: %w", err)
} }
// viewing Raw content is only supported for blob content // viewing Raw content is only supported for blob content
if treeNodeOutput.Node.Type != git.TreeNodeTypeBlob { if treeNodeOutput.Node.Type != git.TreeNodeTypeBlob {
return nil, 0, sha.Nil, usererror.BadRequestf( return nil, usererror.BadRequestf(
"Object in '%s' at '/%s' is of type '%s'. Only objects of type %s support raw viewing.", "Object in '%s' at '/%s' is of type '%s'. Only objects of type %s support raw viewing.",
gitRef, path, treeNodeOutput.Node.Type, git.TreeNodeTypeBlob) gitRef, path, treeNodeOutput.Node.Type, git.TreeNodeTypeBlob)
} }
@ -69,8 +76,32 @@ func (c *Controller) Raw(ctx context.Context,
SizeLimit: 0, // no size limit, we stream whatever data there is SizeLimit: 0, // no size limit, we stream whatever data there is
}) })
if err != nil { if err != nil {
return nil, 0, sha.Nil, fmt.Errorf("failed to read blob: %w", err) return nil, fmt.Errorf("failed to read blob: %w", err)
} }
return blobReader.Content, blobReader.ContentSize, blobReader.SHA, nil // check if blob is LFS
content, err := io.ReadAll(io.LimitReader(blobReader.Content, parser.LfsPointerMaxSize))
if err != nil {
return nil, fmt.Errorf("failed to read LFS file content: %w", err)
}
lfsInfo, ok := parser.IsLFSPointer(ctx, content, blobReader.Size)
if !ok {
return &RawContent{
Data: blobReader.Content,
Size: blobReader.ContentSize,
SHA: blobReader.SHA,
}, nil
}
file, err := c.lfsCtrl.DownloadNoAuth(ctx, repo.ID, lfsInfo.OID)
if err != nil {
return nil, fmt.Errorf("failed to download LFS file: %w", err)
}
return &RawContent{
Data: file,
Size: lfsInfo.Size,
SHA: blobReader.SHA,
}, nil
} }

View File

@ -15,6 +15,7 @@
package repo package repo
import ( import (
"github.com/harness/gitness/app/api/controller/lfs"
"github.com/harness/gitness/app/api/controller/limiter" "github.com/harness/gitness/app/api/controller/limiter"
"github.com/harness/gitness/app/auth/authz" "github.com/harness/gitness/app/auth/authz"
repoevents "github.com/harness/gitness/app/events/repo" repoevents "github.com/harness/gitness/app/events/repo"
@ -84,6 +85,7 @@ func ProvideController(
userGroupService usergroup.SearchService, userGroupService usergroup.SearchService,
rulesSvc *rules.Service, rulesSvc *rules.Service,
sseStreamer sse.Streamer, sseStreamer sse.Streamer,
lfsCtrl *lfs.Controller,
) *Controller { ) *Controller {
return NewController(config, tx, urlProvider, return NewController(config, tx, urlProvider,
authorizer, authorizer,
@ -92,7 +94,7 @@ func ProvideController(
principalInfoCache, protectionManager, rpcClient, spaceFinder, repoFinder, importer, principalInfoCache, protectionManager, rpcClient, spaceFinder, repoFinder, importer,
codeOwners, repoReporter, indexer, limiter, locker, auditService, mtxManager, identifierCheck, codeOwners, repoReporter, indexer, limiter, locker, auditService, mtxManager, identifierCheck,
repoChecks, publicAccess, labelSvc, instrumentation, userGroupStore, userGroupService, repoChecks, publicAccess, labelSvc, instrumentation, userGroupStore, userGroupService,
rulesSvc, sseStreamer, rulesSvc, sseStreamer, lfsCtrl,
) )
} }

View File

@ -16,6 +16,7 @@ package lfs
import ( import (
"errors" "errors"
"fmt"
"net/http" "net/http"
apiauth "github.com/harness/gitness/app/api/auth" apiauth "github.com/harness/gitness/app/api/auth"
@ -24,6 +25,8 @@ import (
"github.com/harness/gitness/app/api/request" "github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/app/auth" "github.com/harness/gitness/app/auth"
"github.com/harness/gitness/app/url" "github.com/harness/gitness/app/url"
"github.com/rs/zerolog/log"
) )
func HandleLFSDownload(controller *lfs.Controller, urlProvider url.Provider) http.HandlerFunc { func HandleLFSDownload(controller *lfs.Controller, urlProvider url.Provider) http.HandlerFunc {
@ -42,7 +45,7 @@ func HandleLFSDownload(controller *lfs.Controller, urlProvider url.Provider) htt
return return
} }
file, err := controller.Download(ctx, session, repoRef, oid) resp, err := controller.Download(ctx, session, repoRef, oid)
if errors.Is(err, apiauth.ErrNotAuthorized) && auth.IsAnonymousSession(session) { if errors.Is(err, apiauth.ErrNotAuthorized) && auth.IsAnonymousSession(session) {
render.GitBasicAuth(ctx, w, urlProvider) render.GitBasicAuth(ctx, w, urlProvider)
return return
@ -51,8 +54,14 @@ func HandleLFSDownload(controller *lfs.Controller, urlProvider url.Provider) htt
render.TranslatedUserError(ctx, w, err) render.TranslatedUserError(ctx, w, err)
return return
} }
defer file.Close() defer func() {
if err := resp.Data.Close(); err != nil {
log.Ctx(ctx).Warn().Err(err).Msgf("failed to close LFS file reader for %q.", oid)
}
}()
w.Header().Add("Content-Length", fmt.Sprint(resp.Size))
// apply max byte size // apply max byte size
render.Reader(ctx, w, http.StatusOK, file) render.Reader(ctx, w, http.StatusOK, resp.Data)
} }
} }

View File

@ -41,26 +41,26 @@ func HandleRaw(repoCtrl *repo.Controller) http.HandlerFunc {
gitRef := request.GetGitRefFromQueryOrDefault(r, "") gitRef := request.GetGitRefFromQueryOrDefault(r, "")
path := request.GetOptionalRemainderFromPath(r) path := request.GetOptionalRemainderFromPath(r)
dataReader, dataLength, sha, err := repoCtrl.Raw(ctx, session, repoRef, gitRef, path) resp, err := repoCtrl.Raw(ctx, session, repoRef, gitRef, path)
if err != nil { if err != nil {
render.TranslatedUserError(ctx, w, err) render.TranslatedUserError(ctx, w, err)
return return
} }
defer func() { defer func() {
if err := dataReader.Close(); err != nil { if err := resp.Data.Close(); err != nil {
log.Ctx(ctx).Warn().Err(err).Msgf("failed to close blob content reader.") log.Ctx(ctx).Warn().Err(err).Msgf("failed to close blob content reader.")
} }
}() }()
ifNoneMatch, ok := request.GetIfNoneMatchFromHeader(r) ifNoneMatch, ok := request.GetIfNoneMatchFromHeader(r)
if ok && ifNoneMatch == sha.String() { if ok && ifNoneMatch == resp.SHA.String() {
w.WriteHeader(http.StatusNotModified) w.WriteHeader(http.StatusNotModified)
return return
} }
w.Header().Add("Content-Length", fmt.Sprint(dataLength)) w.Header().Add("Content-Length", fmt.Sprint(resp.Size))
w.Header().Add(request.HeaderETag, sha.String()) w.Header().Add(request.HeaderETag, resp.SHA.String())
render.Reader(ctx, w, http.StatusOK, dataReader) render.Reader(ctx, w, http.StatusOK, resp.Data)
} }
} }

View File

@ -63,6 +63,12 @@ func processGitRequest(r *http.Request) (bool, error) {
const receivePackPath = "/" + receivePack const receivePackPath = "/" + receivePack
const serviceParam = "service" const serviceParam = "service"
const lfsTransferPath = "/info/lfs/objects"
const lfsTransferBatchPath = lfsTransferPath + "/batch"
const oidParam = "oid"
const sizeParam = "size"
allowedServices := []string{ allowedServices := []string{
uploadPack, uploadPack,
receivePack, receivePack,
@ -84,6 +90,11 @@ func processGitRequest(r *http.Request) (bool, error) {
} }
return pathTerminatedWithMarkerAndURL(r, "", infoRefsPath, infoRefsPath, urlPath) return pathTerminatedWithMarkerAndURL(r, "", infoRefsPath, infoRefsPath, urlPath)
} }
// check if request is coming from git lfs client
if strings.HasSuffix(urlPath, lfsTransferPath) && r.URL.Query().Has(oidParam) {
return pathTerminatedWithMarkerAndURL(r, "", lfsTransferPath, lfsTransferPath, urlPath)
}
case http.MethodPost: case http.MethodPost:
if strings.HasSuffix(urlPath, uploadPackPath) { if strings.HasSuffix(urlPath, uploadPackPath) {
return pathTerminatedWithMarkerAndURL(r, "", uploadPackPath, uploadPackPath, urlPath) return pathTerminatedWithMarkerAndURL(r, "", uploadPackPath, uploadPackPath, urlPath)
@ -92,6 +103,16 @@ func processGitRequest(r *http.Request) (bool, error) {
if strings.HasSuffix(urlPath, receivePackPath) { if strings.HasSuffix(urlPath, receivePackPath) {
return pathTerminatedWithMarkerAndURL(r, "", receivePackPath, receivePackPath, urlPath) return pathTerminatedWithMarkerAndURL(r, "", receivePackPath, receivePackPath, urlPath)
} }
if strings.HasSuffix(urlPath, lfsTransferBatchPath) {
return pathTerminatedWithMarkerAndURL(r, "", lfsTransferBatchPath, lfsTransferBatchPath, urlPath)
}
case http.MethodPut:
if strings.HasSuffix(urlPath, lfsTransferPath) &&
r.URL.Query().Has(oidParam) && r.URL.Query().Has(sizeParam) {
return pathTerminatedWithMarkerAndURL(r, "", lfsTransferPath, lfsTransferPath, urlPath)
}
} }
// no other APIs are called by git - just treat it as a full repo path. // no other APIs are called by git - just treat it as a full repo path.

View File

@ -122,7 +122,7 @@ func GitLFSHandler(r chi.Router, lfsCtrl *lfs.Controller, urlProvider url.Provid
r.Route("/info/lfs", func(r chi.Router) { r.Route("/info/lfs", func(r chi.Router) {
r.Route("/objects", func(r chi.Router) { r.Route("/objects", func(r chi.Router) {
r.Post("/batch", handlerlfs.HandleLFSTransfer(lfsCtrl, urlProvider)) r.Post("/batch", handlerlfs.HandleLFSTransfer(lfsCtrl, urlProvider))
// direct download and upload handlers for lfs objects // direct upload and download handlers for lfs objects
r.Put("/", handlerlfs.HandleLFSUpload(lfsCtrl, urlProvider)) r.Put("/", handlerlfs.HandleLFSUpload(lfsCtrl, urlProvider))
r.Get("/", handlerlfs.HandleLFSDownload(lfsCtrl, urlProvider)) r.Get("/", handlerlfs.HandleLFSDownload(lfsCtrl, urlProvider))
}) })

View File

@ -282,7 +282,18 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
return nil, err return nil, err
} }
rulesService := rules.ProvideService(transactor, ruleStore, repoStore, spaceStore, protectionManager, auditService, instrumentService, principalInfoCache, userGroupStore, searchService, reporter2, streamer) rulesService := rules.ProvideService(transactor, ruleStore, repoStore, spaceStore, protectionManager, auditService, instrumentService, principalInfoCache, userGroupStore, searchService, reporter2, streamer)
repoController := repo.ProvideController(config, transactor, provider, authorizer, repoStore, spaceStore, pipelineStore, principalStore, executionStore, ruleStore, checkStore, pullReqStore, settingsService, principalInfoCache, protectionManager, gitInterface, spaceFinder, repoFinder, repository, codeownersService, eventsReporter, indexer, resourceLimiter, lockerLocker, auditService, mutexManager, repoIdentifier, repoCheck, publicaccessService, labelService, instrumentService, userGroupStore, searchService, rulesService, streamer) lfsObjectStore := database.ProvideLFSObjectStore(db)
blobConfig, err := server.ProvideBlobStoreConfig(config)
if err != nil {
return nil, err
}
blobStore, err := blob.ProvideStore(ctx, blobConfig)
if err != nil {
return nil, err
}
remoteauthService := remoteauth.ProvideRemoteAuth(tokenStore, principalStore)
lfsController := lfs.ProvideController(authorizer, repoFinder, principalStore, lfsObjectStore, blobStore, remoteauthService, provider)
repoController := repo.ProvideController(config, transactor, provider, authorizer, repoStore, spaceStore, pipelineStore, principalStore, executionStore, ruleStore, checkStore, pullReqStore, settingsService, principalInfoCache, protectionManager, gitInterface, spaceFinder, repoFinder, repository, codeownersService, eventsReporter, indexer, resourceLimiter, lockerLocker, auditService, mutexManager, repoIdentifier, repoCheck, publicaccessService, labelService, instrumentService, userGroupStore, searchService, rulesService, streamer, lfsController)
reposettingsController := reposettings.ProvideController(authorizer, repoFinder, settingsService, auditService) reposettingsController := reposettings.ProvideController(authorizer, repoFinder, settingsService, auditService)
stageStore := database.ProvideStageStore(db) stageStore := database.ProvideStageStore(db)
schedulerScheduler, err := scheduler.ProvideScheduler(stageStore, mutexManager) schedulerScheduler, err := scheduler.ProvideScheduler(stageStore, mutexManager)
@ -438,7 +449,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
if err != nil { if err != nil {
return nil, err return nil, err
} }
lfsObjectStore := database.ProvideLFSObjectStore(db)
githookController := githook.ProvideController(authorizer, principalStore, repoStore, repoFinder, reporter9, eventsReporter, gitInterface, pullReqStore, provider, protectionManager, clientFactory, resourceLimiter, settingsService, preReceiveExtender, updateExtender, postReceiveExtender, streamer, lfsObjectStore) githookController := githook.ProvideController(authorizer, principalStore, repoStore, repoFinder, reporter9, eventsReporter, gitInterface, pullReqStore, provider, protectionManager, clientFactory, resourceLimiter, settingsService, preReceiveExtender, updateExtender, postReceiveExtender, streamer, lfsObjectStore)
serviceaccountController := serviceaccount.NewController(principalUID, authorizer, principalStore, spaceStore, repoStore, tokenStore) serviceaccountController := serviceaccount.NewController(principalUID, authorizer, principalStore, spaceStore, repoStore, tokenStore)
principalController := principal.ProvideController(principalStore, authorizer) principalController := principal.ProvideController(principalStore, authorizer)
@ -446,14 +456,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
v2 := check2.ProvideCheckSanitizers() v2 := check2.ProvideCheckSanitizers()
checkController := check2.ProvideController(transactor, authorizer, spaceStore, checkStore, spaceFinder, repoFinder, gitInterface, v2, streamer) checkController := check2.ProvideController(transactor, authorizer, spaceStore, checkStore, spaceFinder, repoFinder, gitInterface, v2, streamer)
systemController := system.NewController(principalStore, config) systemController := system.NewController(principalStore, config)
blobConfig, err := server.ProvideBlobStoreConfig(config)
if err != nil {
return nil, err
}
blobStore, err := blob.ProvideStore(ctx, blobConfig)
if err != nil {
return nil, err
}
uploadController := upload.ProvideController(authorizer, repoFinder, blobStore) uploadController := upload.ProvideController(authorizer, repoFinder, blobStore)
searcher := keywordsearch.ProvideSearcher(localIndexSearcher) searcher := keywordsearch.ProvideSearcher(localIndexSearcher)
keywordsearchController := keywordsearch2.ProvideController(authorizer, searcher, repoController, spaceController) keywordsearchController := keywordsearch2.ProvideController(authorizer, searcher, repoController, spaceController)
@ -541,8 +543,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
handler4 := router.PackageHandlerProvider(packagesHandler, mavenHandler, genericHandler, pythonHandler, nugetHandler) handler4 := router.PackageHandlerProvider(packagesHandler, mavenHandler, genericHandler, pythonHandler, nugetHandler)
appRouter := router.AppRouterProvider(registryOCIHandler, apiHandler, handler2, handler3, handler4) appRouter := router.AppRouterProvider(registryOCIHandler, apiHandler, handler2, handler3, handler4)
sender := usage.ProvideMediator(ctx, config, spaceFinder, usageMetricStore) sender := usage.ProvideMediator(ctx, config, spaceFinder, usageMetricStore)
remoteauthService := remoteauth.ProvideRemoteAuth(tokenStore, principalStore)
lfsController := lfs.ProvideController(authorizer, repoFinder, principalStore, lfsObjectStore, blobStore, remoteauthService, provider)
routerRouter := router2.ProvideRouter(ctx, config, authenticator, repoController, reposettingsController, executionController, logsController, spaceController, pipelineController, secretController, triggerController, connectorController, templateController, pluginController, pullreqController, webhookController, githookController, gitInterface, serviceaccountController, controller, principalController, usergroupController, checkController, systemController, uploadController, keywordsearchController, infraproviderController, gitspaceController, migrateController, provider, openapiService, appRouter, sender, lfsController) routerRouter := router2.ProvideRouter(ctx, config, authenticator, repoController, reposettingsController, executionController, logsController, spaceController, pipelineController, secretController, triggerController, connectorController, templateController, pluginController, pullreqController, webhookController, githookController, gitInterface, serviceaccountController, controller, principalController, usergroupController, checkController, systemController, uploadController, keywordsearchController, infraproviderController, gitspaceController, migrateController, provider, openapiService, appRouter, sender, lfsController)
serverServer := server2.ProvideServer(config, routerRouter) serverServer := server2.ProvideServer(config, routerRouter)
publickeyService := publickey.ProvidePublicKey(publicKeyStore, principalInfoCache) publickeyService := publickey.ProvidePublicKey(publicKeyStore, principalInfoCache)

View File

@ -25,11 +25,6 @@ import (
"github.com/harness/gitness/git/sha" "github.com/harness/gitness/git/sha"
) )
// lfsPointerMaxSize is the maximum size for an LFS pointer file.
// This is used to identify blobs that are too large to be valid LFS pointers.
// lfs-pointer specification ref: https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md#the-pointer
const lfsPointerMaxSize = 200
type GetBlobParams struct { type GetBlobParams struct {
ReadParams ReadParams
SHA string SHA string
@ -94,7 +89,7 @@ func (s *Service) FindLFSPointers(
var candidateObjects []parser.BatchCheckObject var candidateObjects []parser.BatchCheckObject
for _, obj := range objects { for _, obj := range objects {
if obj.Type == string(TreeNodeTypeBlob) && obj.Size <= lfsPointerMaxSize { if obj.Type == string(TreeNodeTypeBlob) && obj.Size <= parser.LfsPointerMaxSize {
candidateObjects = append(candidateObjects, obj) candidateObjects = append(candidateObjects, obj)
} }
} }

View File

@ -16,15 +16,29 @@ package parser
import ( import (
"bytes" "bytes"
"context"
"errors" "errors"
"regexp" "regexp"
"strconv"
"github.com/rs/zerolog/log"
) )
// LfsPointerMaxSize is the maximum size for an LFS pointer file.
// This is used to identify blobs that are too large to be valid LFS pointers.
// lfs-pointer specification ref: https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md#the-pointer
const LfsPointerMaxSize = 200
const lfsPointerVersionPrefix = "version https://git-lfs.github.com/spec" const lfsPointerVersionPrefix = "version https://git-lfs.github.com/spec"
type LFSPointer struct {
OID string
Size int64
}
var ( var (
regexLFSOID = regexp.MustCompile(`(?m)^oid sha256:([a-f0-9]{64})$`) regexLFSOID = regexp.MustCompile(`(?m)^oid sha256:([a-f0-9]{64})$`)
regexLFSSize = regexp.MustCompile(`(?m)^size [0-9]+$`) regexLFSSize = regexp.MustCompile(`(?m)^size (\d+)+$`)
ErrInvalidLFSPointer = errors.New("invalid lfs pointer") ErrInvalidLFSPointer = errors.New("invalid lfs pointer")
) )
@ -45,3 +59,35 @@ func GetLFSObjectID(content []byte) (string, error) {
return string(oidMatch[1]), nil return string(oidMatch[1]), nil
} }
func IsLFSPointer(
ctx context.Context,
content []byte,
size int64,
) (*LFSPointer, bool) {
if size > LfsPointerMaxSize {
return nil, false
}
if !bytes.HasPrefix(content, []byte(lfsPointerVersionPrefix)) {
return nil, false
}
oidMatch := regexLFSOID.FindSubmatch(content)
if oidMatch == nil {
return nil, false
}
sizeMatch := regexLFSSize.FindSubmatch(content)
if sizeMatch == nil {
return nil, false
}
contentSize, err := strconv.ParseInt(string(sizeMatch[1]), 10, 64)
if err != nil {
log.Ctx(ctx).Warn().Err(err).Msgf("failed to parse lfs pointer size for object ID %s", oidMatch[1])
return nil, false
}
return &LFSPointer{OID: string(oidMatch[1]), Size: contentSize}, true
}

View File

@ -248,7 +248,7 @@ func (s *Service) findLFSPointers(
) (*FindLFSPointersOutput, error) { ) (*FindLFSPointersOutput, error) {
var candidateObjects []parser.BatchCheckObject var candidateObjects []parser.BatchCheckObject
for _, obj := range objects { for _, obj := range objects {
if obj.Type == string(TreeNodeTypeBlob) && obj.Size <= lfsPointerMaxSize { if obj.Type == string(TreeNodeTypeBlob) && obj.Size <= parser.LfsPointerMaxSize {
candidateObjects = append(candidateObjects, obj) candidateObjects = append(candidateObjects, obj)
} }
} }

View File

@ -610,6 +610,7 @@ export interface StringsMap {
language: string language: string
lastTriggeredAt: string lastTriggeredAt: string
leaveAComment: string leaveAComment: string
lfsInfo: string
license: string license: string
lineBreaks: string lineBreaks: string
loading: string loading: string

View File

@ -602,6 +602,7 @@ tagEmpty: There are no tags in your repo. Click the button below to create a tag
newTag: New Tag newTag: New Tag
overview: Overview overview: Overview
fileTooLarge: File is too large to open. {download} fileTooLarge: File is too large to open. {download}
lfsInfo: Stored with Git LFS
clickHereToDownload: Click here to download. clickHereToDownload: Click here to download.
viewFileHistory: View the file at this point in the history viewFileHistory: View the file at this point in the history
viewRepo: View the repository at this point in the history viewRepo: View the repository at this point in the history

View File

@ -25,7 +25,8 @@ import {
Layout, Layout,
StringSubstitute, StringSubstitute,
Tabs, Tabs,
Utils Utils,
Text
} from '@harnessio/uicore' } from '@harnessio/uicore'
import { Icon } from '@harnessio/icons' import { Icon } from '@harnessio/icons'
import { Color } from '@harnessio/design-system' import { Color } from '@harnessio/design-system'
@ -36,6 +37,7 @@ import type { EditorDidMount } from 'react-monaco-editor'
import type { editor } from 'monaco-editor' import type { editor } from 'monaco-editor'
import { SourceCodeViewer } from 'components/SourceCodeViewer/SourceCodeViewer' import { SourceCodeViewer } from 'components/SourceCodeViewer/SourceCodeViewer'
import type { OpenapiContentInfo, RepoFileContent, TypesCommit } from 'services/code' import type { OpenapiContentInfo, RepoFileContent, TypesCommit } from 'services/code'
import { import {
normalizeGitRef, normalizeGitRef,
decodeGitContent, decodeGitContent,
@ -84,7 +86,7 @@ export function FileContent({
const { routes } = useAppContext() const { routes } = useAppContext()
const { getString } = useStrings() const { getString } = useStrings()
const downloadFile = useDownloadRawFile() const downloadFile = useDownloadRawFile()
const { category, isText, isFileTooLarge, isViewable, filename, extension, size, base64Data, rawURL } = const { category, isFileTooLarge, isText, isFileLFS, isViewable, filename, extension, size, base64Data, rawURL } =
useFileContentViewerDecision({ repoMetadata, gitRef, resourcePath, resourceContent }) useFileContentViewerDecision({ repoMetadata, gitRef, resourcePath, resourceContent })
const history = useHistory() const history = useHistory()
const [activeTab, setActiveTab] = React.useState<string>(FileSection.CONTENT) const [activeTab, setActiveTab] = React.useState<string>(FileSection.CONTENT)
@ -171,7 +173,10 @@ export function FileContent({
}, },
lazy: !repoMetadata lazy: !repoMetadata
}) })
const editButtonDisabled = useMemo(() => permsFinal.disabled || !isText, [permsFinal.disabled, isText]) const editButtonDisabled = useMemo(
() => permsFinal.disabled || (!isText && !isFileLFS),
[permsFinal.disabled, isText, isFileLFS]
)
const editAsText = useMemo( const editAsText = useMemo(
() => editButtonDisabled && !isFileTooLarge && category === FileCategory.OTHER, () => editButtonDisabled && !isFileTooLarge && category === FileCategory.OTHER,
[editButtonDisabled, isFileTooLarge, category] [editButtonDisabled, isFileTooLarge, category]
@ -205,6 +210,10 @@ export function FileContent({
} }
} }
const fullRawURL = standalone
? `${window.location.origin}${rawURL.replace(/^\/code/, '')}`
: `${window.location.origin}${getConfig(rawURL)}`.replace('//', '/')
return ( return (
<Container className={css.tabsContainer} ref={ref}> <Container className={css.tabsContainer} ref={ref}>
<Tabs <Tabs
@ -229,8 +238,18 @@ export function FileContent({
/> />
<Container className={css.container} background={Color.WHITE}> <Container className={css.container} background={Color.WHITE}>
<Layout.Horizontal padding="small" className={css.heading}> <Layout.Horizontal padding="small" className={css.heading}>
<Heading level={5} color={Color.BLACK}> <Heading level={5}>
{resourceContent.name} <Layout.Horizontal spacing="small" flex={{ alignItems: 'center' }}>
<span style={{ color: Color.BLACK }}>{resourceContent.name}</span>
{isFileLFS && (
<Layout.Horizontal spacing="xsmall" flex={{ alignItems: 'center' }}>
<Icon name="info" size={12} color={Color.GREY_500} padding={{ left: 'small' }} />
<Text font={{ size: 'small' }} color={Color.GREY_500}>
{getString('lfsInfo')}
</Text>
</Layout.Horizontal>
)}
</Layout.Horizontal>
</Heading> </Heading>
<FlexExpander /> <FlexExpander />
<Layout.Horizontal spacing="xsmall" style={{ alignItems: 'center' }}> <Layout.Horizontal spacing="xsmall" style={{ alignItems: 'center' }}>
@ -408,21 +427,27 @@ export function FileContent({
<Match expr={category}> <Match expr={category}>
<Case val={FileCategory.SVG}> <Case val={FileCategory.SVG}>
<img <img
src={`data:image/svg+xml;base64,${base64Data}`} src={
isFileLFS ? `${fullRawURL}` : `data:image/svg+xml;base64,${base64Data}`
}
alt={filename} alt={filename}
style={{ maxWidth: '100%', maxHeight: '100%' }} style={{ maxWidth: '100%', maxHeight: '100%' }}
/> />
</Case> </Case>
<Case val={FileCategory.IMAGE}> <Case val={FileCategory.IMAGE}>
<img <img
src={`data:image/${extension};base64,${base64Data}`} src={
isFileLFS
? `${fullRawURL}`
: `data:image/${extension};base64,${base64Data}`
}
alt={filename} alt={filename}
style={{ maxWidth: '100%', maxHeight: '100%' }} style={{ maxWidth: '100%', maxHeight: '100%' }}
/> />
</Case> </Case>
<Case val={FileCategory.PDF}> <Case val={FileCategory.PDF}>
<Document <Document
file={`data:application/pdf;base64,${base64Data}`} file={isFileLFS ? fullRawURL : `data:application/pdf;base64,${base64Data}`}
options={{ options={{
// TODO: Configure this to use a local worker/webpack loader // TODO: Configure this to use a local worker/webpack loader
cMapUrl: `https://unpkg.com/pdfjs-dist@${pdfjs.version}/cmaps/`, cMapUrl: `https://unpkg.com/pdfjs-dist@${pdfjs.version}/cmaps/`,
@ -443,19 +468,27 @@ export function FileContent({
</Case> </Case>
<Case val={FileCategory.AUDIO}> <Case val={FileCategory.AUDIO}>
<audio controls> <audio controls>
<source src={`data:audio/${extension};base64,${base64Data}`} /> <source
src={
isFileLFS ? fullRawURL : `data:audio/${extension};base64,${base64Data}`
}
/>
</audio> </audio>
</Case> </Case>
<Case val={FileCategory.VIDEO}> <Case val={FileCategory.VIDEO}>
<video controls height={500}> <video controls height={500}>
<source src={`data:video/${extension};base64,${base64Data}`} /> <source
src={
isFileLFS ? fullRawURL : `data:video/${extension};base64,${base64Data}`
}
/>
</video> </video>
</Case> </Case>
<Case val={FileCategory.TEXT}> <Case val={FileCategory.TEXT}>
<SourceCodeViewer <SourceCodeViewer
editorDidMount={onEditorMount} editorDidMount={onEditorMount}
language={filenameToLanguage(filename)} language={filenameToLanguage(filename)}
source={decodeGitContent(base64Data)} source={isFileLFS ? fullRawURL : decodeGitContent(base64Data)}
/> />
</Case> </Case>
<Case val={FileCategory.SUBMODULE}> <Case val={FileCategory.SUBMODULE}>

View File

@ -533,6 +533,12 @@ export interface OpenapiGetContentOutput {
type?: OpenapiContentType type?: OpenapiContentType
} }
export interface OpenapiRawOutput {
data?: ArrayBuffer
size?: number
sha?: string
}
export interface OpenapiLoginRequest { export interface OpenapiLoginRequest {
login_identifier?: string login_identifier?: string
password?: string password?: string
@ -858,6 +864,8 @@ export interface RepoFileContent {
data_size?: number data_size?: number
encoding?: EnumContentEncodingType encoding?: EnumContentEncodingType
size?: number size?: number
lfs_object_id?: string
lfs_object_size?: number
} }
export interface RepoListPathsOutput { export interface RepoListPathsOutput {
@ -6448,22 +6456,28 @@ export interface GetRawPathParams {
path: string path: string
} }
export type GetRawProps = Omit<GetProps<void, UsererrorError, GetRawQueryParams, GetRawPathParams>, 'path'> & export type GetRawProps = Omit<
GetProps<OpenapiRawOutput, UsererrorError, GetRawQueryParams, GetRawPathParams>,
'path'
> &
GetRawPathParams GetRawPathParams
export const GetRaw = ({ repo_ref, path, ...props }: GetRawProps) => ( export const GetRaw = ({ repo_ref, path, ...props }: GetRawProps) => (
<Get<void, UsererrorError, GetRawQueryParams, GetRawPathParams> <Get<OpenapiRawOutput, UsererrorError, GetRawQueryParams, GetRawPathParams>
path={`/repos/${repo_ref}/raw/${path}`} path={`/repos/${repo_ref}/raw/${path}`}
base={getConfig('code/api/v1')} base={getConfig('code/api/v1')}
{...props} {...props}
/> />
) )
export type UseGetRawProps = Omit<UseGetProps<void, UsererrorError, GetRawQueryParams, GetRawPathParams>, 'path'> & export type UseGetRawProps = Omit<
UseGetProps<OpenapiRawOutput, UsererrorError, GetRawQueryParams, GetRawPathParams>,
'path'
> &
GetRawPathParams GetRawPathParams
export const useGetRaw = ({ repo_ref, path, ...props }: UseGetRawProps) => export const useGetRaw = ({ repo_ref, path, ...props }: UseGetRawProps) =>
useGet<void, UsererrorError, GetRawQueryParams, GetRawPathParams>( useGet<OpenapiRawOutput, UsererrorError, GetRawQueryParams, GetRawPathParams>(
(paramsInPath: GetRawPathParams) => `/repos/${paramsInPath.repo_ref}/raw/${paramsInPath.path}`, (paramsInPath: GetRawPathParams) => `/repos/${paramsInPath.repo_ref}/raw/${paramsInPath.path}`,
{ base: getConfig('code/api/v1'), pathParams: { repo_ref, path }, ...props } { base: getConfig('code/api/v1'), pathParams: { repo_ref, path }, ...props }
) )

View File

@ -6581,6 +6581,10 @@ paths:
type: string type: string
responses: responses:
'200': '200':
content:
application/json:
schema:
$ref: '#/components/schemas/OpenapiRawOutput'
description: OK description: OK
'401': '401':
content: content:
@ -13701,6 +13705,19 @@ components:
$ref: '#/components/schemas/EnumContentEncodingType' $ref: '#/components/schemas/EnumContentEncodingType'
size: size:
type: integer type: integer
lfs_object_id:
type: string
lfs_object_size:
type: integer
type: object
OpenapiRawOutput:
properties:
data:
type: string
size:
type: integer
sha:
type: string
type: object type: object
RepoListPathsOutput: RepoListPathsOutput:
properties: properties:

View File

@ -29,6 +29,7 @@ type UseFileViewerDecisionProps = Pick<GitInfoProps, 'repoMetadata' | 'gitRef' |
interface UseFileViewerDecisionResult { interface UseFileViewerDecisionResult {
category: FileCategory category: FileCategory
isFileTooLarge: boolean isFileTooLarge: boolean
isFileLFS: boolean
isViewable: string | boolean isViewable: string | boolean
filename: string filename: string
extension: string extension: string
@ -95,19 +96,28 @@ export function useFileContentViewerDecision({
: FileCategory.OTHER : FileCategory.OTHER
const isViewable = isPdf || isSVG || isImage || isAudio || isVideo || isText || isSubmodule || isSymlink const isViewable = isPdf || isSVG || isImage || isAudio || isVideo || isText || isSubmodule || isSymlink
const resourceData = resourceContent?.content as RepoContentExtended const resourceData = resourceContent?.content as RepoContentExtended
const isFileLFS = resourceData?.lfs_object_id ? true : false
const isFileTooLarge = const isFileTooLarge =
resourceData?.size && resourceData?.data_size ? resourceData?.size !== resourceData?.data_size : false (isFileLFS
? resourceData?.data_size &&
resourceData?.lfs_object_size &&
resourceData?.lfs_object_size > MAX_VIEWABLE_FILE_SIZE
: resourceData?.data_size && resourceData?.size && resourceData?.data_size !== resourceData?.size) || false
const rawURL = `/code/api/v1/repos/${repoMetadata?.path}/+/raw/${resourcePath}?routingId=${routingId}&git_ref=${gitRef}` const rawURL = `/code/api/v1/repos/${repoMetadata?.path}/+/raw/${resourcePath}?routingId=${routingId}&git_ref=${gitRef}`
return { return {
category, category,
isFileTooLarge, isFileTooLarge,
isViewable,
isText, isText,
isFileLFS,
isViewable,
filename, filename,
extension, extension,
size: resourceData?.size || 0, size: isFileLFS ? resourceData?.lfs_object_size || 0 : resourceData?.size || 0,
// base64 data returned from content API. This snapshot can be truncated by backend // base64 data returned from content API. This snapshot can be truncated by backend
base64Data: resourceData?.data || resourceData?.target || resourceData?.url || '', base64Data: resourceData?.data || resourceData?.target || resourceData?.url || '',
@ -119,7 +129,7 @@ export function useFileContentViewerDecision({
return metadata return metadata
} }
export const MAX_VIEWABLE_FILE_SIZE = 100 * 1024 * 1024 // 100 MB export const MAX_VIEWABLE_FILE_SIZE = 10 * 1024 * 1024 // 10 MB
export enum FileCategory { export enum FileCategory {
MARKDOWN = 'MARKDOWN', MARKDOWN = 'MARKDOWN',