From a3d828f0a2b6a1cae0ea9339cc2dff3d16e45ed9 Mon Sep 17 00:00:00 2001 From: Tudor Macari Date: Wed, 16 Apr 2025 06:05:50 +0000 Subject: [PATCH] feat: [AH-1060]: RPM flows skeleton, RPM package upload (#3599) * resolve PR comments * fix lint * resolve PR comments * fix lint issue * resolve PR comments * resolve PR comments * fix lint issue * feat: [AH-1060]: RPM flows skeleton, RPM package upload/install --- cmd/gitness/wire_gen.go | 9 +- go.mod | 13 +- go.sum | 35 +- .../metadata/get_registry_artifacts.go | 4 +- .../app/api/controller/pkg/rpm/controller.go | 84 +++ .../app/api/controller/pkg/rpm/download.go | 78 +++ .../app/api/controller/pkg/rpm/metadata.go | 69 +++ .../app/api/controller/pkg/rpm/response.go | 55 ++ .../api/controller/pkg/rpm/upload_package.go | 71 +++ registry/app/api/controller/pkg/rpm/wire.go | 41 ++ registry/app/api/handler/packages/handler.go | 2 + registry/app/api/handler/rpm/download.go | 78 +++ registry/app/api/handler/rpm/handler.go | 59 ++ registry/app/api/handler/rpm/repodata.go | 78 +++ registry/app/api/handler/rpm/upload.go | 56 ++ .../api/middleware/request_package_access.go | 1 - registry/app/api/openapi/api.yaml | 1 + .../openapi/contracts/artifact/types.gen.go | 1 + registry/app/api/router/packages/route.go | 15 + registry/app/api/router/wire.go | 12 +- registry/app/api/wire.go | 13 + registry/app/metadata/file.go | 19 +- registry/app/metadata/rpm/metadata.go | 103 ++++ registry/app/pkg/base/base.go | 17 +- registry/app/pkg/filemanager/file_manager.go | 9 +- registry/app/pkg/generic/controller.go | 6 +- registry/app/pkg/maven/local.go | 6 +- registry/app/pkg/maven/utils/utils.go | 2 +- registry/app/pkg/nuget/local.go | 28 +- registry/app/pkg/rpm/helper.go | 509 ++++++++++++++++++ registry/app/pkg/rpm/local.go | 179 ++++++ registry/app/pkg/rpm/local_helper.go | 316 +++++++++++ registry/app/pkg/rpm/proxy.go | 113 ++++ registry/app/pkg/rpm/registry.go | 49 ++ registry/app/pkg/rpm/remote_helper.go | 25 + registry/app/pkg/rpm/types.go | 173 ++++++ registry/app/pkg/rpm/wire.go | 65 +++ registry/app/pkg/types/rpm/types.go | 54 ++ registry/app/store/database.go | 8 +- registry/app/store/database/artifact.go | 50 +- registry/app/store/database/tag.go | 3 + registry/types/tag.go | 3 + 42 files changed, 2445 insertions(+), 67 deletions(-) create mode 100644 registry/app/api/controller/pkg/rpm/controller.go create mode 100644 registry/app/api/controller/pkg/rpm/download.go create mode 100644 registry/app/api/controller/pkg/rpm/metadata.go create mode 100644 registry/app/api/controller/pkg/rpm/response.go create mode 100644 registry/app/api/controller/pkg/rpm/upload_package.go create mode 100644 registry/app/api/controller/pkg/rpm/wire.go create mode 100644 registry/app/api/handler/rpm/download.go create mode 100644 registry/app/api/handler/rpm/handler.go create mode 100644 registry/app/api/handler/rpm/repodata.go create mode 100644 registry/app/api/handler/rpm/upload.go create mode 100644 registry/app/metadata/rpm/metadata.go create mode 100644 registry/app/pkg/rpm/helper.go create mode 100644 registry/app/pkg/rpm/local.go create mode 100644 registry/app/pkg/rpm/local_helper.go create mode 100644 registry/app/pkg/rpm/proxy.go create mode 100644 registry/app/pkg/rpm/registry.go create mode 100644 registry/app/pkg/rpm/remote_helper.go create mode 100644 registry/app/pkg/rpm/types.go create mode 100644 registry/app/pkg/rpm/wire.go create mode 100644 registry/app/pkg/types/rpm/types.go diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 616aed1b0..84e9997ad 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -129,6 +129,7 @@ import ( npm2 "github.com/harness/gitness/registry/app/api/controller/pkg/npm" nuget2 "github.com/harness/gitness/registry/app/api/controller/pkg/nuget" python2 "github.com/harness/gitness/registry/app/api/controller/pkg/python" + rpm2 "github.com/harness/gitness/registry/app/api/controller/pkg/rpm" "github.com/harness/gitness/registry/app/api/router" events12 "github.com/harness/gitness/registry/app/events" "github.com/harness/gitness/registry/app/pkg" @@ -140,6 +141,7 @@ import ( "github.com/harness/gitness/registry/app/pkg/npm" "github.com/harness/gitness/registry/app/pkg/nuget" "github.com/harness/gitness/registry/app/pkg/python" + "github.com/harness/gitness/registry/app/pkg/rpm" database2 "github.com/harness/gitness/registry/app/store/database" "github.com/harness/gitness/registry/gc" webhook3 "github.com/harness/gitness/registry/services/webhook" @@ -548,7 +550,12 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro npmProxy := npm.ProxyProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider, spaceFinder, secretService, npmLocalRegistryHelper) npmController := npm2.ControllerProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, downloadStatRepository, provider, npmLocalRegistry, npmProxy) npmHandler := api2.NewNPMHandlerProvider(npmController, packagesHandler) - handler4 := router.PackageHandlerProvider(packagesHandler, mavenHandler, genericHandler, pythonHandler, nugetHandler, npmHandler) + rpmLocalRegistryHelper := rpm.LocalRegistryHelperProvider(fileManager, artifactRepository) + rpmLocalRegistry := rpm.LocalRegistryProvider(localBase, fileManager, upstreamProxyConfigRepository, transactor, registryRepository, imageRepository, artifactRepository, provider, rpmLocalRegistryHelper) + rpmProxy := rpm.ProxyProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider) + rpmController := rpm2.ControllerProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider, rpmLocalRegistry, rpmProxy) + rpmHandler := api2.NewRpmHandlerProvider(rpmController, packagesHandler) + handler4 := router.PackageHandlerProvider(packagesHandler, mavenHandler, genericHandler, pythonHandler, nugetHandler, npmHandler, rpmHandler) appRouter := router.AppRouterProvider(registryOCIHandler, apiHandler, handler2, handler3, handler4) sender := usage.ProvideMediator(ctx, config, spaceFinder, usageMetricStore) routerRouter := router2.ProvideRouter(ctx, config, authenticator, repoController, reposettingsController, executionController, logsController, spaceController, pipelineController, secretController, triggerController, connectorController, templateController, pluginController, pullreqController, webhookController, githookController, gitInterface, serviceaccountController, controller, principalController, usergroupController, checkController, systemController, uploadController, keywordsearchController, infraproviderController, gitspaceController, migrateController, provider, openapiService, appRouter, sender, lfsController) diff --git a/go.mod b/go.mod index d1570ca8d..bddfebe7a 100644 --- a/go.mod +++ b/go.mod @@ -62,11 +62,12 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0 github.com/pkg/errors v0.9.1 + github.com/posthog/posthog-go v1.3.3 github.com/rs/xid v1.5.0 github.com/rs/zerolog v1.33.0 + github.com/sassoftware/go-rpmutils v0.4.0 github.com/sercand/kuberesolver/v5 v5.1.1 github.com/sirupsen/logrus v1.9.3 - github.com/slack-go/slack v0.14.0 github.com/stretchr/testify v1.10.0 github.com/swaggest/openapi-go v0.2.23 github.com/swaggest/swgui v1.8.1 @@ -98,7 +99,9 @@ require ( dario.cat/mergo v1.0.1 // indirect github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e // indirect github.com/BobuSumisu/aho-corasick v1.0.3 // indirect + github.com/DataDog/zstd v1.5.5 // indirect github.com/KyleBanks/depth v1.2.1 // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/antonmedv/expr v1.15.5 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect @@ -108,6 +111,7 @@ require ( github.com/buildkite/yaml v2.1.0+incompatible // indirect github.com/charmbracelet/lipgloss v0.12.1 // indirect github.com/charmbracelet/x/ansi v0.1.4 // indirect + github.com/cloudflare/circl v1.3.8 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/drone/envsubst v1.0.3 // indirect github.com/fatih/semgroup v1.2.0 // indirect @@ -125,7 +129,6 @@ require ( github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect github.com/h2non/filetype v1.1.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/invopop/yaml v0.2.0 // indirect @@ -133,6 +136,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -145,7 +149,6 @@ require ( github.com/onsi/gomega v1.27.10 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect - github.com/posthog/posthog-go v1.3.3 // indirect github.com/prometheus/client_golang v1.19.1 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect @@ -162,6 +165,8 @@ require ( github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a // indirect + github.com/ulikunitz/xz v0.5.12 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect @@ -189,7 +194,7 @@ require ( cloud.google.com/go/profiler v0.3.1 github.com/Microsoft/go-winio v0.6.1 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect - github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b + github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect diff --git a/go.sum b/go.sum index 2d8709ea4..3a46ca19b 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg6 github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g= github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= @@ -40,6 +42,8 @@ github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA4 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -91,6 +95,7 @@ github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= github.com/bool64/shared v0.1.5/go.mod h1:081yz68YC9jeFB3+Bbmno2RFWvGKv1lPKkMP6MHJlPs= github.com/buildkite/yaml v2.1.0+incompatible h1:xirI+ql5GzfikVNDmt+yeiXpf/v1Gt03qXTtT5WXdr8= github.com/buildkite/yaml v2.1.0+incompatible/go.mod h1:UoU8vbcwu1+vjZq01+KrpSeLBgQQIjL/H7Y6KwikUrI= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -106,6 +111,9 @@ github.com/charmbracelet/x/ansi v0.1.4 h1:IEU3D6+dWwPSgZ6HBH+v6oUuZ/nVawMiWj5831 github.com/charmbracelet/x/ansi v0.1.4/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI= +github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= @@ -254,7 +262,6 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -302,7 +309,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -339,8 +345,6 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gotidy/ptr v1.4.0 h1:7++suUs+HNHMnyz6/AW3SE+4EnBhupPSQTSI7QNijVc= github.com/gotidy/ptr v1.4.0/go.mod h1:MjRBG6/IETiiZGWI8LrRtISXEji+8b/jigmj2q0mEyM= github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -483,6 +487,8 @@ github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -688,6 +694,8 @@ github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sassoftware/go-rpmutils v0.4.0 h1:ojND82NYBxgwrV+mX1CWsd5QJvvEZTKddtCdFLPWhpg= +github.com/sassoftware/go-rpmutils v0.4.0/go.mod h1:3goNWi7PGAT3/dlql2lv3+MSN5jNYPjT5mVcQcIsYzI= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sercand/kuberesolver/v5 v5.1.1 h1:CYH+d67G0sGBj7q5wLK61yzqJJ8gLLC8aeprPTHb6yY= @@ -703,8 +711,6 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/slack-go/slack v0.14.0 h1:6c0UTfbRnvRssZUsZ2qe0Iu07VAMPjRqOa6oX8ewF4k= -github.com/slack-go/slack v0.14.0/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -743,7 +749,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -772,6 +777,8 @@ github.com/tidwall/jsonc v0.3.2/go.mod h1:dw+3CIxqHi+t8eFSpzzMlcVYxKp08UP5CD8/uS github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/unrolled/secure v1.15.0 h1:q7x+pdp8jAHnbzxu6UheP8fRlG/rwYTb8TPuQ3rn9Og= github.com/unrolled/secure v1.15.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -779,6 +786,8 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/vearutop/statigz v1.4.0 h1:RQL0KG3j/uyA/PFpHeZ/L6l2ta920/MxlOAIGEOuwmU= github.com/vearutop/statigz v1.4.0/go.mod h1:LYTolBLiz9oJISwiVKnOQoIwhO1LWX1A7OECawGS8XE= github.com/vinzenz/yaml v0.0.0-20170920082545-91409cdd725d/go.mod h1:mb5taDqMnJiZNRQ3+02W2IFG+oEz1+dTuCXkp4jpkfo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= @@ -824,6 +833,8 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -849,6 +860,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= @@ -894,7 +907,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= @@ -947,6 +962,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -956,7 +973,9 @@ golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= @@ -967,7 +986,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= diff --git a/registry/app/api/controller/metadata/get_registry_artifacts.go b/registry/app/api/controller/metadata/get_registry_artifacts.go index c8e164988..e5eb38264 100644 --- a/registry/app/api/controller/metadata/get_registry_artifacts.go +++ b/registry/app/api/controller/metadata/get_registry_artifacts.go @@ -100,10 +100,10 @@ func (c *APIController) GetAllArtifactsByRegistry( }, nil } } else { - artifacts, err = c.ArtifactStore.GetAllArtifactsByRepo( + artifacts, err = c.ArtifactStore.GetArtifactsByRepo( ctx, regInfo.parentID, regInfo.RegistryIdentifier, regInfo.sortByField, regInfo.sortByOrder, regInfo.limit, regInfo.offset, regInfo.searchTerm, regInfo.labels) - count, _ = c.ArtifactStore.CountAllArtifactsByRepo( + count, _ = c.ArtifactStore.CountArtifactsByRepo( ctx, regInfo.parentID, regInfo.RegistryIdentifier, regInfo.searchTerm, regInfo.labels) if err != nil { diff --git a/registry/app/api/controller/pkg/rpm/controller.go b/registry/app/api/controller/pkg/rpm/controller.go new file mode 100644 index 000000000..68e8c17ca --- /dev/null +++ b/registry/app/api/controller/pkg/rpm/controller.go @@ -0,0 +1,84 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "context" + "mime/multipart" + + urlprovider "github.com/harness/gitness/app/url" + "github.com/harness/gitness/registry/app/pkg/filemanager" + "github.com/harness/gitness/registry/app/pkg/rpm" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/store/database/dbtx" +) + +type Controller interface { + UploadPackageFile( + ctx context.Context, + info rpmtype.ArtifactInfo, + file multipart.File, + ) *PutArtifactResponse + + DownloadPackageFile( + ctx context.Context, + info rpmtype.ArtifactInfo, + ) *GetArtifactResponse + + GetRepoData( + ctx context.Context, + info rpmtype.ArtifactInfo, + fileName string, + ) *GetRepoDataResponse +} + +// Controller handles RPM package operations. +type controller struct { + fileManager filemanager.FileManager + proxyStore store.UpstreamProxyConfigRepository + tx dbtx.Transactor + registryDao store.RegistryRepository + imageDao store.ImageRepository + artifactDao store.ArtifactRepository + urlProvider urlprovider.Provider + local rpm.LocalRegistry + proxy rpm.Proxy +} + +// NewController creates a new RPM controller. +func NewController( + proxyStore store.UpstreamProxyConfigRepository, + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + fileManager filemanager.FileManager, + tx dbtx.Transactor, + urlProvider urlprovider.Provider, + local rpm.LocalRegistry, + proxy rpm.Proxy, +) Controller { + return &controller{ + proxyStore: proxyStore, + registryDao: registryDao, + imageDao: imageDao, + artifactDao: artifactDao, + fileManager: fileManager, + tx: tx, + urlProvider: urlProvider, + local: local, + proxy: proxy, + } +} diff --git a/registry/app/api/controller/pkg/rpm/download.go b/registry/app/api/controller/pkg/rpm/download.go new file mode 100644 index 000000000..f003f8093 --- /dev/null +++ b/registry/app/api/controller/pkg/rpm/download.go @@ -0,0 +1,78 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "context" + "fmt" + + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/response" + "github.com/harness/gitness/registry/app/pkg/rpm" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" + registrytypes "github.com/harness/gitness/registry/types" +) + +func (c *controller) DownloadPackageFile( + ctx context.Context, + info rpmtype.ArtifactInfo, +) *GetArtifactResponse { + f := func(registry registrytypes.Registry, a pkg.Artifact) response.Response { + info.RegIdentifier = registry.Name + info.RegistryID = registry.ID + info.Registry = registry + rpmRegistry, ok := a.(rpm.Registry) + if !ok { + return &GetArtifactResponse{ + BaseResponse{ + fmt.Errorf("invalid registry type: expected rpm.Registry"), + nil, + }, + "", nil, nil, + } + } + headers, fileReader, readCloser, redirectURL, err := rpmRegistry.DownloadPackageFile(ctx, info) + return &GetArtifactResponse{ + BaseResponse{ + err, + headers, + }, + redirectURL, fileReader, readCloser, + } + } + + result, err := base.ProxyWrapper(ctx, c.registryDao, f, info) + if err != nil { + return &GetArtifactResponse{ + BaseResponse{ + err, + nil, + }, + "", nil, nil, + } + } + getResponse, ok := result.(*GetArtifactResponse) + if !ok { + return &GetArtifactResponse{ + BaseResponse{ + fmt.Errorf("invalid response type: expected GetArtifactResponse"), + nil, + }, + "", nil, nil, + } + } + return getResponse +} diff --git a/registry/app/api/controller/pkg/rpm/metadata.go b/registry/app/api/controller/pkg/rpm/metadata.go new file mode 100644 index 000000000..60a9f9712 --- /dev/null +++ b/registry/app/api/controller/pkg/rpm/metadata.go @@ -0,0 +1,69 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "context" + "fmt" + + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/response" + "github.com/harness/gitness/registry/app/pkg/rpm" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" + registrytypes "github.com/harness/gitness/registry/types" +) + +// GetRepoData represents the metadata of a RPM package. +func (c *controller) GetRepoData(ctx context.Context, info rpmtype.ArtifactInfo, fileName string) *GetRepoDataResponse { + f := func(registry registrytypes.Registry, a pkg.Artifact) response.Response { + info.RegIdentifier = registry.Name + info.RegistryID = registry.ID + info.Registry = registry + rpmRegistry, ok := a.(rpm.Registry) + if !ok { + return &GetRepoDataResponse{ + BaseResponse{ + fmt.Errorf("invalid registry type: expected rpm.Registry"), + nil, + }, + "", nil, nil, + } + } + + responseHeaders, fileReader, _, redirectURL, err := rpmRegistry.GetRepoData(ctx, info, fileName) + + return &GetRepoDataResponse{ + BaseResponse{ + err, + responseHeaders, + }, + redirectURL, fileReader, fileReader, + } + } + + result, err := base.ProxyWrapper(ctx, c.registryDao, f, info) + metadataResponse, ok := result.(*GetRepoDataResponse) + if !ok { + return &GetRepoDataResponse{ + BaseResponse{ + err, + nil, + }, + "", nil, nil, + } + } + return metadataResponse +} diff --git a/registry/app/api/controller/pkg/rpm/response.go b/registry/app/api/controller/pkg/rpm/response.go new file mode 100644 index 000000000..a94af02d9 --- /dev/null +++ b/registry/app/api/controller/pkg/rpm/response.go @@ -0,0 +1,55 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "io" + + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/response" + "github.com/harness/gitness/registry/app/storage" +) + +var _ response.Response = (*GetRepoDataResponse)(nil) +var _ response.Response = (*GetArtifactResponse)(nil) +var _ response.Response = (*PutArtifactResponse)(nil) + +type BaseResponse struct { + Error error + ResponseHeaders *commons.ResponseHeaders +} + +func (r BaseResponse) GetError() error { + return r.Error +} + +type GetRepoDataResponse struct { + BaseResponse + RedirectURL string + Body *storage.FileReader + ReadCloser io.ReadCloser +} + +type GetArtifactResponse struct { + BaseResponse + RedirectURL string + Body *storage.FileReader + ReadCloser io.ReadCloser +} + +type PutArtifactResponse struct { + BaseResponse + Sha256 string +} diff --git a/registry/app/api/controller/pkg/rpm/upload_package.go b/registry/app/api/controller/pkg/rpm/upload_package.go new file mode 100644 index 000000000..03b431e46 --- /dev/null +++ b/registry/app/api/controller/pkg/rpm/upload_package.go @@ -0,0 +1,71 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "context" + "fmt" + "mime/multipart" + + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/response" + "github.com/harness/gitness/registry/app/pkg/rpm" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" + registrytypes "github.com/harness/gitness/registry/types" +) + +// UploadPackageFile uploads the package file to the storage. +func (c *controller) UploadPackageFile( + ctx context.Context, + info rpmtype.ArtifactInfo, + file multipart.File, +) *PutArtifactResponse { + f := func(registry registrytypes.Registry, a pkg.Artifact) response.Response { + info.RegIdentifier = registry.Name + info.RegistryID = registry.ID + rpmRegistry, ok := a.(rpm.Registry) + if !ok { + return &PutArtifactResponse{ + BaseResponse{ + Error: fmt.Errorf("invalid registry type: expected rpm.Registry"), + ResponseHeaders: nil, + }, + "", + } + } + headers, sha256, err := rpmRegistry.UploadPackageFile(ctx, info, file) + return &PutArtifactResponse{ + BaseResponse{ + Error: err, + ResponseHeaders: headers, + }, + sha256, + } + } + + result, err := base.NoProxyWrapper(ctx, c.registryDao, f, info) + rs, ok := result.(*PutArtifactResponse) + if !ok { + return &PutArtifactResponse{ + BaseResponse{ + Error: err, + ResponseHeaders: nil, + }, + "", + } + } + return rs +} diff --git a/registry/app/api/controller/pkg/rpm/wire.go b/registry/app/api/controller/pkg/rpm/wire.go new file mode 100644 index 000000000..08c88d956 --- /dev/null +++ b/registry/app/api/controller/pkg/rpm/wire.go @@ -0,0 +1,41 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + urlprovider "github.com/harness/gitness/app/url" + "github.com/harness/gitness/registry/app/pkg/filemanager" + "github.com/harness/gitness/registry/app/pkg/rpm" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/store/database/dbtx" + + "github.com/google/wire" +) + +func ControllerProvider( + proxyStore store.UpstreamProxyConfigRepository, + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + fileManager filemanager.FileManager, + tx dbtx.Transactor, + urlProvider urlprovider.Provider, + local rpm.LocalRegistry, + proxy rpm.Proxy, +) Controller { + return NewController(proxyStore, registryDao, imageDao, artifactDao, fileManager, tx, urlProvider, local, proxy) +} + +var ControllerSet = wire.NewSet(ControllerProvider) diff --git a/registry/app/api/handler/packages/handler.go b/registry/app/api/handler/packages/handler.go index ae8156916..44179141d 100644 --- a/registry/app/api/handler/packages/handler.go +++ b/registry/app/api/handler/packages/handler.go @@ -101,6 +101,7 @@ const ( PathPackageTypePython PathPackageType = "python" PathPackageTypeNuget PathPackageType = "nuget" PathPackageTypeNpm PathPackageType = "npm" + PathPackageTypeRPM PathPackageType = "rpm" ) var packageTypeMap = map[PathPackageType]artifact2.PackageType{ @@ -109,6 +110,7 @@ var packageTypeMap = map[PathPackageType]artifact2.PackageType{ PathPackageTypePython: artifact2.PackageTypePYTHON, PathPackageTypeNuget: artifact2.PackageTypeNUGET, PathPackageTypeNpm: artifact2.PackageTypeNPM, + PathPackageTypeRPM: artifact2.PackageTypeRPM, } func (h *handler) GetAuthenticator() authn.Authenticator { diff --git a/registry/app/api/handler/rpm/download.go b/registry/app/api/handler/rpm/download.go new file mode 100644 index 000000000..e936d6d6d --- /dev/null +++ b/registry/app/api/handler/rpm/download.go @@ -0,0 +1,78 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "fmt" + "net/http" + + "github.com/harness/gitness/registry/app/pkg/commons" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" + "github.com/harness/gitness/registry/request" + + "github.com/rs/zerolog/log" +) + +func (h *handler) DownloadPackageFile(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + info, ok := request.ArtifactInfoFrom(ctx).(*rpmtype.ArtifactInfo) + if !ok { + h.HandleErrors(ctx, []error{fmt.Errorf("failed to fetch info from context")}, w) + return + } + info.Version = r.PathValue("version") + info.Image = r.PathValue("name") + info.Arch = r.PathValue("architecture") + info.FileName = r.PathValue("file") + response := h.controller.DownloadPackageFile(ctx, *info) + if response == nil { + h.HandleErrors(ctx, []error{fmt.Errorf("failed to get response from controller")}, w) + return + } + + defer func() { + if response.Body != nil { + err := response.Body.Close() + if err != nil { + log.Ctx(ctx).Error().Msgf("Failed to close body: %v", err) + } + } + + if response.ReadCloser != nil { + err := response.ReadCloser.Close() + if err != nil { + log.Ctx(ctx).Error().Msgf("Failed to close read closer: %v", err) + } + } + }() + + if response.GetError() != nil { + h.HandleError(ctx, w, response.GetError()) + return + } + + if response.RedirectURL != "" { + http.Redirect(w, r, response.RedirectURL, http.StatusTemporaryRedirect) + return + } + + err := commons.ServeContent(w, r, response.Body, info.FileName, response.ReadCloser) + if err != nil { + log.Ctx(ctx).Error().Msgf("Failed to serve content: %v", err) + h.HandleError(ctx, w, err) + return + } + response.ResponseHeaders.WriteToResponse(w) +} diff --git a/registry/app/api/handler/rpm/handler.go b/registry/app/api/handler/rpm/handler.go new file mode 100644 index 000000000..d4861a637 --- /dev/null +++ b/registry/app/api/handler/rpm/handler.go @@ -0,0 +1,59 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "net/http" + + rpm "github.com/harness/gitness/registry/app/api/controller/pkg/rpm" + "github.com/harness/gitness/registry/app/api/handler/packages" + "github.com/harness/gitness/registry/app/pkg" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" +) + +type Handler interface { + pkg.ArtifactInfoProvider + UploadPackageFile(writer http.ResponseWriter, request *http.Request) + GetRepoData(writer http.ResponseWriter, request *http.Request) + DownloadPackageFile(http.ResponseWriter, *http.Request) +} + +type handler struct { + packages.Handler + controller rpm.Controller +} + +func NewHandler( + controller rpm.Controller, + packageHandler packages.Handler, +) Handler { + return &handler{ + Handler: packageHandler, + controller: controller, + } +} + +var _ Handler = (*handler)(nil) + +func (h *handler) GetPackageArtifactInfo(r *http.Request) (pkg.PackageArtifactInfo, error) { + info, err := h.Handler.GetArtifactInfo(r) + if err != nil { + return nil, err + } + + return &rpmtype.ArtifactInfo{ + ArtifactInfo: info, + }, nil +} diff --git a/registry/app/api/handler/rpm/repodata.go b/registry/app/api/handler/rpm/repodata.go new file mode 100644 index 000000000..a0ae12a1e --- /dev/null +++ b/registry/app/api/handler/rpm/repodata.go @@ -0,0 +1,78 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "fmt" + "net/http" + + "github.com/harness/gitness/app/api/render" + "github.com/harness/gitness/registry/app/pkg/commons" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" + "github.com/harness/gitness/registry/request" + + "github.com/rs/zerolog/log" +) + +func (h *handler) GetRepoData(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + contextInfo := request.ArtifactInfoFrom(ctx) + info, ok := contextInfo.(*rpmtype.ArtifactInfo) + if !ok { + render.TranslatedUserError(r.Context(), w, fmt.Errorf("invalid request context")) + return + } + fileName := r.PathValue("file") + + packageData := h.controller.GetRepoData(r.Context(), *info, fileName) + if packageData == nil { + h.HandleErrors(ctx, []error{fmt.Errorf("failed to get response from controller")}, w) + return + } + + defer func() { + if packageData.Body != nil { + err := packageData.Body.Close() + if err != nil { + log.Ctx(ctx).Error().Msgf("Failed to close body: %v", err) + } + } + + if packageData.ReadCloser != nil { + err := packageData.ReadCloser.Close() + if err != nil { + log.Ctx(ctx).Error().Msgf("Failed to close read closer: %v", err) + } + } + }() + + if packageData.GetError() != nil { + h.HandleError(ctx, w, packageData.GetError()) + return + } + + if packageData.RedirectURL != "" { + http.Redirect(w, r, packageData.RedirectURL, http.StatusTemporaryRedirect) + return + } + + err := commons.ServeContent(w, r, packageData.Body, info.FileName, packageData.ReadCloser) + if err != nil { + log.Ctx(ctx).Error().Msgf("Failed to serve content: %v", err) + h.HandleError(ctx, w, err) + return + } + packageData.ResponseHeaders.WriteToResponse(w) +} diff --git a/registry/app/api/handler/rpm/upload.go b/registry/app/api/handler/rpm/upload.go new file mode 100644 index 000000000..1f8a8a445 --- /dev/null +++ b/registry/app/api/handler/rpm/upload.go @@ -0,0 +1,56 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "fmt" + "net/http" + + "github.com/harness/gitness/registry/app/dist_temp/errcode" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" + "github.com/harness/gitness/registry/request" +) + +const formFileKey = "file" + +func (h *handler) UploadPackageFile(w http.ResponseWriter, r *http.Request) { + file, _, err := r.FormFile(formFileKey) + if err != nil { + h.HandleErrors2(r.Context(), errcode.ErrCodeInvalidRequest.WithMessage(fmt.Sprintf("failed to parse file: %s, "+ + "please provide correct file path ", err.Error())), w) + return + } + defer file.Close() + + contextInfo := request.ArtifactInfoFrom(r.Context()) + info, ok := contextInfo.(*rpmtype.ArtifactInfo) + if !ok { + h.HandleErrors2(r.Context(), errcode.ErrCodeInvalidRequest.WithMessage("failed to fetch info from context"), w) + return + } + + response := h.controller.UploadPackageFile(r.Context(), *info, file) + if response.GetError() != nil { + h.HandleError(r.Context(), w, response.GetError()) + return + } + + response.ResponseHeaders.WriteToResponse(w) + _, err = w.Write([]byte(fmt.Sprintf("Pushed.\nSha256: %s", response.Sha256))) + if err != nil { + h.HandleError(r.Context(), w, err) + return + } +} diff --git a/registry/app/api/middleware/request_package_access.go b/registry/app/api/middleware/request_package_access.go index bbbedac83..66ea91d38 100644 --- a/registry/app/api/middleware/request_package_access.go +++ b/registry/app/api/middleware/request_package_access.go @@ -22,7 +22,6 @@ import ( "github.com/harness/gitness/types/enum" ) -// StoreOriginalURL stores the original URL in the context. func RequestPackageAccess( packageHandler packages.Handler, reqPermissions ...enum.Permission, diff --git a/registry/app/api/openapi/api.yaml b/registry/app/api/openapi/api.yaml index b5fee0100..1606bdc84 100644 --- a/registry/app/api/openapi/api.yaml +++ b/registry/app/api/openapi/api.yaml @@ -2707,6 +2707,7 @@ components: - HELM - NUGET - NPM + - RPM SectionType: type: string description: refers to client setup section type diff --git a/registry/app/api/openapi/contracts/artifact/types.gen.go b/registry/app/api/openapi/contracts/artifact/types.gen.go index 917b4b5b7..42c4ccaf8 100644 --- a/registry/app/api/openapi/contracts/artifact/types.gen.go +++ b/registry/app/api/openapi/contracts/artifact/types.gen.go @@ -33,6 +33,7 @@ const ( PackageTypeNPM PackageType = "NPM" PackageTypeNUGET PackageType = "NUGET" PackageTypePYTHON PackageType = "PYTHON" + PackageTypeRPM PackageType = "RPM" ) // Defines values for RegistryType. diff --git a/registry/app/api/router/packages/route.go b/registry/app/api/router/packages/route.go index e0c784727..034fd6d0f 100644 --- a/registry/app/api/router/packages/route.go +++ b/registry/app/api/router/packages/route.go @@ -25,6 +25,7 @@ import ( "github.com/harness/gitness/registry/app/api/handler/nuget" "github.com/harness/gitness/registry/app/api/handler/packages" "github.com/harness/gitness/registry/app/api/handler/python" + "github.com/harness/gitness/registry/app/api/handler/rpm" "github.com/harness/gitness/registry/app/api/middleware" "github.com/harness/gitness/types/enum" @@ -48,6 +49,7 @@ func NewRouter( pythonHandler python.Handler, nugetHandler nuget.Handler, npmHandler npm.Handler, + rpmHandler rpm.Handler, ) Handler { r := chi.NewRouter() @@ -190,6 +192,19 @@ func NewRouter( registerRevisionRoutes(r, npmHandler, packageHandler) }) }) + r.Route("/rpm", func(r chi.Router) { + r.Use(middlewareauthn.Attempt(packageHandler.GetAuthenticator())) + r.Use(middleware.CheckAuth()) + r.With(middleware.StoreArtifactInfo(rpmHandler)). + With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsUpload)). + Put("/*", rpmHandler.UploadPackageFile) + r.With(middleware.StoreArtifactInfo(rpmHandler)). + With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)). + Get("/repodata/{file}", rpmHandler.GetRepoData) + r.With(middleware.StoreArtifactInfo(rpmHandler)). + With(middleware.RequestPackageAccess(packageHandler, enum.PermissionArtifactsDownload)). + Get("/package/{name}/{version}/{architecture}/{file}", rpmHandler.DownloadPackageFile) + }) }) return r diff --git a/registry/app/api/router/wire.go b/registry/app/api/router/wire.go index 346ad9e3e..516b0a0a6 100644 --- a/registry/app/api/router/wire.go +++ b/registry/app/api/router/wire.go @@ -29,6 +29,7 @@ import ( hoci "github.com/harness/gitness/registry/app/api/handler/oci" "github.com/harness/gitness/registry/app/api/handler/packages" "github.com/harness/gitness/registry/app/api/handler/python" + rpm "github.com/harness/gitness/registry/app/api/handler/rpm" generic2 "github.com/harness/gitness/registry/app/api/router/generic" "github.com/harness/gitness/registry/app/api/router/harness" mavenRouter "github.com/harness/gitness/registry/app/api/router/maven" @@ -122,8 +123,17 @@ func PackageHandlerProvider( pypiHandler python.Handler, nugetHandler nuget.Handler, npmHandler npm.Handler, + rpmHandler rpm.Handler, ) packagerrouter.Handler { - return packagerrouter.NewRouter(handler, mavenHandler, genericHandler, pypiHandler, nugetHandler, npmHandler) + return packagerrouter.NewRouter( + handler, + mavenHandler, + genericHandler, + pypiHandler, + nugetHandler, + npmHandler, + rpmHandler, + ) } var WireSet = wire.NewSet(APIHandlerProvider, OCIHandlerProvider, AppRouterProvider, diff --git a/registry/app/api/wire.go b/registry/app/api/wire.go index aaed93ec1..37c216e26 100644 --- a/registry/app/api/wire.go +++ b/registry/app/api/wire.go @@ -24,6 +24,7 @@ import ( "github.com/harness/gitness/registry/app/api/controller/pkg/npm" nuget2 "github.com/harness/gitness/registry/app/api/controller/pkg/nuget" python2 "github.com/harness/gitness/registry/app/api/controller/pkg/python" + rpm2 "github.com/harness/gitness/registry/app/api/controller/pkg/rpm" "github.com/harness/gitness/registry/app/api/handler/generic" mavenhandler "github.com/harness/gitness/registry/app/api/handler/maven" npm2 "github.com/harness/gitness/registry/app/api/handler/npm" @@ -31,6 +32,7 @@ import ( ocihandler "github.com/harness/gitness/registry/app/api/handler/oci" "github.com/harness/gitness/registry/app/api/handler/packages" pypi2 "github.com/harness/gitness/registry/app/api/handler/python" + rpm "github.com/harness/gitness/registry/app/api/handler/rpm" "github.com/harness/gitness/registry/app/api/router" storagedriver "github.com/harness/gitness/registry/app/driver" "github.com/harness/gitness/registry/app/driver/factory" @@ -45,6 +47,7 @@ import ( npm22 "github.com/harness/gitness/registry/app/pkg/npm" "github.com/harness/gitness/registry/app/pkg/nuget" "github.com/harness/gitness/registry/app/pkg/python" + rpmregistry "github.com/harness/gitness/registry/app/pkg/rpm" "github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/registry/app/store/database" "github.com/harness/gitness/registry/config" @@ -155,6 +158,13 @@ func NewNPMHandlerProvider( return npm2.NewHandler(controller, packageHandler) } +func NewRpmHandlerProvider( + controller rpm2.Controller, + packageHandler packages.Handler, +) rpm.Handler { + return rpm.NewHandler(controller, packageHandler) +} + func NewGenericHandlerProvider( spaceStore corestore.SpaceStore, controller *generic2.Controller, tokenStore corestore.TokenStore, userCtrl *usercontroller.Controller, authenticator authn.Authenticator, urlProvider urlprovider.Provider, @@ -180,6 +190,7 @@ var WireSet = wire.NewSet( NewPythonHandlerProvider, NewNugetHandlerProvider, NewNPMHandlerProvider, + NewRpmHandlerProvider, database.WireSet, pkg.WireSet, docker.WireSet, @@ -195,6 +206,8 @@ var WireSet = wire.NewSet( nuget2.ControllerSet, npm.ControllerSet, base.WireSet, + rpm2.ControllerSet, + rpmregistry.WireSet, ) func Wire(_ *types.Config) (RegistryApp, error) { diff --git a/registry/app/metadata/file.go b/registry/app/metadata/file.go index 4654e13da..16c9de4c8 100644 --- a/registry/app/metadata/file.go +++ b/registry/app/metadata/file.go @@ -1,16 +1,16 @@ // Copyright 2023 Harness, Inc. // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package metadata @@ -18,4 +18,5 @@ type File struct { Size int64 `json:"size"` Filename string `json:"file_name"` CreatedAt int64 `json:"created_at"` + Sha256 string `json:"sha256"` } diff --git a/registry/app/metadata/rpm/metadata.go b/registry/app/metadata/rpm/metadata.go new file mode 100644 index 000000000..74d45b6d8 --- /dev/null +++ b/registry/app/metadata/rpm/metadata.go @@ -0,0 +1,103 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import "github.com/harness/gitness/registry/app/metadata" + +var _ metadata.Metadata = (*RpmMetadata)(nil) + +type Metadata struct { + VersionMetadata VersionMetadata `json:"version_metadata,omitempty"` + FileMetadata FileMetadata `json:"file_metadata,omitempty"` +} + +type VersionMetadata struct { + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` +} + +type FileMetadata struct { + Architecture string `json:"architecture,omitempty"` + Epoch string `json:"epoch,omitempty"` + Version string `json:"version,omitempty"` + Release string `json:"release,omitempty"` + Vendor string `json:"vendor,omitempty"` + Group string `json:"group,omitempty"` + Packager string `json:"packager,omitempty"` + SourceRpm string `json:"source_rpm,omitempty"` + BuildHost string `json:"build_host,omitempty"` + BuildTime uint64 `json:"build_time,omitempty"` + FileTime uint64 `json:"file_time,omitempty"` + InstalledSize uint64 `json:"installed_size,omitempty"` + ArchiveSize uint64 `json:"archive_size,omitempty"` + + Provides []*Entry `json:"provide,omitempty"` + Requires []*Entry `json:"require,omitempty"` + Conflicts []*Entry `json:"conflict,omitempty"` + Obsoletes []*Entry `json:"obsolete,omitempty"` + + Files []*File `json:"files,omitempty"` + + Changelogs []*Changelog `json:"changelogs,omitempty"` +} + +type Entry struct { + Name string `json:"name" xml:"name,attr"` + Flags string `json:"flags,omitempty" xml:"flags,attr,omitempty"` + Version string `json:"version,omitempty" xml:"ver,attr,omitempty"` + Epoch string `json:"epoch,omitempty" xml:"epoch,attr,omitempty"` + Release string `json:"release,omitempty" xml:"rel,attr,omitempty"` +} + +type File struct { + Path string `json:"path" xml:",chardata"` // nolint: tagliatelle + Type string `json:"type,omitempty" xml:"type,attr,omitempty"` + IsExecutable bool `json:"is_executable" xml:"-"` +} + +type Changelog struct { + Author string `json:"author,omitempty" xml:"author,attr"` + Date int64 `json:"date,omitempty" xml:"date,attr"` + Text string `json:"text,omitempty" xml:",chardata"` // nolint: tagliatelle +} + +// RpmMetadata represents the metadata for a RPM package. +// +//nolint:revive +type RpmMetadata struct { + Metadata + Files []metadata.File `json:"files"` + FileCount int64 `json:"file_count"` + Size int64 `json:"size"` +} + +func (p *RpmMetadata) GetSize() int64 { + return p.Size +} + +func (p *RpmMetadata) UpdateSize(size int64) { + p.Size += size +} + +func (p *RpmMetadata) GetFiles() []metadata.File { + return p.Files +} + +func (p *RpmMetadata) SetFiles(files []metadata.File) { + p.Files = files + p.FileCount = int64(len(files)) +} diff --git a/registry/app/pkg/base/base.go b/registry/app/pkg/base/base.go index 08c5a503f..5a99b4fac 100644 --- a/registry/app/pkg/base/base.go +++ b/registry/app/pkg/base/base.go @@ -56,7 +56,9 @@ type LocalBase interface { Upload( ctx context.Context, info pkg.ArtifactInfo, - fileName, version, path string, + fileName, + version, + path string, file io.ReadCloser, metadata metadata.Metadata, ) (*commons.ResponseHeaders, string, error) @@ -121,7 +123,9 @@ func (l *localBase) UploadFile( func (l *localBase) Upload( ctx context.Context, info pkg.ArtifactInfo, - fileName, version, path string, + fileName, + version, + path string, file io.ReadCloser, metadata metadata.Metadata, ) (*commons.ResponseHeaders, string, error) { @@ -228,10 +232,8 @@ func (l *localBase) Download( path := "/" + info.Image + "/" + version + "/" + fileName reg, _ := l.registryDao.GetByRootParentIDAndName(ctx, info.RootParentID, info.RegIdentifier) - fileReader, _, redirectURL, err := l.fileManager.DownloadFile(ctx, path, types.Registry{ - ID: reg.ID, - Name: info.RegIdentifier, - }, info.RootIdentifier) + fileReader, _, redirectURL, err := l.fileManager.DownloadFile(ctx, path, reg.ID, + info.RegIdentifier, info.RootIdentifier) if err != nil { return responseHeaders, nil, "", err } @@ -296,6 +298,7 @@ func (l *localBase) updateMetadata( files = append(files, metadata.File{ Size: fileInfo.Size, Filename: fileInfo.Filename, CreatedAt: time.Now().UnixMilli(), + Sha256: fileInfo.Sha256, }) inputMetadata.SetFiles(files) inputMetadata.UpdateSize(fileInfo.Size) @@ -303,7 +306,7 @@ func (l *localBase) updateMetadata( } else { files = append(files, metadata.File{ Size: fileInfo.Size, Filename: fileInfo.Filename, - CreatedAt: time.Now().UnixMilli(), + Sha256: fileInfo.Sha256, CreatedAt: time.Now().UnixMilli(), }) inputMetadata.SetFiles(files) inputMetadata.UpdateSize(fileInfo.Size) diff --git a/registry/app/pkg/filemanager/file_manager.go b/registry/app/pkg/filemanager/file_manager.go index c513e615a..19941bf8d 100644 --- a/registry/app/pkg/filemanager/file_manager.go +++ b/registry/app/pkg/filemanager/file_manager.go @@ -209,13 +209,14 @@ func (f *FileManager) SaveNode( func (f *FileManager) DownloadFile( ctx context.Context, filePath string, - regInfo types.Registry, + registryID int64, + registryIdentifier string, rootIdentifier string, ) (fileReader *storage.FileReader, size int64, redirectURL string, err error) { - node, err := f.nodesDao.GetByPathAndRegistryID(ctx, regInfo.ID, filePath) + node, err := f.nodesDao.GetByPathAndRegistryID(ctx, registryID, filePath) if err != nil { return nil, 0, "", fmt.Errorf("failed to get the file for path: %s, "+ - "with registry: %s", filePath, regInfo.Name) + "with registry: %s", filePath, registryIdentifier) } blob, err := f.genericBlobDao.FindByID(ctx, node.BlobID) @@ -226,7 +227,7 @@ func (f *FileManager) DownloadFile( completeFilaPath := path.Join(rootPathString + rootIdentifier + rootPathString + files + rootPathString + blob.Sha256) // - blobContext := f.App.GetBlobsContext(ctx, regInfo.Name, rootIdentifier) + blobContext := f.App.GetBlobsContext(ctx, registryIdentifier, rootIdentifier) reader, redirectURL, err := blobContext.genericBlobStore.Get(ctx, completeFilaPath, blob.Size) if err != nil { diff --git a/registry/app/pkg/generic/controller.go b/registry/app/pkg/generic/controller.go index 2cae2fe74..384d8e991 100644 --- a/registry/app/pkg/generic/controller.go +++ b/registry/app/pkg/generic/controller.go @@ -225,10 +225,8 @@ func (c Controller) PullArtifact(ctx context.Context, info pkg.GenericArtifactIn } path := "/" + info.Image + "/" + info.Version + "/" + info.FileName - fileReader, _, redirectURL, err := c.fileManager.DownloadFile(ctx, path, types.Registry{ - ID: info.RegistryID, - Name: info.RegIdentifier, - }, info.RootIdentifier) + fileReader, _, redirectURL, err := c.fileManager.DownloadFile(ctx, path, info.RegistryID, + info.RegIdentifier, info.RootIdentifier) if err != nil { return responseHeaders, nil, "", errcode.ErrCodeRootNotFound.WithDetail(err) } diff --git a/registry/app/pkg/maven/local.go b/registry/app/pkg/maven/local.go index 5b3a62bec..250978fdd 100644 --- a/registry/app/pkg/maven/local.go +++ b/registry/app/pkg/maven/local.go @@ -106,10 +106,8 @@ func (r *LocalRegistry) FetchArtifact(ctx context.Context, info pkg.MavenArtifac } var fileReader *storage.FileReader if serveFile { - fileReader, _, redirectURL, err = r.fileManager.DownloadFile(ctx, filePath, types.Registry{ - ID: info.RegistryID, - Name: info.RootIdentifier, - }, info.RootIdentifier) + fileReader, _, redirectURL, err = r.fileManager.DownloadFile(ctx, filePath, info.RegistryID, + info.RootIdentifier, info.RootIdentifier) if err != nil { return processError(err) } diff --git a/registry/app/pkg/maven/utils/utils.go b/registry/app/pkg/maven/utils/utils.go index 3b761f4c9..e04f51e67 100644 --- a/registry/app/pkg/maven/utils/utils.go +++ b/registry/app/pkg/maven/utils/utils.go @@ -95,7 +95,7 @@ func SetHeaders( responseHeaders.Code = http.StatusOK responseHeaders.Headers["Content-Length"] = fmt.Sprintf("%d", fileInfo.Size) responseHeaders.Headers["LastModified"] = fmt.Sprintf("%d", fileInfo.CreatedAt.Unix()) - responseHeaders.Headers["Filename"] = fileInfo.Filename + responseHeaders.Headers["FileName"] = fileInfo.Filename switch ext { case extensionJar: responseHeaders.Headers["Content-Type"] = contentTypeJar diff --git a/registry/app/pkg/nuget/local.go b/registry/app/pkg/nuget/local.go index 0576c4331..463d81437 100644 --- a/registry/app/pkg/nuget/local.go +++ b/registry/app/pkg/nuget/local.go @@ -36,7 +36,6 @@ import ( nugettype "github.com/harness/gitness/registry/app/pkg/types/nuget" "github.com/harness/gitness/registry/app/storage" "github.com/harness/gitness/registry/app/store" - "github.com/harness/gitness/registry/types" "github.com/harness/gitness/store/database/dbtx" "github.com/google/uuid" @@ -67,14 +66,17 @@ type localRegistry struct { urlProvider urlprovider.Provider } -func (c *localRegistry) GetServiceEndpoint(ctx context.Context, - info nugettype.ArtifactInfo) *nugettype.ServiceEndpoint { +func (c *localRegistry) GetServiceEndpoint( + ctx context.Context, + info nugettype.ArtifactInfo, +) *nugettype.ServiceEndpoint { baseURL := c.urlProvider.RegistryURL(ctx, "pkg", info.RootIdentifier, info.RegIdentifier, "nuget") serviceEndpoints := buildServiceEndpoint(baseURL) return serviceEndpoints } -func (c *localRegistry) UploadPackage(ctx context.Context, +func (c *localRegistry) UploadPackage( + ctx context.Context, info nugettype.ArtifactInfo, fileReader io.ReadCloser, ) (headers *commons.ResponseHeaders, sha256 string, err error) { @@ -96,8 +98,10 @@ func (c *localRegistry) UploadPackage(ctx context.Context, }) } -func (c *localRegistry) buildMetadata(info nugettype.ArtifactInfo, - fileReader io.Reader) (metadata nugetmetadata.Metadata, err error) { +func (c *localRegistry) buildMetadata( + info nugettype.ArtifactInfo, + fileReader io.Reader, +) (metadata nugetmetadata.Metadata, err error) { pathUUID := uuid.NewString() tmpFile, err2 := os.CreateTemp(os.TempDir(), info.RootIdentifier+"-"+pathUUID+"*") if err2 != nil { @@ -154,8 +158,10 @@ func (c *localRegistry) parseMetadata(f io.Reader) (metadata nugetmetadata.Metad return p, nil } -func (c *localRegistry) DownloadPackage(ctx context.Context, - info nugettype.ArtifactInfo) (*commons.ResponseHeaders, *storage.FileReader, string, error) { +func (c *localRegistry) DownloadPackage( + ctx context.Context, + info nugettype.ArtifactInfo, +) (*commons.ResponseHeaders, *storage.FileReader, string, error) { responseHeaders := &commons.ResponseHeaders{ Headers: make(map[string]string), Code: 0, @@ -163,10 +169,8 @@ func (c *localRegistry) DownloadPackage(ctx context.Context, path := "/" + info.Image + "/" + info.Version + "/" + info.Filename - fileReader, _, redirectURL, err := c.fileManager.DownloadFile(ctx, path, types.Registry{ - ID: info.RegistryID, - Name: info.RegIdentifier, - }, info.RootIdentifier) + fileReader, _, redirectURL, err := c.fileManager.DownloadFile(ctx, path, info.RegistryID, + info.RegIdentifier, info.RootIdentifier) if err != nil { return responseHeaders, nil, "", err } diff --git a/registry/app/pkg/rpm/helper.go b/registry/app/pkg/rpm/helper.go new file mode 100644 index 000000000..ad362b21a --- /dev/null +++ b/registry/app/pkg/rpm/helper.go @@ -0,0 +1,509 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +//nolint:gosec +import ( + "bytes" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding" + "errors" + "fmt" + "hash" + "io" + "math" + "os" + "strings" + + rpmmetadata "github.com/harness/gitness/registry/app/metadata/rpm" + "github.com/harness/gitness/registry/validation" + + "github.com/sassoftware/go-rpmutils" +) + +const ( + sIFMT = 0xf000 + sIFDIR = 0x4000 + sIXUSR = 0x40 + sIXGRP = 0x8 + sIXOTH = 0x1 + + sizeMD5 = 92 + sizeSHA1 = 96 + sizeSHA256 = 108 + sizeSHA512 = 204 + size = sizeMD5 + sizeSHA1 + sizeSHA256 + sizeSHA512 + + RepoMdFile = "repomd.xml" + RepoDataPrefix = "repodata/" + + DefaultMemorySize = 32 * 1024 * 1024 +) + +var ( + ErrInvalidMemorySize = errors.New("memory size must be greater 0 and lower math.MaxInt32") + ErrWriteAfterRead = errors.New("write is unsupported after a read operation") +) + +func parsePackage(r io.Reader) (*rpmPackage, error) { + rpm, err := rpmutils.ReadRpm(r) + if err != nil { + return nil, err + } + + nevra, err := rpm.Header.GetNEVRA() + if err != nil { + return nil, err + } + + version := fmt.Sprintf("%s-%s", nevra.Version, nevra.Release) + if nevra.Epoch != "" && nevra.Epoch != "0" { + version = fmt.Sprintf("%s-%s", nevra.Epoch, version) + } + + p := &rpmPackage{ + Name: nevra.Name, + Version: version, + VersionMetadata: &rpmmetadata.VersionMetadata{ + Summary: getString(rpm.Header, rpmutils.SUMMARY), + Description: getString(rpm.Header, rpmutils.DESCRIPTION), + License: getString(rpm.Header, rpmutils.LICENSE), + ProjectURL: getString(rpm.Header, rpmutils.URL), + }, + FileMetadata: &rpmmetadata.FileMetadata{ + Architecture: nevra.Arch, + Epoch: nevra.Epoch, + Version: nevra.Version, + Release: nevra.Release, + Vendor: getString(rpm.Header, rpmutils.VENDOR), + Group: getString(rpm.Header, rpmutils.GROUP), + Packager: getString(rpm.Header, rpmutils.PACKAGER), + SourceRpm: getString(rpm.Header, rpmutils.SOURCERPM), + BuildHost: getString(rpm.Header, rpmutils.BUILDHOST), + BuildTime: getUInt64(rpm.Header, rpmutils.BUILDTIME), + FileTime: getUInt64(rpm.Header, rpmutils.FILEMTIMES), + InstalledSize: getUInt64(rpm.Header, rpmutils.SIZE), + ArchiveSize: getUInt64(rpm.Header, rpmutils.SIG_PAYLOADSIZE), + + Provides: getEntries(rpm.Header, rpmutils.PROVIDENAME, rpmutils.PROVIDEVERSION, rpmutils.PROVIDEFLAGS), + Requires: getEntries(rpm.Header, rpmutils.REQUIRENAME, rpmutils.REQUIREVERSION, rpmutils.REQUIREFLAGS), + Conflicts: getEntries(rpm.Header, rpmutils.CONFLICTNAME, rpmutils.CONFLICTVERSION, rpmutils.CONFLICTFLAGS), + Obsoletes: getEntries(rpm.Header, rpmutils.OBSOLETENAME, rpmutils.OBSOLETEVERSION, rpmutils.OBSOLETEFLAGS), + Files: getFiles(rpm.Header), + Changelogs: getChangelogs(rpm.Header), + }, + } + + if !validation.IsValidURL(p.VersionMetadata.ProjectURL) { + p.VersionMetadata.ProjectURL = "" + } + + return p, nil +} + +func getString(h *rpmutils.RpmHeader, tag int) string { + values, err := h.GetStrings(tag) + if err != nil || len(values) < 1 { + return "" + } + return values[0] +} + +func getUInt64(h *rpmutils.RpmHeader, tag int) uint64 { + values, err := h.GetUint64s(tag) + if err != nil || len(values) < 1 { + return 0 + } + return values[0] +} + +// nolint: gocritic +func getEntries(h *rpmutils.RpmHeader, namesTag, versionsTag, flagsTag int) []*rpmmetadata.Entry { + names, err := h.GetStrings(namesTag) + if err != nil || len(names) == 0 { + return nil + } + flags, err := h.GetUint64s(flagsTag) + if err != nil || len(flags) == 0 { + return nil + } + versions, err := h.GetStrings(versionsTag) + if err != nil || len(versions) == 0 { + return nil + } + if len(names) != len(flags) || len(names) != len(versions) { + return nil + } + + entries := make([]*rpmmetadata.Entry, 0, len(names)) + for i := range names { + e := &rpmmetadata.Entry{ + Name: names[i], + } + + flags := flags[i] + if (flags&rpmutils.RPMSENSE_GREATER) != 0 && (flags&rpmutils.RPMSENSE_EQUAL) != 0 { + e.Flags = "GE" + } else if (flags&rpmutils.RPMSENSE_LESS) != 0 && (flags&rpmutils.RPMSENSE_EQUAL) != 0 { + e.Flags = "LE" + } else if (flags & rpmutils.RPMSENSE_GREATER) != 0 { + e.Flags = "GT" + } else if (flags & rpmutils.RPMSENSE_LESS) != 0 { + e.Flags = "LT" + } else if (flags & rpmutils.RPMSENSE_EQUAL) != 0 { + e.Flags = "EQ" + } + + version := versions[i] + if version != "" { + parts := strings.Split(version, "-") + + versionParts := strings.Split(parts[0], ":") + if len(versionParts) == 2 { + e.Version = versionParts[1] + e.Epoch = versionParts[0] + } else { + e.Version = versionParts[0] + e.Epoch = "0" + } + + if len(parts) > 1 { + e.Release = parts[1] + } + } + + entries = append(entries, e) + } + return entries +} + +func getFiles(h *rpmutils.RpmHeader) []*rpmmetadata.File { + baseNames, _ := h.GetStrings(rpmutils.BASENAMES) + dirNames, _ := h.GetStrings(rpmutils.DIRNAMES) + dirIndexes, _ := h.GetUint32s(rpmutils.DIRINDEXES) + fileFlags, _ := h.GetUint32s(rpmutils.FILEFLAGS) + fileModes, _ := h.GetUint32s(rpmutils.FILEMODES) + + files := make([]*rpmmetadata.File, 0, len(baseNames)) + for i := range baseNames { + if len(dirIndexes) <= i { + continue + } + dirIndex := dirIndexes[i] + if len(dirNames) <= int(dirIndex) { + continue + } + + var fileType string + var isExecutable bool + if i < len(fileFlags) && (fileFlags[i]&rpmutils.RPMFILE_GHOST) != 0 { + fileType = "ghost" + } else if i < len(fileModes) { + if (fileModes[i] & sIFMT) == sIFDIR { + fileType = "dir" + } else { + mode := fileModes[i] & ^uint32(sIFMT) + isExecutable = (mode&sIXUSR) != 0 || (mode&sIXGRP) != 0 || (mode&sIXOTH) != 0 + } + } + + files = append(files, &rpmmetadata.File{ + Path: dirNames[dirIndex] + baseNames[i], + Type: fileType, + IsExecutable: isExecutable, + }) + } + + return files +} + +func getChangelogs(h *rpmutils.RpmHeader) []*rpmmetadata.Changelog { + texts, err := h.GetStrings(rpmutils.CHANGELOGTEXT) + if err != nil || len(texts) == 0 { + return nil + } + authors, err := h.GetStrings(rpmutils.CHANGELOGNAME) + if err != nil || len(authors) == 0 { + return nil + } + times, err := h.GetUint32s(rpmutils.CHANGELOGTIME) + if err != nil || len(times) == 0 { + return nil + } + if len(texts) != len(authors) || len(texts) != len(times) { + return nil + } + + changelogs := make([]*rpmmetadata.Changelog, 0, len(texts)) + for i := range texts { + changelogs = append(changelogs, &rpmmetadata.Changelog{ + Author: authors[i], + Date: int64(times[i]), + Text: texts[i], + }) + } + return changelogs +} + +type writtenCounter struct { + written int64 +} + +func (wc *writtenCounter) Write(buf []byte) (int, error) { + n := len(buf) + + wc.written += int64(n) + + return n, nil +} + +func (wc *writtenCounter) Written() int64 { + return wc.written +} + +type readAtSeeker interface { + io.ReadSeeker + io.ReaderAt +} + +type FileBackedBuffer struct { + maxMemorySize int64 + size int64 + buffer bytes.Buffer + file *os.File + reader readAtSeeker +} + +func NewFileBackedBuffer(maxMemorySize int) (*FileBackedBuffer, error) { + if maxMemorySize < 0 || maxMemorySize > math.MaxInt32 { + return nil, ErrInvalidMemorySize + } + + return &FileBackedBuffer{ + maxMemorySize: int64(maxMemorySize), + }, nil +} + +//nolint:nestif +func (b *FileBackedBuffer) Write(p []byte) (int, error) { + if b.reader != nil { + return 0, ErrWriteAfterRead + } + + var n int + var err error + + if b.file != nil { + n, err = b.file.Write(p) + } else { + if b.size+int64(len(p)) > b.maxMemorySize { + b.file, err = os.CreateTemp("", "gitness-buffer-") + if err != nil { + return 0, err + } + + _, err = io.Copy(b.file, &b.buffer) + if err != nil { + return 0, err + } + + return b.Write(p) + } + + n, err = b.buffer.Write(p) + } + + if err != nil { + return n, err + } + b.size += int64(n) + return n, nil +} + +func (b *FileBackedBuffer) Size() int64 { + return b.size +} + +func (b *FileBackedBuffer) switchToReader() error { + if b.reader != nil { + return nil + } + + if b.file != nil { + if _, err := b.file.Seek(0, io.SeekStart); err != nil { + return err + } + b.reader = b.file + } else { + b.reader = bytes.NewReader(b.buffer.Bytes()) + } + return nil +} + +func (b *FileBackedBuffer) Read(p []byte) (int, error) { + if err := b.switchToReader(); err != nil { + return 0, err + } + + return b.reader.Read(p) +} + +func (b *FileBackedBuffer) ReadAt(p []byte, off int64) (int, error) { + if err := b.switchToReader(); err != nil { + return 0, err + } + + return b.reader.ReadAt(p, off) +} + +func (b *FileBackedBuffer) Seek(offset int64, whence int) (int64, error) { + if err := b.switchToReader(); err != nil { + return 0, err + } + + return b.reader.Seek(offset, whence) +} + +func (b *FileBackedBuffer) Close() error { + if b.file != nil { + err := b.file.Close() + os.Remove(b.file.Name()) + b.file = nil + return err + } + return nil +} + +type HashedBuffer struct { + *FileBackedBuffer + hash *MultiHasher + combinedWriter io.Writer +} + +func NewHashedBuffer() (*HashedBuffer, error) { + return NewHashedBufferWithSize(DefaultMemorySize) +} + +func NewHashedBufferWithSize(maxMemorySize int) (*HashedBuffer, error) { + b, err := NewFileBackedBuffer(maxMemorySize) + if err != nil { + return nil, err + } + + hash := NewMultiHasher() + + combinedWriter := io.MultiWriter(b, hash) + + return &HashedBuffer{ + b, + hash, + combinedWriter, + }, nil +} + +func CreateHashedBufferFromReader(r io.Reader) (*HashedBuffer, error) { + return CreateHashedBufferFromReaderWithSize(r, DefaultMemorySize) +} + +func CreateHashedBufferFromReaderWithSize(r io.Reader, maxMemorySize int) (*HashedBuffer, error) { + b, err := NewHashedBufferWithSize(maxMemorySize) + if err != nil { + return nil, err + } + + _, err = io.Copy(b, r) + if err != nil { + return nil, err + } + + return b, nil +} + +func (b *HashedBuffer) Write(p []byte) (int, error) { + return b.combinedWriter.Write(p) +} + +func (b *HashedBuffer) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) { + return b.hash.Sums() +} + +type MultiHasher struct { + md5 hash.Hash + sha1 hash.Hash + sha256 hash.Hash + sha512 hash.Hash + + combinedWriter io.Writer +} + +//nolint:gosec +func NewMultiHasher() *MultiHasher { + md5 := md5.New() + sha1 := sha1.New() + sha256 := sha256.New() + sha512 := sha512.New() + + combinedWriter := io.MultiWriter(md5, sha1, sha256, sha512) + + return &MultiHasher{ + md5, + sha1, + sha256, + sha512, + combinedWriter, + } +} + +// nolint:errcheck +func (h *MultiHasher) MarshalBinary() ([]byte, error) { + md5Bytes, err := h.md5.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha1Bytes, err := h.sha1.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha256Bytes, err := h.sha256.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + sha512Bytes, err := h.sha512.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + return nil, err + } + + b := make([]byte, 0, size) + b = append(b, md5Bytes...) + b = append(b, sha1Bytes...) + b = append(b, sha256Bytes...) + b = append(b, sha512Bytes...) + return b, nil +} + +func (h *MultiHasher) Write(p []byte) (int, error) { + return h.combinedWriter.Write(p) +} + +func (h *MultiHasher) Sums() (hashMD5, hashSHA1, hashSHA256, hashSHA512 []byte) { + hashMD5 = h.md5.Sum(nil) + hashSHA1 = h.sha1.Sum(nil) + hashSHA256 = h.sha256.Sum(nil) + hashSHA512 = h.sha512.Sum(nil) + return hashMD5, hashSHA1, hashSHA256, hashSHA512 +} diff --git a/registry/app/pkg/rpm/local.go b/registry/app/pkg/rpm/local.go new file mode 100644 index 000000000..fbc3e248b --- /dev/null +++ b/registry/app/pkg/rpm/local.go @@ -0,0 +1,179 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "context" + "fmt" + "io" + "mime/multipart" + "net/http" + + urlprovider "github.com/harness/gitness/app/url" + "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" + rpmmetadata "github.com/harness/gitness/registry/app/metadata/rpm" + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/filemanager" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" + "github.com/harness/gitness/registry/app/storage" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/store/database/dbtx" + + "github.com/rs/zerolog/log" +) + +var _ pkg.Artifact = (*localRegistry)(nil) +var _ Registry = (*localRegistry)(nil) + +type localRegistry struct { + localBase base.LocalBase + fileManager filemanager.FileManager + proxyStore store.UpstreamProxyConfigRepository + tx dbtx.Transactor + registryDao store.RegistryRepository + imageDao store.ImageRepository + artifactDao store.ArtifactRepository + urlProvider urlprovider.Provider + localRegistryHelper LocalRegistryHelper +} + +type LocalRegistry interface { + Registry +} + +func NewLocalRegistry( + localBase base.LocalBase, + fileManager filemanager.FileManager, + proxyStore store.UpstreamProxyConfigRepository, + tx dbtx.Transactor, + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + urlProvider urlprovider.Provider, + localRegistryHelper LocalRegistryHelper, +) LocalRegistry { + return &localRegistry{ + localBase: localBase, + fileManager: fileManager, + proxyStore: proxyStore, + tx: tx, + registryDao: registryDao, + imageDao: imageDao, + artifactDao: artifactDao, + urlProvider: urlProvider, + localRegistryHelper: localRegistryHelper, + } +} + +func (c *localRegistry) GetArtifactType() artifact.RegistryType { + return artifact.RegistryTypeVIRTUAL +} + +func (c *localRegistry) GetPackageTypes() []artifact.PackageType { + return []artifact.PackageType{artifact.PackageTypeRPM} +} + +func (c *localRegistry) UploadPackageFile( + ctx context.Context, + info rpmtype.ArtifactInfo, + file multipart.File, +) (headers *commons.ResponseHeaders, sha256 string, err error) { + buf, err := CreateHashedBufferFromReader(file) + if err != nil { + return nil, "", err + } + defer buf.Close() + + pkg, err := parsePackage(buf) + if err != nil { + log.Printf("failded to parse rpm package: %v", err) + return nil, "", err + } + + if _, err := buf.Seek(0, io.SeekStart); err != nil { + return nil, "", err + } + + info.Image = pkg.Name + info.Version = pkg.Version + "." + pkg.FileMetadata.Architecture + info.Metadata = rpmmetadata.Metadata{ + VersionMetadata: *pkg.VersionMetadata, + FileMetadata: *pkg.FileMetadata, + } + + fileName := fmt.Sprintf("%s-%s.%s.rpm", pkg.Name, pkg.Version, pkg.FileMetadata.Architecture) + if info.FileName == "" { + info.FileName = fileName + } + + path := fmt.Sprintf("%s/%s/%s/%s", pkg.Name, pkg.Version, pkg.FileMetadata.Architecture, fileName) + rs, sha256, err := c.localBase.Upload(ctx, info.ArtifactInfo, fileName, info.Version, path, buf, + &rpmmetadata.RpmMetadata{ + Metadata: info.Metadata, + }) + + if err != nil { + return nil, "", err + } + + //TODO: make it async / atomic operation, implement artifact status (sync successful, sync failed..... statuses) + err = c.localRegistryHelper.BuildRegistryFiles(ctx, info) + if err != nil { + return nil, "", err + } + return rs, sha256, err +} + +func (c *localRegistry) GetRepoData( + ctx context.Context, + info rpmtype.ArtifactInfo, + fileName string, +) (*commons.ResponseHeaders, + *storage.FileReader, + io.ReadCloser, + string, + error, +) { + responseHeaders := &commons.ResponseHeaders{ + Headers: make(map[string]string), + Code: 0, + } + + fileReader, _, redirectURL, err := c.fileManager.DownloadFile( + ctx, "/"+RepoDataPrefix+fileName, info.RegistryID, info.RegIdentifier, info.RootIdentifier, + ) + if err != nil { + return responseHeaders, nil, nil, "", err + } + responseHeaders.Code = http.StatusOK + return responseHeaders, fileReader, nil, redirectURL, nil +} + +func (c *localRegistry) DownloadPackageFile( + ctx context.Context, + info rpmtype.ArtifactInfo, +) (*commons.ResponseHeaders, *storage.FileReader, io.ReadCloser, string, error) { + headers, fileReader, redirectURL, err := c.localBase.Download( + ctx, info.ArtifactInfo, + fmt.Sprintf("%s/%s", info.Version, info.Arch), + info.FileName, + ) + if err != nil { + return nil, nil, nil, "", err + } + return headers, fileReader, nil, redirectURL, nil +} diff --git a/registry/app/pkg/rpm/local_helper.go b/registry/app/pkg/rpm/local_helper.go new file mode 100644 index 000000000..367086629 --- /dev/null +++ b/registry/app/pkg/rpm/local_helper.go @@ -0,0 +1,316 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "net/url" + "time" + + rpmmetadata "github.com/harness/gitness/registry/app/metadata/rpm" + "github.com/harness/gitness/registry/app/pkg/filemanager" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" + "github.com/harness/gitness/registry/app/store" +) + +const artifactBatchLimit = 50 + +type LocalRegistryHelper interface { + BuildRegistryFiles(ctx context.Context, info rpmtype.ArtifactInfo) error +} + +type localRegistryHelper struct { + fileManager filemanager.FileManager + artifactDao store.ArtifactRepository +} + +func NewLocalRegistryHelper( + fileManager filemanager.FileManager, + artifactDao store.ArtifactRepository, +) LocalRegistryHelper { + return &localRegistryHelper{ + fileManager: fileManager, + artifactDao: artifactDao, + } +} + +func (l *localRegistryHelper) BuildRegistryFiles(ctx context.Context, info rpmtype.ArtifactInfo) error { + lastArtifactID := int64(0) + var packageInfos []*packageInfo + + for { + artifacts, err := l.artifactDao.GetAllArtifactsByRepo(ctx, info.RegistryID, artifactBatchLimit, lastArtifactID) + if err != nil { + return err + } + + for _, a := range *artifacts { + metadata := rpmmetadata.RpmMetadata{} + err := json.Unmarshal(a.Metadata, &metadata) + if err != nil { + return err + } + + packageInfos = append(packageInfos, &packageInfo{ + Name: a.Name, + Sha256: metadata.GetFiles()[0].Sha256, + Size: metadata.GetFiles()[0].Size, + VersionMetadata: &metadata.VersionMetadata, + FileMetadata: &metadata.FileMetadata, + }) + if a.ID > lastArtifactID { + lastArtifactID = a.ID + } + } + if len(*artifacts) < artifactBatchLimit { + break + } + } + + primary, err := l.buildPrimary(ctx, packageInfos, info) + if err != nil { + return err + } + fileLists, err := l.buildFilelists(ctx, packageInfos, info) + if err != nil { + return err + } + other, err := l.buildOther(ctx, packageInfos, info) + if err != nil { + return err + } + return l.buildRepomd(ctx, []*repoData{ + primary, + fileLists, + other, + }, info) +} + +func (l *localRegistryHelper) buildPrimary( + ctx context.Context, + pds []*packageInfo, + info rpmtype.ArtifactInfo, +) (*repoData, error) { + packages := make([]*primaryPackage, 0, len(pds)) + for _, pd := range pds { + files := make([]*rpmmetadata.File, 0, 3) + for _, f := range pd.FileMetadata.Files { + if f.IsExecutable { + files = append(files, f) + } + } + packageVersion := fmt.Sprintf("%s-%s", pd.FileMetadata.Version, pd.FileMetadata.Release) + packages = append(packages, &primaryPackage{ + Type: "rpm", + Name: pd.Name, + Architecture: pd.FileMetadata.Architecture, + Version: primaryVersion{ + Epoch: pd.FileMetadata.Epoch, + Version: pd.FileMetadata.Version, + Release: pd.FileMetadata.Release, + }, + Checksum: primaryChecksum{ + Type: "sha256", + Checksum: pd.Sha256, + Pkgid: "YES", + }, + Summary: pd.VersionMetadata.Summary, + Description: pd.VersionMetadata.Description, + Packager: pd.FileMetadata.Packager, + URL: pd.VersionMetadata.ProjectURL, + Time: primaryTimes{ + File: pd.FileMetadata.FileTime, + Build: pd.FileMetadata.BuildTime, + }, + Size: primarySizes{ + Package: pd.Size, + Installed: pd.FileMetadata.InstalledSize, + Archive: pd.FileMetadata.ArchiveSize, + }, + Location: PrimaryLocation{ + Href: fmt.Sprintf("package/%s/%s/%s/%s", + url.PathEscape(pd.Name), + url.PathEscape(packageVersion), + url.PathEscape(pd.FileMetadata.Architecture), + url.PathEscape(fmt.Sprintf("%s-%s.%s.rpm", pd.Name, packageVersion, pd.FileMetadata.Architecture))), + }, + Format: primaryFormat{ + License: pd.VersionMetadata.License, + Vendor: pd.FileMetadata.Vendor, + Group: pd.FileMetadata.Group, + Buildhost: pd.FileMetadata.BuildHost, + Sourcerpm: pd.FileMetadata.SourceRpm, + Provides: primaryEntryList{ + Entries: pd.FileMetadata.Provides, + }, + Requires: primaryEntryList{ + Entries: pd.FileMetadata.Requires, + }, + Conflicts: primaryEntryList{ + Entries: pd.FileMetadata.Conflicts, + }, + Obsoletes: primaryEntryList{ + Entries: pd.FileMetadata.Obsoletes, + }, + Files: files, + }, + }) + } + + return l.addDataAsFileToRepo(ctx, "primary", &primaryMetadata{ + Xmlns: "http://linux.duke.edu/metadata/common", + XmlnsRpm: "http://linux.duke.edu/metadata/rpm", + PackageCount: len(pds), + Packages: packages, + }, info) +} + +func (l *localRegistryHelper) buildOther( + ctx context.Context, + pds []*packageInfo, + info rpmtype.ArtifactInfo, +) (*repoData, error) { + packages := make([]*otherPackage, 0, len(pds)) + for _, pd := range pds { + packages = append(packages, &otherPackage{ + Pkgid: pd.Sha256, + Name: pd.Name, + Architecture: pd.FileMetadata.Architecture, + Version: otherVersion{ + Epoch: pd.FileMetadata.Epoch, + Version: pd.FileMetadata.Version, + Release: pd.FileMetadata.Release, + }, + Changelogs: pd.FileMetadata.Changelogs, + }) + } + + return l.addDataAsFileToRepo(ctx, "other", &otherdata{ + Xmlns: "http://linux.duke.edu/metadata/other", + PackageCount: len(pds), + Packages: packages, + }, info) +} + +func (l *localRegistryHelper) buildFilelists( + ctx context.Context, + pds []*packageInfo, + info rpmtype.ArtifactInfo, +) (*repoData, error) { //nolint:dupl + packages := make([]*fileListPackage, 0, len(pds)) + for _, pd := range pds { + packages = append(packages, &fileListPackage{ + Pkgid: pd.Sha256, + Name: pd.Name, + Architecture: pd.FileMetadata.Architecture, + Version: fileListVersion{ + Epoch: pd.FileMetadata.Epoch, + Version: pd.FileMetadata.Version, + Release: pd.FileMetadata.Release, + }, + Files: pd.FileMetadata.Files, + }) + } + + return l.addDataAsFileToRepo(ctx, "filelists", &filelists{ + Xmlns: "http://linux.duke.edu/metadata/other", + PackageCount: len(pds), + Packages: packages, + }, info) +} + +func (l *localRegistryHelper) buildRepomd( + ctx context.Context, + data []*repoData, + info rpmtype.ArtifactInfo, +) error { + var buf bytes.Buffer + buf.WriteString(xml.Header) + if err := xml.NewEncoder(&buf).Encode(&repomd{ + Xmlns: "http://linux.duke.edu/metadata/repo", + XmlnsRpm: "http://linux.duke.edu/metadata/rpm", + Data: data, + }); err != nil { + return err + } + repomdContent, _ := CreateHashedBufferFromReader(&buf) + defer repomdContent.Close() + + _, err := l.fileManager.UploadFile(ctx, RepoDataPrefix+RepoMdFile, info.RegIdentifier, info.RegistryID, + info.RootParentID, info.RootIdentifier, repomdContent, repomdContent, RepoMdFile) + if err != nil { + return err + } + return nil +} + +func (l *localRegistryHelper) addDataAsFileToRepo( + ctx context.Context, + filetype string, + obj any, + info rpmtype.ArtifactInfo, +) (*repoData, error) { + content, _ := NewHashedBuffer() + defer content.Close() + + gzw := gzip.NewWriter(content) + wc := &writtenCounter{} + h := sha256.New() + + w := io.MultiWriter(gzw, wc, h) + _, _ = w.Write([]byte(xml.Header)) + + if err := xml.NewEncoder(w).Encode(obj); err != nil { + return nil, err + } + + if err := gzw.Close(); err != nil { + return nil, err + } + + filename := filetype + ".xml.gz" + _, err := l.fileManager.UploadFile(ctx, RepoDataPrefix+filename, info.RegIdentifier, info.RegistryID, + info.RootParentID, info.RootIdentifier, content, content, filename) + if err != nil { + return nil, err + } + _, _, hashSHA256, _ := content.Sums() + + return &repoData{ + Type: filetype, + Checksum: repoChecksum{ + Type: "sha256", + Value: hex.EncodeToString(hashSHA256), + }, + OpenChecksum: repoChecksum{ + Type: "sha256", + Value: hex.EncodeToString(h.Sum(nil)), + }, + Location: repoLocation{ + Href: "repodata/" + filename, + }, + Timestamp: time.Now().Unix(), + Size: content.Size(), + OpenSize: wc.Written(), + }, nil +} diff --git a/registry/app/pkg/rpm/proxy.go b/registry/app/pkg/rpm/proxy.go new file mode 100644 index 000000000..fe1f5e1e8 --- /dev/null +++ b/registry/app/pkg/rpm/proxy.go @@ -0,0 +1,113 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "context" + "fmt" + "io" + "mime/multipart" + + urlprovider "github.com/harness/gitness/app/url" + "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" + "github.com/harness/gitness/registry/app/dist_temp/errcode" + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/filemanager" + rpmtype "github.com/harness/gitness/registry/app/pkg/types/rpm" + "github.com/harness/gitness/registry/app/storage" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/store/database/dbtx" + + "github.com/rs/zerolog/log" +) + +var _ pkg.Artifact = (*proxy)(nil) +var _ Registry = (*proxy)(nil) + +type proxy struct { + fileManager filemanager.FileManager + proxyStore store.UpstreamProxyConfigRepository + tx dbtx.Transactor + registryDao store.RegistryRepository + imageDao store.ImageRepository + artifactDao store.ArtifactRepository + urlProvider urlprovider.Provider +} + +func (r *proxy) DownloadPackageFile( + _ context.Context, + _ rpmtype.ArtifactInfo, +) (*commons.ResponseHeaders, *storage.FileReader, io.ReadCloser, string, error) { + // TODO implement me + panic("implement me") +} + +type Proxy interface { + Registry +} + +func NewProxy( + fileManager filemanager.FileManager, + proxyStore store.UpstreamProxyConfigRepository, + tx dbtx.Transactor, + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + urlProvider urlprovider.Provider, +) Proxy { + return &proxy{ + proxyStore: proxyStore, + registryDao: registryDao, + imageDao: imageDao, + artifactDao: artifactDao, + fileManager: fileManager, + tx: tx, + urlProvider: urlProvider, + } +} + +func (r *proxy) GetArtifactType() artifact.RegistryType { + return artifact.RegistryTypeUPSTREAM +} + +func (r *proxy) GetPackageTypes() []artifact.PackageType { + return []artifact.PackageType{artifact.PackageTypeRPM} +} + +// GetPackageMetadata returns the metadata of a RPM package. +func (r *proxy) GetRepoData( + _ context.Context, + _ rpmtype.ArtifactInfo, + _ string, +) (*commons.ResponseHeaders, + *storage.FileReader, + io.ReadCloser, + string, + error, +) { + return nil, nil, nil, "", nil +} + +// UploadPackageFile FIXME: Extract this upload function for all types of packageTypes +// uploads the package file to the storage. +func (r *proxy) UploadPackageFile( + ctx context.Context, + _ rpmtype.ArtifactInfo, + _ multipart.File, +) (*commons.ResponseHeaders, string, error) { + log.Error().Ctx(ctx).Msg("Not implemented") + return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("not implemented")) +} diff --git a/registry/app/pkg/rpm/registry.go b/registry/app/pkg/rpm/registry.go new file mode 100644 index 000000000..23ac88f14 --- /dev/null +++ b/registry/app/pkg/rpm/registry.go @@ -0,0 +1,49 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "context" + "io" + "mime/multipart" + + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/commons" + rpm "github.com/harness/gitness/registry/app/pkg/types/rpm" + "github.com/harness/gitness/registry/app/storage" +) + +type Registry interface { + pkg.Artifact + + UploadPackageFile( + ctx context.Context, + info rpm.ArtifactInfo, + file multipart.File, + ) (*commons.ResponseHeaders, string, error) + + DownloadPackageFile(ctx context.Context, info rpm.ArtifactInfo) ( + *commons.ResponseHeaders, + *storage.FileReader, + io.ReadCloser, + string, + error, + ) + GetRepoData(ctx context.Context, info rpm.ArtifactInfo, fileName string) (*commons.ResponseHeaders, + *storage.FileReader, + io.ReadCloser, + string, + error) +} diff --git a/registry/app/pkg/rpm/remote_helper.go b/registry/app/pkg/rpm/remote_helper.go new file mode 100644 index 000000000..dbf3a7bef --- /dev/null +++ b/registry/app/pkg/rpm/remote_helper.go @@ -0,0 +1,25 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +type RemoteRegistryHelper interface { +} + +type remoteRegistryHelper struct { +} + +func NewRemoteRegistryHelper() RemoteRegistryHelper { + return &remoteRegistryHelper{} +} diff --git a/registry/app/pkg/rpm/types.go b/registry/app/pkg/rpm/types.go new file mode 100644 index 000000000..d2888f3c6 --- /dev/null +++ b/registry/app/pkg/rpm/types.go @@ -0,0 +1,173 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "encoding/xml" + + rpmmetadata "github.com/harness/gitness/registry/app/metadata/rpm" +) + +type primaryVersion struct { + Epoch string `xml:"epoch,attr"` + Version string `xml:"ver,attr"` + Release string `xml:"rel,attr"` +} + +type primaryChecksum struct { + Checksum string `xml:",chardata"` //nolint: tagliatelle + Type string `xml:"type,attr"` + Pkgid string `xml:"pkgid,attr"` +} + +type primaryTimes struct { + File uint64 `xml:"file,attr"` + Build uint64 `xml:"build,attr"` +} + +type primarySizes struct { + Package int64 `xml:"package,attr"` + Installed uint64 `xml:"installed,attr"` + Archive uint64 `xml:"archive,attr"` +} + +type PrimaryLocation struct { + Href string `xml:"href,attr"` +} + +type primaryEntryList struct { + Entries []*rpmmetadata.Entry `xml:"rpm:entry"` +} + +type primaryFormat struct { + License string `xml:"rpm:license"` + Vendor string `xml:"rpm:vendor"` + Group string `xml:"rpm:group"` + Buildhost string `xml:"rpm:buildhost"` + Sourcerpm string `xml:"rpm:sourcerpm"` + Provides primaryEntryList `xml:"rpm:provides"` + Requires primaryEntryList `xml:"rpm:requires"` + Conflicts primaryEntryList `xml:"rpm:conflicts"` + Obsoletes primaryEntryList `xml:"rpm:obsoletes"` + Files []*rpmmetadata.File `xml:"file"` +} + +type primaryPackage struct { + XMLName xml.Name `xml:"package"` + Type string `xml:"type,attr"` + Name string `xml:"name"` + Architecture string `xml:"arch"` + Version primaryVersion `xml:"version"` + Checksum primaryChecksum `xml:"checksum"` + Summary string `xml:"summary"` + Description string `xml:"description"` + Packager string `xml:"packager"` + URL string `xml:"url"` + Time primaryTimes `xml:"time"` + Size primarySizes `xml:"size"` + Location PrimaryLocation `xml:"location"` + Format primaryFormat `xml:"format"` +} + +type primaryMetadata struct { + XMLName xml.Name `xml:"metadata"` + Xmlns string `xml:"xmlns,attr"` + XmlnsRpm string `xml:"xmlns:rpm,attr"` + PackageCount int `xml:"packages,attr"` + Packages []*primaryPackage `xml:"package"` +} + +type otherVersion struct { + Epoch string `xml:"epoch,attr"` + Version string `xml:"ver,attr"` + Release string `xml:"rel,attr"` +} + +type otherPackage struct { + Pkgid string `xml:"pkgid,attr"` + Name string `xml:"name,attr"` + Architecture string `xml:"arch,attr"` + Version otherVersion `xml:"version"` + Changelogs []*rpmmetadata.Changelog `xml:"changelog"` +} + +type otherdata struct { + XMLName xml.Name `xml:"otherdata"` + Xmlns string `xml:"xmlns,attr"` + PackageCount int `xml:"packages,attr"` + Packages []*otherPackage `xml:"package"` +} + +type fileListVersion struct { + Epoch string `xml:"epoch,attr"` + Version string `xml:"ver,attr"` + Release string `xml:"rel,attr"` +} + +type fileListPackage struct { + Pkgid string `xml:"pkgid,attr"` + Name string `xml:"name,attr"` + Architecture string `xml:"arch,attr"` + Version fileListVersion `xml:"version"` + Files []*rpmmetadata.File `xml:"file"` +} + +type filelists struct { + XMLName xml.Name `xml:"filelists"` + Xmlns string `xml:"xmlns,attr"` + PackageCount int `xml:"packages,attr"` + Packages []*fileListPackage `xml:"package"` +} + +type repomd struct { + XMLName xml.Name `xml:"repomd"` + Xmlns string `xml:"xmlns,attr"` + XmlnsRpm string `xml:"xmlns:rpm,attr"` + Data []*repoData `xml:"data"` +} + +type repoChecksum struct { + Value string `xml:",chardata"` //nolint: tagliatelle + Type string `xml:"type,attr"` +} + +type repoLocation struct { + Href string `xml:"href,attr"` +} + +type repoData struct { + Type string `xml:"type,attr"` + Checksum repoChecksum `xml:"checksum"` + OpenChecksum repoChecksum `xml:"open-checksum"` //nolint: tagliatelle + Location repoLocation `xml:"location"` + Timestamp int64 `xml:"timestamp"` + Size int64 `xml:"size"` + OpenSize int64 `xml:"open-size"` //nolint: tagliatelle +} + +type packageInfo struct { + Name string + Sha256 string + Size int64 + VersionMetadata *rpmmetadata.VersionMetadata + FileMetadata *rpmmetadata.FileMetadata +} + +type rpmPackage struct { + Name string + Version string + VersionMetadata *rpmmetadata.VersionMetadata + FileMetadata *rpmmetadata.FileMetadata +} diff --git a/registry/app/pkg/rpm/wire.go b/registry/app/pkg/rpm/wire.go new file mode 100644 index 000000000..03ccf87ee --- /dev/null +++ b/registry/app/pkg/rpm/wire.go @@ -0,0 +1,65 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + urlprovider "github.com/harness/gitness/app/url" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/filemanager" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/store/database/dbtx" + + "github.com/google/wire" +) + +func LocalRegistryProvider( + localBase base.LocalBase, + fileManager filemanager.FileManager, + proxyStore store.UpstreamProxyConfigRepository, + tx dbtx.Transactor, + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + urlProvider urlprovider.Provider, + helper LocalRegistryHelper, +) LocalRegistry { + registry := NewLocalRegistry(localBase, fileManager, proxyStore, tx, registryDao, imageDao, artifactDao, + urlProvider, helper) + base.Register(registry) + return registry +} + +func ProxyProvider( + proxyStore store.UpstreamProxyConfigRepository, + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + fileManager filemanager.FileManager, + tx dbtx.Transactor, + urlProvider urlprovider.Provider, +) Proxy { + proxy := NewProxy(fileManager, proxyStore, tx, registryDao, imageDao, artifactDao, urlProvider) + base.Register(proxy) + return proxy +} + +func LocalRegistryHelperProvider( + fileManager filemanager.FileManager, + artifactDao store.ArtifactRepository, +) LocalRegistryHelper { + return NewLocalRegistryHelper(fileManager, artifactDao) +} + +var WireSet = wire.NewSet(LocalRegistryProvider, ProxyProvider, LocalRegistryHelperProvider) diff --git a/registry/app/pkg/types/rpm/types.go b/registry/app/pkg/types/rpm/types.go new file mode 100644 index 000000000..4cd19d2fa --- /dev/null +++ b/registry/app/pkg/types/rpm/types.go @@ -0,0 +1,54 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rpm + +import ( + "github.com/harness/gitness/registry/app/metadata/rpm" + "github.com/harness/gitness/registry/app/pkg" +) + +type ArtifactInfo struct { + pkg.ArtifactInfo + Version string + Arch string + FileName string + Metadata rpm.Metadata +} + +func (a ArtifactInfo) GetVersion() string { + return a.Version +} + +// BaseArtifactInfo implements pkg.PackageArtifactInfo interface. +func (a ArtifactInfo) BaseArtifactInfo() pkg.ArtifactInfo { + return a.ArtifactInfo +} + +func (a ArtifactInfo) GetImageVersion() (exists bool, imageVersion string) { + if a.Image != "" && a.Version != "" { + return true, pkg.JoinWithSeparator(":", a.Image, a.Version) + } + return false, "" +} + +type File struct { + FileURL string + Name string +} + +type PackageMetadata struct { + Name string + Files []File +} diff --git a/registry/app/store/database.go b/registry/app/store/database.go index dfcfe68aa..849bbd285 100644 --- a/registry/app/store/database.go +++ b/registry/app/store/database.go @@ -459,12 +459,12 @@ type ArtifactRepository interface { ctx context.Context, parentID int64, registryIDs *[]string, search string, latestVersion bool, packageTypes []string, ) (int64, error) - GetAllArtifactsByRepo( + GetArtifactsByRepo( ctx context.Context, parentID int64, repoKey string, sortByField string, sortByOrder string, limit int, offset int, search string, labels []string, ) (*[]types.ArtifactMetadata, error) - CountAllArtifactsByRepo( + CountArtifactsByRepo( ctx context.Context, parentID int64, repoKey string, search string, labels []string, ) (int64, error) @@ -494,6 +494,10 @@ type ArtifactRepository interface { DeleteByVersionAndImageName(ctx context.Context, image string, version string, regID int64) (err error) GetLatestByImageID(ctx context.Context, imageID int64) (*types.Artifact, error) + + GetAllArtifactsByRepo( + ctx context.Context, registryID int64, batchSize int, artifactID int64, + ) (*[]types.ArtifactMetadata, error) } type DownloadStatRepository interface { diff --git a/registry/app/store/database/artifact.go b/registry/app/store/database/artifact.go index 64283e13c..b7f2e7733 100644 --- a/registry/app/store/database/artifact.go +++ b/registry/app/store/database/artifact.go @@ -99,7 +99,8 @@ func (a ArtifactDao) GetByRegistryIDAndImage(ctx context.Context, registryID int } artifacts := make([]types.Artifact, len(dst)) - for i, d := range dst { + for i := range dst { + d := dst[i] art, err := a.mapToArtifact(ctx, &d) if err != nil { return nil, errors.Wrap(err, "Failed to map artifact") @@ -203,8 +204,10 @@ func (a ArtifactDao) DeleteByImageNameAndRegistryID(ctx context.Context, regID i return nil } -func (a ArtifactDao) DeleteByVersionAndImageName(ctx context.Context, image string, - version string, regID int64) (err error) { +func (a ArtifactDao) DeleteByVersionAndImageName( + ctx context.Context, image string, + version string, regID int64, +) (err error) { delStmt := databaseg.Builder.Delete("artifacts"). Where("artifact_id IN (SELECT a.artifact_id FROM artifacts a JOIN images i ON i.image_id = a.artifact_image_id"+ " WHERE a.artifact_name = ? AND i.image_name = ? AND i.image_registry_id = ?)", version, image, regID) @@ -399,8 +402,8 @@ func (a ArtifactDao) CountAllArtifactsByParentID( return count, nil } -func (a ArtifactDao) GetAllArtifactsByRepo( - ctx context.Context, parentID int64, repoKey string, +func (a ArtifactDao) GetArtifactsByRepo( + ctx context.Context, registryParentID int64, repoKey string, sortByField string, sortByOrder string, limit int, offset int, search string, labels []string, ) (*[]types.ArtifactMetadata, error) { @@ -417,7 +420,7 @@ func (a ArtifactDao) GetAllArtifactsByRepo( JOIN images i ON i.image_id = a.artifact_image_id JOIN registries r ON i.image_registry_id = r.registry_id WHERE r.registry_parent_id = ? AND r.registry_name = ? ) AS a1 - ON a.artifact_id = a1.id`, parentID, repoKey, // nolint:goconst + ON a.artifact_id = a1.id`, registryParentID, repoKey, // nolint:goconst ). Join("images i ON i.image_id = a.artifact_image_id"). Join("registries r ON i.image_registry_id = r.registry_id"). @@ -430,7 +433,7 @@ func (a ArtifactDao) GetAllArtifactsByRepo( JOIN images i ON i.image_id = t1.artifact_image_id JOIN registries r ON r.registry_id = i.image_registry_id WHERE r.registry_parent_id = ? AND r.registry_name = ? GROUP BY i.image_name) as t2 - ON i.image_name = t2.image_name`, parentID, repoKey, + ON i.image_name = t2.image_name`, registryParentID, repoKey, ). Where("a1.rank = 1 ") @@ -469,7 +472,7 @@ func (a ArtifactDao) GetAllArtifactsByRepo( } // nolint:goconst -func (a ArtifactDao) CountAllArtifactsByRepo( +func (a ArtifactDao) CountArtifactsByRepo( ctx context.Context, parentID int64, repoKey string, search string, labels []string, ) (int64, error) { @@ -747,11 +750,41 @@ func (a ArtifactDao) GetArtifactMetadata( return a.mapToArtifactMetadata(ctx, dst) } +func (a ArtifactDao) GetAllArtifactsByRepo( + ctx context.Context, registryID int64, batchSize int, artifactID int64, +) (*[]types.ArtifactMetadata, error) { + q := databaseg.Builder.Select( + `r.registry_name as repo_name, i.image_name as name, + a.artifact_id as artifact_id, a.artifact_version as version, a.artifact_metadata as metadata`, + ). + From("artifacts a"). + Join("images i ON i.image_id = a.artifact_image_id"). + Join("registries r ON i.image_registry_id = r.registry_id"). + Where("artifact_id > ? AND r.registry_id = ?", artifactID, registryID). + OrderBy("artifact_id ASC"). + Limit(util.SafeIntToUInt64(batchSize)) + + sql, args, err := q.ToSql() + if err != nil { + return nil, errors.Wrap(err, "Failed to convert query to sql") + } + + db := dbtx.GetAccessor(ctx, a.db) + + var dst []*artifactMetadataDB + if err = db.SelectContext(ctx, &dst, sql, args...); err != nil { + return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed executing GetAllArtifactsByRepo query") + } + + return a.mapToArtifactMetadataList(ctx, dst) +} + func (a ArtifactDao) mapToArtifactMetadata( _ context.Context, dst *artifactMetadataDB, ) (*types.ArtifactMetadata, error) { return &types.ArtifactMetadata{ + ID: dst.ID, Name: dst.Name, RepoName: dst.RepoName, DownloadCount: dst.DownloadCount, @@ -761,6 +794,7 @@ func (a ArtifactDao) mapToArtifactMetadata( CreatedAt: time.UnixMilli(dst.CreatedAt), ModifiedAt: time.UnixMilli(dst.ModifiedAt), Version: dst.Version, + Metadata: *dst.Metadata, }, nil } diff --git a/registry/app/store/database/tag.go b/registry/app/store/database/tag.go index 55378e75f..e967a8bdf 100644 --- a/registry/app/store/database/tag.go +++ b/registry/app/store/database/tag.go @@ -17,6 +17,7 @@ package database import ( "context" "database/sql" + "encoding/json" "fmt" "sort" "strings" @@ -73,6 +74,7 @@ type tagDB struct { } type artifactMetadataDB struct { + ID int64 `db:"artifact_id"` Name string `db:"name"` RepoName string `db:"repo_name"` DownloadCount int64 `db:"download_count"` @@ -83,6 +85,7 @@ type artifactMetadataDB struct { ModifiedAt int64 `db:"modified_at"` Tag *string `db:"tag"` Version string `db:"version"` + Metadata *json.RawMessage `db:"metadata"` } type tagMetadataDB struct { diff --git a/registry/types/tag.go b/registry/types/tag.go index 72e5d10b5..e04971819 100644 --- a/registry/types/tag.go +++ b/registry/types/tag.go @@ -15,6 +15,7 @@ package types import ( + "encoding/json" "time" "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" @@ -34,6 +35,7 @@ type Tag struct { } type ArtifactMetadata struct { + ID int64 Name string RepoName string DownloadCount int64 @@ -43,6 +45,7 @@ type ArtifactMetadata struct { CreatedAt time.Time ModifiedAt time.Time Version string + Metadata json.RawMessage } type ImageMetadata struct {