cmd/go: add subdirectory support to go-import meta tag

This CL adds ability to specify a subdirectory in the go-import meta tag.
A go-import meta tag now will support:
<meta name="go-import" content="root-path vcs repo-url subdir">

Fixes: #34055
Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64-longtest,gotip-windows-amd64-longtest
Change-Id: Iedac520f97e0646254cc1bd2f97d5a9a5236829b
Reviewed-on: https://go-review.googlesource.com/c/go/+/625577
Reviewed-by: Michael Matloob <matloob@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Sam Thanawalla <samthanawalla@google.com>

src/cmd/go/alldocs.go

@@ -2952,6 +2952,11 @@
 //
 //	<meta name="go-import" content="import-prefix vcs repo-root">
 //
+// Starting in Go 1.25, an optional subdirectory will be recognized by the
+// go command:
+//
+//	<meta name="go-import" content="import-prefix vcs repo-root subdir">
+//
 // The import-prefix is the import path corresponding to the repository
 // root. It must be a prefix or an exact match of the package being
 // fetched with "go get". If it's not an exact match, another http
@@ -2966,6 +2971,12 @@
 // The repo-root is the root of the version control system
 // containing a scheme and not containing a .vcs qualifier.
 //
+// The subdir specifies the directory within the repo-root where the
+// Go module's root (including its go.mod file) is located. It allows
+// you to organize your repository with the Go module code in a subdirectory
+// rather than directly at the repository's root.
+// If set, all vcs tags must be prefixed with "subdir". i.e. "subdir/v1.2.3"
+//
 // For example,
 //
 //	import "example.org/pkg/foo"
@@ -2980,8 +2991,15 @@
 //	<meta name="go-import" content="example.org git https://code.org/r/p/exproj">
 //
 // the go tool will verify that https://example.org/?go-get=1 contains the
-// same meta tag and then git clone https://code.org/r/p/exproj into
+// same meta tag and then download the code from the Git repository at https://code.org/r/p/exproj
-// GOPATH/src/example.org.
+//
+// If that page contains the meta tag
+//
+//	<meta name="go-import" content="example.org git https://code.org/r/p/exproj foo/subdir">
+//
+// the go tool will verify that https://example.org/?go-get=1 contains the same meta
+// tag and then download the code from the "foo/subdir" subdirectory within the Git repository
+// at https://code.org/r/p/exproj
 //
 // Downloaded packages are stored in the module cache.
 // See https://golang.org/ref/mod#module-cache.

src/cmd/go/internal/help/helpdoc.go

@@ -241,6 +241,11 @@ The meta tag has the form:
 
 	<meta name="go-import" content="import-prefix vcs repo-root">
 
+Starting in Go 1.25, an optional subdirectory will be recognized by the
+go command:
+
+	<meta name="go-import" content="import-prefix vcs repo-root subdir">
+
 The import-prefix is the import path corresponding to the repository
 root. It must be a prefix or an exact match of the package being
 fetched with "go get". If it's not an exact match, another http
@@ -255,6 +260,12 @@ The vcs is one of "bzr", "fossil", "git", "hg", "svn".
 The repo-root is the root of the version control system
 containing a scheme and not containing a .vcs qualifier.
 
+The subdir specifies the directory within the repo-root where the
+Go module's root (including its go.mod file) is located. It allows
+you to organize your repository with the Go module code in a subdirectory
+rather than directly at the repository's root.
+If set, all vcs tags must be prefixed with "subdir". i.e. "subdir/v1.2.3"
+
 For example,
 
 	import "example.org/pkg/foo"
@@ -269,8 +280,15 @@ If that page contains the meta tag
 	<meta name="go-import" content="example.org git https://code.org/r/p/exproj">
 
 the go tool will verify that https://example.org/?go-get=1 contains the
-same meta tag and then git clone https://code.org/r/p/exproj into
+same meta tag and then download the code from the Git repository at https://code.org/r/p/exproj
-GOPATH/src/example.org.
+
+If that page contains the meta tag
+
+	<meta name="go-import" content="example.org git https://code.org/r/p/exproj foo/subdir">
+
+the go tool will verify that https://example.org/?go-get=1 contains the same meta
+tag and then download the code from the "foo/subdir" subdirectory within the Git repository
+at https://code.org/r/p/exproj
 
 Downloaded packages are stored in the module cache.
 See https://golang.org/ref/mod#module-cache.

src/cmd/go/internal/modfetch/coderepo.go

@@ -59,9 +59,11 @@ type codeRepo struct {
 }
 
 // newCodeRepo returns a Repo that reads the source code for the module with the
-// given path, from the repo stored in code, with the root of the repo
+// given path, from the repo stored in code.
-// containing the path given by codeRoot.
+// codeRoot gives the import path corresponding to the root of the repository,
-func newCodeRepo(code codehost.Repo, codeRoot, path string) (Repo, error) {
+// and subdir gives the subdirectory within the repo containing the module.
+// If subdir is empty, the module is at the root of the repo.
+func newCodeRepo(code codehost.Repo, codeRoot, subdir, path string) (Repo, error) {
 	if !hasPathPrefix(path, codeRoot) {
 		return nil, fmt.Errorf("mismatched repo: found %s for %s", codeRoot, path)
 	}
@@ -108,6 +110,16 @@ func newCodeRepo(code codehost.Repo, codeRoot, path string) (Repo, error) {
 	//	pathMajor = .v2
 	//	pseudoMajor = v2
 	//
+	// Starting in 1.25, subdir may be passed in by the go-import meta tag.
+	// So it may be the case that:
+	//	path = github.com/rsc/foo/v2
+	//	codeRoot = github.com/rsc/foo
+	//	subdir = bar/subdir
+	//	pathPrefix = github.com/rsc/foo
+	//	pathMajor = /v2
+	//	pseudoMajor = v2
+	// which means that codeDir = bar/subdir
+
 	codeDir := ""
 	if codeRoot != path {
 		if !hasPathPrefix(pathPrefix, codeRoot) {
@@ -115,6 +127,9 @@ func newCodeRepo(code codehost.Repo, codeRoot, path string) (Repo, error) {
 		}
 		codeDir = strings.Trim(pathPrefix[len(codeRoot):], "/")
 	}
+	if subdir != "" {
+		codeDir = filepath.ToSlash(filepath.Join(codeDir, subdir))
+	}
 
 	r := &codeRepo{
 		modPath:     path,

src/cmd/go/internal/modfetch/coderepo_test.go

@@ -884,6 +884,16 @@ var latestTests = []struct {
 		path:    "swtch.com/testmod",
 		version: "v1.1.1",
 	},
+	{
+		vcs:     "git",
+		path:    "vcs-test.golang.org/go/gitreposubdir",
+		version: "v1.2.3",
+	},
+	{
+		vcs:     "git",
+		path:    "vcs-test.golang.org/go/gitreposubdirv2/v2",
+		version: "v2.0.0",
+	},
 }
 
 func TestLatest(t *testing.T) {
@@ -950,7 +960,7 @@ func TestNonCanonicalSemver(t *testing.T) {
 		},
 	}
 
-	cr, err := newCodeRepo(ch, root, root)
+	cr, err := newCodeRepo(ch, root, "", root)
 	if err != nil {
 		t.Fatal(err)
 	}

src/cmd/go/internal/modfetch/repo.go

@@ -240,7 +240,7 @@ func LookupLocal(ctx context.Context, codeRoot string, path string, dir string)
 			if err != nil {
 				return nil, err
 			}
-			r, err := newCodeRepo(code, codeRoot, path)
+			r, err := newCodeRepo(code, codeRoot, "", path)
 			if err == nil && traceRepo {
 				r = newLoggingRepo(r)
 			}
@@ -319,7 +319,7 @@ func lookupDirect(ctx context.Context, path string) (Repo, error) {
 	if err != nil {
 		return nil, err
 	}
-	return newCodeRepo(code, rr.Root, path)
+	return newCodeRepo(code, rr.Root, rr.SubDir, path)
 }
 
 func lookupCodeRepo(ctx context.Context, rr *vcs.RepoRoot, local bool) (codehost.Repo, error) {

src/cmd/go/internal/vcs/discovery.go

@@ -54,12 +54,17 @@ func parseMetaGoImports(r io.Reader, mod ModuleMode) ([]metaImport, error) {
 		if attrValue(e.Attr, "name") != "go-import" {
 			continue
 		}
-		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 {
+		if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 || len(f) == 4 {
-			imports = append(imports, metaImport{
+			mi := metaImport{
 				Prefix:   f[0],
 				VCS:      f[1],
 				RepoRoot: f[2],
-			})
+			}
+			// An optional subdirectory may be provided.
+			if len(f) == 4 {
+				mi.SubDir = f[3]
+			}
+			imports = append(imports, mi)
 		}
 	}
 

src/cmd/go/internal/vcs/discovery_test.go

@@ -18,15 +18,15 @@ var parseMetaGoImportsTests = []struct {
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
 		IgnoreMod,
-		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
+		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar", ""}},
 	},
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		<meta name="go-import" content="baz/quux git http://github.com/rsc/baz/quux">`,
 		IgnoreMod,
 		[]metaImport{
-			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
+			{"foo/bar", "git", "https://github.com/rsc/foo/bar", ""},
-			{"baz/quux", "git", "http://github.com/rsc/baz/quux"},
+			{"baz/quux", "git", "http://github.com/rsc/baz/quux", ""},
 		},
 	},
 	{
@@ -34,7 +34,7 @@ var parseMetaGoImportsTests = []struct {
 		<meta name="go-import" content="foo/bar mod http://github.com/rsc/baz/quux">`,
 		IgnoreMod,
 		[]metaImport{
-			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
+			{"foo/bar", "git", "https://github.com/rsc/foo/bar", ""},
 		},
 	},
 	{
@@ -42,7 +42,7 @@ var parseMetaGoImportsTests = []struct {
 		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
 		IgnoreMod,
 		[]metaImport{
-			{"foo/bar", "git", "https://github.com/rsc/foo/bar"},
+			{"foo/bar", "git", "https://github.com/rsc/foo/bar", ""},
 		},
 	},
 	{
@@ -50,7 +50,7 @@ var parseMetaGoImportsTests = []struct {
 		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
 		PreferMod,
 		[]metaImport{
-			{"foo/bar", "mod", "http://github.com/rsc/baz/quux"},
+			{"foo/bar", "mod", "http://github.com/rsc/baz/quux", ""},
 		},
 	},
 	{
@@ -58,31 +58,31 @@ var parseMetaGoImportsTests = []struct {
 		<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		</head>`,
 		IgnoreMod,
-		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
+		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar", ""}},
 	},
 	{
 		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">
 		<body>`,
 		IgnoreMod,
-		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
+		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar", ""}},
 	},
 	{
 		`<!doctype html><meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar">`,
 		IgnoreMod,
-		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar"}},
+		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar", ""}},
 	},
 	{
 		// XML doesn't like <div style=position:relative>.
 		`<!doctype html><title>Page Not Found</title><meta name=go-import content="chitin.io/chitin git https://github.com/chitin-io/chitin"><div style=position:relative>DRAFT</div>`,
 		IgnoreMod,
-		[]metaImport{{"chitin.io/chitin", "git", "https://github.com/chitin-io/chitin"}},
+		[]metaImport{{"chitin.io/chitin", "git", "https://github.com/chitin-io/chitin", ""}},
 	},
 	{
 		`<meta name="go-import" content="myitcv.io git https://github.com/myitcv/x">
 	        <meta name="go-import" content="myitcv.io/blah2 mod https://raw.githubusercontent.com/myitcv/pubx/master">
 	        `,
 		IgnoreMod,
-		[]metaImport{{"myitcv.io", "git", "https://github.com/myitcv/x"}},
+		[]metaImport{{"myitcv.io", "git", "https://github.com/myitcv/x", ""}},
 	},
 	{
 		`<meta name="go-import" content="myitcv.io git https://github.com/myitcv/x">
@@ -90,10 +90,20 @@ var parseMetaGoImportsTests = []struct {
 	        `,
 		PreferMod,
 		[]metaImport{
-			{"myitcv.io/blah2", "mod", "https://raw.githubusercontent.com/myitcv/pubx/master"},
+			{"myitcv.io/blah2", "mod", "https://raw.githubusercontent.com/myitcv/pubx/master", ""},
-			{"myitcv.io", "git", "https://github.com/myitcv/x"},
+			{"myitcv.io", "git", "https://github.com/myitcv/x", ""},
 		},
 	},
+	{
+		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar subdir">`,
+		IgnoreMod,
+		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar", "subdir"}},
+	},
+	{
+		`<meta name="go-import" content="foo/bar git https://github.com/rsc/foo/bar subdir/path">`,
+		IgnoreMod,
+		[]metaImport{{"foo/bar", "git", "https://github.com/rsc/foo/bar", "subdir/path"}},
+	},
 }
 
 func TestParseMetaGoImports(t *testing.T) {

src/cmd/go/internal/vcs/vcs.go

@@ -1056,7 +1056,8 @@ func checkGOVCS(vcs *Cmd, root string) error {
 // RepoRoot describes the repository root for a tree of source code.
 type RepoRoot struct {
 	Repo     string // repository URL, including scheme
-	Root     string // import path corresponding to root of repo
+	Root     string // import path corresponding to the SubDir
+	SubDir   string // subdirectory within the repo (empty for root)
 	IsCustom bool   // defined by served <meta> tags (as opposed to hard-coded pattern)
 	VCS      *Cmd
 }
@@ -1368,6 +1369,7 @@ func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.Se
 	rr := &RepoRoot{
 		Repo:     repoURL,
 		Root:     mmi.Prefix,
+		SubDir:   mmi.SubDir,
 		IsCustom: true,
 		VCS:      vcs,
 	}
@@ -1457,9 +1459,9 @@ type fetchResult struct {
 }
 
 // metaImport represents the parsed <meta name="go-import"
-// content="prefix vcs reporoot" /> tags from HTML files.
+// content="prefix vcs reporoot subdir" /> tags from HTML files.
 type metaImport struct {
-	Prefix, VCS, RepoRoot string
+	Prefix, VCS, RepoRoot, SubDir string
 }
 
 // An ImportMismatchError is returned where metaImport/s are present

src/cmd/go/internal/vcs/vcs_test.go

@@ -425,6 +425,28 @@ func TestMatchGoImport(t *testing.T) {
 			path: "myitcv.io/other",
 			mi:   metaImport{Prefix: "myitcv.io", VCS: "git", RepoRoot: "https://github.com/myitcv/x"},
 		},
+		{
+			imports: []metaImport{
+				{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target", SubDir: "subdir"},
+			},
+			path: "example.com/user/foo",
+			mi:   metaImport{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target", SubDir: "subdir"},
+		},
+		{
+			imports: []metaImport{
+				{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target", SubDir: "foo/subdir"},
+			},
+			path: "example.com/user/foo",
+			mi:   metaImport{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target", SubDir: "foo/subdir"},
+		},
+		{
+			imports: []metaImport{
+				{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target", SubDir: "subdir"},
+				{Prefix: "example.com/user/foo", VCS: "git", RepoRoot: "https://example.com/repo/target", SubDir: ""},
+			},
+			path: "example.com/user/foo",
+			err:  errors.New("multiple meta tags match import path"),
+		},
 	}
 
 	for _, test := range tests {

src/cmd/go/testdata/script/mod_get_subdir.txt

@@ -0,0 +1,77 @@
+# golang.org/issue/34055
+# Starting in Go 1.25, go-import meta tag support an optional subdirectory paramater.
+# The corresponding go-import meta tag is specified as
+# <meta name="go-import" content="vcs-test.golang.org/go/gitreposubdir git https://vcs-test.golang.org/git/gitreposubdir foo/subdir">
+# and contains the module in vcs-test.golang.org/git/gitreposubdir/foo/subdir.
+# See testdata/vcstest/go/gitreposubdir.txt and testdata/vcstest/git/gitreposubdir.txt
+
+[short] skip 'builds a go program'
+[!git] skip
+
+env GO111MODULE=on
+env GOPROXY=direct
+env GOSUMDB=off
+
+# Get the module without having to specify the subdir.
+cd a
+cp go.mod go.mod.orig
+go get vcs-test.golang.org/go/gitreposubdir@v1.2.3
+exists $GOPATH/pkg/mod/vcs-test.golang.org/go/gitreposubdir@v1.2.3
+go get vcs-test.golang.org/go/gitreposubdirv2/v2@v2.0.0
+exists $GOPATH/pkg/mod/vcs-test.golang.org/go/gitreposubdirv2/v2@v2.0.0
+
+# Import the module without having to specify the subdir.
+cp go.mod.orig go.mod
+go mod tidy
+
+# Run main.go which has the import.
+go run main.go
+stdout 'hello, world'
+stdout 'hello, world v2'
+
+# Fail if subdir is specified in get.
+! go get vcs-test.golang.org/go/gitreposubdir/foo/subdir
+stderr 'module vcs-test.golang.org/go/gitreposubdir@upgrade found \(v1.2.3\), but does not contain package vcs-test.golang.org/go/gitreposubdir/foo/subdir'
+! go get vcs-test.golang.org/go/gitreposubdirv2/v2/foo/subdir
+stderr 'module vcs-test.golang.org/go/gitreposubdirv2/v2@upgrade found \(v2.0.0\), but does not contain package vcs-test.golang.org/go/gitreposubdirv2/v2/foo/subdir'
+
+# Fail if subdir is specified in the import.
+cd ../b
+! go mod tidy
+stderr 'module vcs-test.golang.org/go/gitreposubdir@latest found \(v1.2.3\), but does not contain package vcs-test.golang.org/go/gitreposubdir/foo/subdir'
+stderr 'module vcs-test.golang.org/go/gitreposubdirv2/v2@latest found \(v2.0.0\), but does not contain package vcs-test.golang.org/go/gitreposubdirv2/v2/foo/subdir'
+
+-- a/main.go --
+package main
+
+import (
+    "fmt"
+    "vcs-test.golang.org/go/gitreposubdir"
+    "vcs-test.golang.org/go/gitreposubdirv2/v2"
+)
+
+func main() {
+    fmt.Println(greeter.Hello())
+    fmt.Println(greeterv2.Hello())
+}
+-- a/go.mod --
+module example
+
+go 1.24
+-- b/main.go --
+package main
+
+import (
+    "fmt"
+    "vcs-test.golang.org/go/gitreposubdir/foo/subdir"
+    "vcs-test.golang.org/go/gitreposubdirv2/v2/foo/subdir"
+)
+
+func main() {
+    fmt.Println(greeter.Hello())
+    fmt.Println(greeterv2.Hello())
+}
+-- b/go.mod --
+module example
+
+go 1.24

src/cmd/go/testdata/vcstest/git/gitreposubdir.txt

@@ -0,0 +1,26 @@
+handle git
+
+env GIT_AUTHOR_NAME='Sam Thanawalla'
+env GIT_AUTHOR_EMAIL='samthanawalla@google.com'
+env GIT_COMMITTER_NAME=$GIT_AUTHOR_NAME
+env GIT_COMMITTER_EMAIL=$GIT_AUTHOR_EMAIL
+
+at 2019-10-07T14:15:32-04:00
+
+git init
+
+git add foo/subdir
+git commit -m 'initial commit'
+git branch -m master
+git tag foo/subdir/v1.2.3
+
+-- foo/subdir/go.mod --
+module vcs-test.golang.org/go/gitreposubdir
+
+go 1.23
+-- foo/subdir/hello.go --
+package greeter
+
+func Hello() string {
+	return "hello, world"
+}

src/cmd/go/testdata/vcstest/git/gitreposubdirv2.txt

@@ -0,0 +1,31 @@
+handle git
+
+env GIT_AUTHOR_NAME='Sam Thanawalla'
+env GIT_AUTHOR_EMAIL='samthanawalla@google.com'
+env GIT_COMMITTER_NAME=$GIT_AUTHOR_NAME
+env GIT_COMMITTER_EMAIL=$GIT_AUTHOR_EMAIL
+
+at 2019-10-07T14:15:32-04:00
+
+git init
+
+git add subdir
+git commit -m 'initial commit'
+git branch -m master
+git tag subdir/v2.0.0
+git show-ref --tags --heads
+cmp stdout .git-refs
+
+-- .git-refs --
+5212d800bfd1f6377da46aee6cbceca2f60d4ea6 refs/heads/master
+5212d800bfd1f6377da46aee6cbceca2f60d4ea6 refs/tags/subdir/v2.0.0
+-- subdir/go.mod --
+module vcs-test.golang.org/go/gitreposubdirv2/v2
+
+go 1.23
+-- subdir/hello.go --
+package greeterv2
+
+func Hello() string {
+	return "hello, world v2"
+}

src/cmd/go/testdata/vcstest/go/gitreposubdir.txt

@@ -0,0 +1,6 @@
+handle dir
+
+-- index.html --
+<!DOCTYPE html>
+<html>
+<meta name="go-import" content="vcs-test.golang.org/go/gitreposubdir git https://vcs-test.golang.org/git/gitreposubdir foo/subdir">

src/cmd/go/testdata/vcstest/go/gitreposubdirv2.txt

@@ -0,0 +1,6 @@
+handle dir
+
+-- v2/index.html --
+<!DOCTYPE html>
+<html>
+<meta name="go-import" content="vcs-test.golang.org/go/gitreposubdirv2/v2 git https://vcs-test.golang.org/git/gitreposubdirv2 subdir">