internal/imports: support vendoring in module mode

Previously, goimports half-supported vendor mode -- it searched the
module cache on some code paths and the vendor dir in others. That
seemed to work okay, probably because people happened to have a populated
module cache. In 1.14, it's much more likely that people will work
solely from the vendor directory.

In this CL we bite the bullet and fully support vendor mode. 1.14 makes
this particularly challenging by disabling list -m ... in vendor mode, and
by enabling it automatically under some circumstances. We need to mirror
that behavior, which means knowing whether we're running with 1.14, and
figuring out whether vendoring should be enabled given that. We collect
the information we need with a list -m -f query just on the main module.

If vendor mode is enabled, we throw away all the modules and replace
them with a single pseudo-module rooted at /vendor. Everything basically
works at that point.

Fixes golang/go#34826

Change-Id: Ia4030344d822d5a4a3bbc010912ab98bf2f5f95b
Reviewed-on: https://go-review.googlesource.com/c/tools/+/203017
Reviewed-by: Bryan C. Mills <bcmills@google.com>

internal/imports/mod.go

@@ -14,10 +14,10 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 
 	"golang.org/x/tools/internal/gopathwalk"
 	"golang.org/x/tools/internal/module"
+	"golang.org/x/tools/internal/semver"
 )
 
 // ModuleResolver implements resolver for modules using the go command as little
@@ -25,6 +25,7 @@ import (
 type ModuleResolver struct {
 	env            *ProcessEnv
 	moduleCacheDir string
+	dummyVendorMod *ModuleJSON // If vendoring is enabled, the pseudo-module that represents the /vendor directory.
 
 	Initialized   bool
 	Main          *ModuleJSON
@@ -37,50 +38,40 @@ type ModuleResolver struct {
 }
 
 type ModuleJSON struct {
-	Path     string           // module path
+	Path      string      // module path
-	Version  string           // module version
+	Replace   *ModuleJSON // replaced by this module
-	Versions []string         // available module versions (with -versions)
+	Main      bool        // is this the main module?
-	Replace  *ModuleJSON      // replaced by this module
+	Dir       string      // directory holding files for this module, if any
-	Time     *time.Time       // time version was created
+	GoMod     string      // path to go.mod file for this module, if any
-	Update   *ModuleJSON      // available update, if any (with -u)
+	GoVersion string      // go version used in module
-	Main     bool             // is this the main module?
-	Indirect bool             // is this module only an indirect dependency of main module?
-	Dir      string           // directory holding files for this module, if any
-	GoMod    string           // path to go.mod file for this module, if any
-	Error    *ModuleErrorJSON // error loading module
-}
-
-type ModuleErrorJSON struct {
-	Err string // the error itself
 }
 
 func (r *ModuleResolver) init() error {
 	if r.Initialized {
 		return nil
 	}
-	stdout, err := r.env.invokeGo("list", "-m", "-json", "...")
+	mainMod, vendorEnabled, err := vendorEnabled(r.env)
 	if err != nil {
 		return err
 	}
-	for dec := json.NewDecoder(stdout); dec.More(); {
+
-		mod := &ModuleJSON{}
+	if mainMod != nil && vendorEnabled {
-		if err := dec.Decode(mod); err != nil {
+		// Vendor mode is on, so all the non-Main modules are irrelevant,
-			return err
+		// and we need to search /vendor for everything.
-		}
+		r.Main = mainMod
-		if mod.Dir == "" {
+		r.dummyVendorMod = &ModuleJSON{
-			if r.env.Debug {
+			Path: "",
-				r.env.Logf("module %v has not been downloaded and will be ignored", mod.Path)
+			Dir:  filepath.Join(mainMod.Dir, "vendor"),
-			}
-			// Can't do anything with a module that's not downloaded.
-			continue
-		}
-		r.ModsByModPath = append(r.ModsByModPath, mod)
-		r.ModsByDir = append(r.ModsByDir, mod)
-		if mod.Main {
-			r.Main = mod
 		}
+		r.ModsByModPath = []*ModuleJSON{mainMod, r.dummyVendorMod}
+		r.ModsByDir = []*ModuleJSON{mainMod, r.dummyVendorMod}
+	} else {
+		// Vendor mode is off, so run go list -m ... to find everything.
+		r.initAllMods()
 	}
 
+	r.moduleCacheDir = filepath.Join(filepath.SplitList(r.env.GOPATH)[0], "/pkg/mod")
+
 	sort.Slice(r.ModsByModPath, func(i, j int) bool {
 		count := func(x int) int {
 			return strings.Count(r.ModsByModPath[x].Path, "/")
@@ -104,11 +95,36 @@ func (r *ModuleResolver) init() error {
 			dirs: map[string]*directoryPackageInfo{},
 		}
 	}
-
 	r.Initialized = true
 	return nil
 }
 
+func (r *ModuleResolver) initAllMods() error {
+	stdout, err := r.env.invokeGo("list", "-m", "-json", "...")
+	if err != nil {
+		return err
+	}
+	for dec := json.NewDecoder(stdout); dec.More(); {
+		mod := &ModuleJSON{}
+		if err := dec.Decode(mod); err != nil {
+			return err
+		}
+		if mod.Dir == "" {
+			if r.env.Debug {
+				r.env.Logf("module %v has not been downloaded and will be ignored", mod.Path)
+			}
+			// Can't do anything with a module that's not downloaded.
+			continue
+		}
+		r.ModsByModPath = append(r.ModsByModPath, mod)
+		r.ModsByDir = append(r.ModsByDir, mod)
+		if mod.Main {
+			r.Main = mod
+		}
+	}
+	return nil
+}
+
 func (r *ModuleResolver) ClearForNewScan() {
 	r.otherCache = &dirInfoCache{
 		dirs: map[string]*directoryPackageInfo{},
@@ -249,6 +265,10 @@ func (r *ModuleResolver) dirIsNestedModule(dir string, mod *ModuleJSON) bool {
 		// so it cannot be a nested module.
 		return false
 	}
+	if mod != nil && mod == r.dummyVendorMod {
+		// The /vendor pseudomodule is flattened and doesn't actually count.
+		return false
+	}
 	modDir, _ := r.modInfo(dir)
 	if modDir == "" {
 		return false
@@ -327,15 +347,15 @@ func (r *ModuleResolver) scan(_ references, loadNames bool, exclude []gopathwalk
 	if r.Main != nil {
 		roots = append(roots, gopathwalk.Root{r.Main.Dir, gopathwalk.RootCurrentModule})
 	}
-	if r.moduleCacheDir == "" {
+	if r.dummyVendorMod != nil {
-		r.moduleCacheDir = filepath.Join(filepath.SplitList(r.env.GOPATH)[0], "/pkg/mod")
+		roots = append(roots, gopathwalk.Root{r.dummyVendorMod.Dir, gopathwalk.RootOther})
-	}
+	} else {
-	roots = append(roots, gopathwalk.Root{r.moduleCacheDir, gopathwalk.RootModuleCache})
+		roots = append(roots, gopathwalk.Root{r.moduleCacheDir, gopathwalk.RootModuleCache})
-
+		// Walk replace targets, just in case they're not in any of the above.
-	// Walk replace targets, just in case they're not in any of the above.
+		for _, mod := range r.ModsByModPath {
-	for _, mod := range r.ModsByModPath {
+			if mod.Replace != nil {
-		if mod.Replace != nil {
+				roots = append(roots, gopathwalk.Root{mod.Dir, gopathwalk.RootOther})
-			roots = append(roots, gopathwalk.Root{mod.Dir, gopathwalk.RootOther})
+			}
 		}
 	}
 
@@ -429,7 +449,7 @@ func (r *ModuleResolver) canonicalize(info directoryPackageInfo) (*pkg, error) {
 	}
 
 	res := &pkg{
-		importPathShort: VendorlessPath(importPath),
+		importPathShort: importPath,
 		dir:             info.dir,
 		packageName:     info.packageName, // may not be populated if the caller didn't ask for it
 	}
@@ -455,14 +475,10 @@ func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) dir
 	}
 	importPath := filepath.ToSlash(subdir)
 	if strings.HasPrefix(importPath, "vendor/") {
-		// Ignore vendor dirs. If -mod=vendor is on, then things
+		// Only enter vendor directories if they're explicitly requested as a root.
-		// should mostly just work, but when it's not vendor/
-		// is a mess. There's no easy way to tell if it's on.
-		// We can still find things in the mod cache and
-		// map them into /vendor when -mod=vendor is on.
 		return directoryPackageInfo{
 			status: directoryScanned,
-			err:    fmt.Errorf("vendor directory"),
+			err:    fmt.Errorf("unwanted vendor directory"),
 		}
 	}
 	switch root.Type {
@@ -487,8 +503,6 @@ func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) dir
 			}
 		}
 		importPath = path.Join(modPath, filepath.ToSlash(matches[3]))
-	case gopathwalk.RootGOROOT:
-		importPath = subdir
 	}
 
 	modDir, modName := r.modInfo(dir)
@@ -562,3 +576,63 @@ func modulePath(mod []byte) string {
 	}
 	return "" // missing module path
 }
+
+var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`)
+
+// vendorEnabled indicates if vendoring is enabled.
+// Inspired by setDefaultBuildMod in modload/init.go
+func vendorEnabled(env *ProcessEnv) (*ModuleJSON, bool, error) {
+	mainMod, go114, err := getMainModuleAnd114(env)
+	if err != nil {
+		return nil, false, err
+	}
+	matches := modFlagRegexp.FindStringSubmatch(env.GOFLAGS)
+	var modFlag string
+	if len(matches) != 0 {
+		modFlag = matches[1]
+	}
+	if modFlag != "" {
+		// Don't override an explicit '-mod=' argument.
+		return mainMod, modFlag == "vendor", nil
+	}
+	if mainMod == nil || !go114 {
+		return mainMod, false, nil
+	}
+	// Check 1.14's automatic vendor mode.
+	if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() {
+		if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 {
+			// The Go version is at least 1.14, and a vendor directory exists.
+			// Set -mod=vendor by default.
+			return mainMod, true, nil
+		}
+	}
+	return mainMod, false, nil
+}
+
+// getMainModuleAnd114 gets the main module's information and whether the
+// go command in use is 1.14+. This is the information needed to figure out
+// if vendoring should be enabled.
+func getMainModuleAnd114(env *ProcessEnv) (*ModuleJSON, bool, error) {
+	const format = `{{.Path}}
+{{.Dir}}
+{{.GoMod}}
+{{.GoVersion}}
+{{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}}
+`
+	stdout, err := env.invokeGo("list", "-m", "-f", format)
+	if err != nil {
+		return nil, false, nil
+	}
+	lines := strings.Split(stdout.String(), "\n")
+	if len(lines) < 5 {
+		return nil, false, fmt.Errorf("unexpected stdout: %q", stdout)
+	}
+	mod := &ModuleJSON{
+		Path:      lines[0],
+		Dir:       lines[1],
+		GoMod:     lines[2],
+		GoVersion: lines[3],
+		Main:      true,
+	}
+	return mod, lines[4] == "go1.14", nil
+}

internal/imports/mod_114_test.go

@@ -0,0 +1,11 @@
+// +build go1.14
+
+package imports
+
+import (
+	"testing"
+)
+
+func TestModVendorAuto_114(t *testing.T) {
+	testModVendorAuto(t, true)
+}

internal/imports/mod_pre114_test.go

@@ -0,0 +1,11 @@
+// +build !go1.14
+
+package imports
+
+import (
+	"testing"
+)
+
+func TestModVendorAuto_Pre114(t *testing.T) {
+	testModVendorAuto(t, false)
+}

internal/imports/mod_test.go

@@ -216,31 +216,60 @@ import _ "rsc.io/quote"
 
 }
 
-// Tests that -mod=vendor sort of works. Adapted from mod_getmode_vendor.txt.
+// Tests that -mod=vendor works. Adapted from mod_vendor_build.txt.
-func TestModGetmodeVendor(t *testing.T) {
+func TestModVendorBuild(t *testing.T) {
-	t.Skip("'go list -m -mod=vendor' currently not allowed: see golang.org/issue/34826")
 	mt := setup(t, `
 -- go.mod --
-module x
+module m
-
+go 1.12
-require rsc.io/quote v1.5.2
+require rsc.io/sampler v1.3.1
 -- x.go --
 package x
-import _ "rsc.io/quote"
+import _ "rsc.io/sampler"
 `, "")
 	defer mt.cleanup()
 
+	// Sanity-check the setup.
+	mt.assertModuleFoundInDir("rsc.io/sampler", "sampler", `pkg.*mod.*/sampler@.*$`)
+
+	// Populate vendor/ and clear out the mod cache so we can't cheat.
+	if _, err := mt.env.invokeGo("mod", "vendor"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := mt.env.invokeGo("clean", "-modcache"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Clear out the resolver's cache, since we've changed the environment.
+	mt.resolver = &ModuleResolver{env: mt.env}
+	mt.env.GOFLAGS = "-mod=vendor"
+	mt.assertModuleFoundInDir("rsc.io/sampler", "sampler", `/vendor/`)
+}
+
+// Tests that -mod=vendor is auto-enabled only for go1.14 and higher.
+// Vaguely inspired by mod_vendor_auto.txt.
+func testModVendorAuto(t *testing.T, wantEnabled bool) {
+	mt := setup(t, `
+-- go.mod --
+module m
+go 1.14
+require rsc.io/sampler v1.3.1
+-- x.go --
+package x
+import _ "rsc.io/sampler"
+`, "")
+	defer mt.cleanup()
+
+	// Populate vendor/.
 	if _, err := mt.env.invokeGo("mod", "vendor"); err != nil {
 		t.Fatal(err)
 	}
 
-	mt.env.GOFLAGS = "-mod=vendor"
+	wantDir := `pkg.*mod.*/sampler@.*$`
-	mt.assertModuleFoundInDir("rsc.io/quote", "quote", `/vendor/`)
+	if wantEnabled {
-
+		wantDir = `/vendor/`
-	mt.env.GOFLAGS = ""
+	}
-	// Clear out the resolver's cache, since we've changed the environment.
+	mt.assertModuleFoundInDir("rsc.io/sampler", "sampler", wantDir)
-	mt.resolver = &ModuleResolver{env: mt.env}
-	mt.assertModuleFoundInDir("rsc.io/quote", "quote", `pkg.*mod.*/quote@.*$`)
 }
 
 // Tests that a module replace works. Adapted from mod_list.txt. We start with
@@ -623,6 +652,8 @@ func setup(t *testing.T, main, wd string) *modTest {
 		GOPROXY:     proxyDirToURL(proxyDir),
 		GOSUMDB:     "off",
 		WorkingDir:  filepath.Join(mainDir, wd),
+		Debug:       *testDebug,
+		Logf:        log.Printf,
 	}
 
 	// go mod download gets mad if we don't have a go.mod, so make sure we do.