cybercyst/go
1
0
Fork 0
mirror of https://github.com/golang/go.git synced 2025-05-05 15:43:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

cmd/goimports, imports: make goimports great again

Browse Source 
I felt the burn of my laptop on my legs, spinning away while processing
goimports, and felt that it was time to make goimports great again.

Over the past few years goimports fell into a slow state of disrepair
with too many feature additions and no attention to the performance
death by a thousand cuts. This was particularly terrible on OS X with
its lackluster filesystem buffering.

This CL makes goimports stronger, together with various optimizations
and more visibility into what goimports is doing.

* adds more internal documentation

* avoids scanning $GOPATH for answers when running goimports on a file
  under $GOROOT (for Go core hackers)

* don't read all $GOROOT & $GOPATH directories' Go code looking for
  their package names until much later. Require the package name of
  missing imports to be present in the last two directory path
  components.  Then only try importing them in order from best to
  worst (shortest to longest, as before), so we can stop early.

* when adding imports, add names to imports when the imported package name
  doesn't match the baes of its import path. For example:
        import foo "example.net/foo/v1"

* don't read all *.go files in a package directory once the first file
  in a directory has revealed itself to be a package we're not looking
  for. For example, if we're looking for the right "client" for "client.Foo",
  we used to consider a directory "bar/client" as a candidate and read
  all 50 of its *.go files instead of stopping after its first *.go
  file had a "package main" line.

* add some fast paths to remove allocations

* add some fast paths to remove disk I/O when looking up the base
  package name of a standard library import (of existing imports in a
  file, which are very common)

* adds a special case for import "C", to avoid some disk I/O.

* add a -verbose flag to goimports for debugging

On my Mac laptop with a huge $GOPATH, with a test file like:

	package foo
	import (
	       "fmt"
	       "net/http"
	)

	/*

	*/
	import "C"

	var _ = cloudbilling.New
	var _ = http.NewRequest
	var _ = client.New

... this took like 10 seconds before, and now 1.3 seconds. (Still
slow; disk-based caching can come later)

Updates golang/go#16367 (goimports is slow)
Updates golang/go#16384 (refactor TestRename is broken on Windows)

Change-Id: I97e85d3016afc9f2ad5501f97babad30c7989183
Reviewed-on: https://go-review.googlesource.com/24941
Reviewed-by: Andrew Gerrand <adg@golang.org>
Reviewed-by: Rob Pike <r@golang.org>
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
This commit is contained in:
Brad Fitzpatrick 2016-07-14 17:08:27 -07:00
parent b3887f7a17
commit e047ae774b
4 changed files with 596 additions and 249 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/goimports/goimports.go
View File

@ -11,6 +11,7 @@ import (
"go/scanner" "go/scanner"
"io" "io"
"io/ioutil" "io/ioutil"
"log"
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
@ -22,10 +23,11 @@ import (
var ( var (
// main operation modes // main operation modes
list = flag.Bool("l", false, "list files whose formatting differs from goimport's") list = flag.Bool("l", false, "list files whose formatting differs from goimport's")
write = flag.Bool("w", false, "write result to (source) file instead of stdout") write = flag.Bool("w", false, "write result to (source) file instead of stdout")
doDiff = flag.Bool("d", false, "display diffs instead of rewriting files") doDiff = flag.Bool("d", false, "display diffs instead of rewriting files")
srcdir = flag.String("srcdir", "", "choose imports as if source code is from `dir`") srcdir = flag.String("srcdir", "", "choose imports as if source code is from `dir`")
verbose = flag.Bool("v", false, "verbose logging")
options = &imports.Options{ options = &imports.Options{
TabWidth: 8, TabWidth: 8,
@ -154,6 +156,10 @@ func gofmtMain() {
flag.Usage = usage flag.Usage = usage
paths := parseFlags() paths := parseFlags()
if *verbose {
log.SetFlags(log.LstdFlags | log.Lmicroseconds)
imports.Debug = true
}
if options.TabWidth < 0 { if options.TabWidth < 0 {
fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", options.TabWidth) fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", options.TabWidth)
exitCode = 2 exitCode = 2

437
imports/fix.go
View File

@ -10,15 +10,21 @@ import (
"go/build" "go/build"
"go/parser" "go/parser"
"go/token" "go/token"
"io/ioutil"
"log"
"os" "os"
"path" "path"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"sync" "sync"
"golang.org/x/tools/go/ast/astutil" "golang.org/x/tools/go/ast/astutil"
) )
// Debug controls verbose logging.
var Debug = false
// importToGroup is a list of functions which map from an import path to // importToGroup is a list of functions which map from an import path to
// a group number. // a group number.
var importToGroup = []func(importPath string) (num int, ok bool){ var importToGroup = []func(importPath string) (num int, ok bool){
@ -58,7 +64,10 @@ func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []stri
if err != nil { if err != nil {
return nil, err return nil, err
} }
srcDir := path.Dir(abs) srcDir := filepath.Dir(abs)
if Debug {
log.Printf("fixImports(filename=%q), abs=%q, srcDir=%q ...", filename, abs, srcDir)
}
// collect potential uses of packages. // collect potential uses of packages.
var visitor visitFn var visitor visitFn
@ -70,10 +79,14 @@ func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []stri
case *ast.ImportSpec: case *ast.ImportSpec:
if v.Name != nil { if v.Name != nil {
decls[v.Name.Name] = v decls[v.Name.Name] = v
} else { break
local := importPathToName(strings.Trim(v.Path.Value, `\"`), srcDir)
decls[local] = v
} }
ipath := strings.Trim(v.Path.Value, `\"`)
if ipath == "C" {
break
}
local := importPathToName(ipath, srcDir)
decls[local] = v
case *ast.SelectorExpr: case *ast.SelectorExpr:
xident, ok := v.X.(*ast.Ident) xident, ok := v.X.(*ast.Ident)
if !ok { if !ok {
@ -114,18 +127,22 @@ func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []stri
astutil.DeleteNamedImport(fset, f, name, ipath) astutil.DeleteNamedImport(fset, f, name, ipath)
} }
for pkgName, symbols := range refs {
if len(symbols) == 0 {
// skip over packages already imported
delete(refs, pkgName)
}
}
// Search for imports matching potential package references. // Search for imports matching potential package references.
searches := 0 searches := 0
type result struct { type result struct {
ipath string ipath string // import path (if err == nil)
name string name string // optional name to rename import as
err error err error
} }
results := make(chan result) results := make(chan result)
for pkgName, symbols := range refs { for pkgName, symbols := range refs {
if len(symbols) == 0 {
continue // skip over packages already imported
}
go func(pkgName string, symbols map[string]bool) { go func(pkgName string, symbols map[string]bool) {
ipath, rename, err := findImport(pkgName, symbols, filename) ipath, rename, err := findImport(pkgName, symbols, filename)
r := result{ipath: ipath, err: err} r := result{ipath: ipath, err: err}
@ -155,7 +172,7 @@ func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []stri
} }
// importPathToName returns the package name for the given import path. // importPathToName returns the package name for the given import path.
var importPathToName = importPathToNameGoPath var importPathToName func(importPath, srcDir string) (packageName string) = importPathToNameGoPath
// importPathToNameBasic assumes the package name is the base of import path. // importPathToNameBasic assumes the package name is the base of import path.
func importPathToNameBasic(importPath, srcDir string) (packageName string) { func importPathToNameBasic(importPath, srcDir string) (packageName string) {
@ -165,24 +182,76 @@ func importPathToNameBasic(importPath, srcDir string) (packageName string) {
// importPathToNameGoPath finds out the actual package name, as declared in its .go files. // importPathToNameGoPath finds out the actual package name, as declared in its .go files.
// If there's a problem, it falls back to using importPathToNameBasic. // If there's a problem, it falls back to using importPathToNameBasic.
func importPathToNameGoPath(importPath, srcDir string) (packageName string) { func importPathToNameGoPath(importPath, srcDir string) (packageName string) {
// Fast path for standard library without going to disk:
if pkg, ok := stdImportPackage[importPath]; ok {
return pkg
}
// TODO(bradfitz): build.Import does too much work, and
// doesn't cache either in-process or long-lived (anything
// found in the first pass from explicit imports aren't used
// again when scanning all directories). Also, it opens+reads
// *_test.go files too. As a baby step, use a cheaper
// mechanism to start (build.FindOnly), and then just read a
// single file (like parser.ParseFile in loadExportsGoPath) and
// skip only "documentation" but otherwise trust the first matching
// file's package name.
if Debug {
log.Printf("build.Import(ip=%q, srcDir=%q) ...", importPath, srcDir)
}
if buildPkg, err := build.Import(importPath, srcDir, 0); err == nil { if buildPkg, err := build.Import(importPath, srcDir, 0); err == nil {
if Debug {
log.Printf("build.Import(%q, srcDir=%q) = %q", importPath, srcDir, buildPkg.Name)
}
return buildPkg.Name return buildPkg.Name
} else { } else {
if Debug {
log.Printf("build.Import(%q, srcDir=%q) error: %v", importPath, srcDir, err)
}
return importPathToNameBasic(importPath, srcDir) return importPathToNameBasic(importPath, srcDir)
} }
} }
var stdImportPackage = map[string]string{} // "net/http" => "http"
func init() {
// Nothing in the standard library has a package name not
// matching its import base name.
for _, pkg := range stdlib {
if _, ok := stdImportPackage[pkg]; !ok {
stdImportPackage[pkg] = path.Base(pkg)
}
}
}
// Directory-scanning state.
var (
// scanGoRootOnce guards calling scanGoRoot (for $GOROOT)
scanGoRootOnce = &sync.Once{}
// scanGoPathOnce guards calling scanGoPath (for $GOPATH)
scanGoPathOnce = &sync.Once{}
dirScanMu sync.RWMutex
dirScan map[string]*pkg // abs dir path => *pkg
)
type pkg struct { type pkg struct {
importpath string // full pkg import path, e.g. "net/http" dir string // absolute file path to pkg directory ("/usr/lib/go/src/net/http")
dir string // absolute file path to pkg directory e.g. "/usr/lib/go/src/fmt" importPath string // full pkg import path ("net/http", "foo/bar/vendor/a/b")
importPathShort string // vendorless import path ("net/http", "a/b")
} }
var pkgIndexOnce = &sync.Once{} // byImportPathShortLength sorts by the short import path length, breaking ties on the
// import string itself.
type byImportPathShortLength []*pkg
func (s byImportPathShortLength) Len() int { return len(s) }
func (s byImportPathShortLength) Less(i, j int) bool {
vi, vj := s[i].importPathShort, s[j].importPathShort
return len(vi) < len(vj) || (len(vi) == len(vj) && vi < vj)
var pkgIndex struct {
sync.Mutex
m map[string][]pkg // shortname => []pkg, e.g "http" => "net/http"
} }
func (s byImportPathShortLength) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// gate is a semaphore for limiting concurrency. // gate is a semaphore for limiting concurrency.
type gate chan struct{} type gate chan struct{}
@ -212,12 +281,14 @@ func shouldTraverse(dir string, fi os.FileInfo) bool {
path := filepath.Join(dir, fi.Name()) path := filepath.Join(dir, fi.Name())
target, err := filepath.EvalSymlinks(path) target, err := filepath.EvalSymlinks(path)
if err != nil { if err != nil {
fmt.Fprint(os.Stderr, err) if !os.IsNotExist(err) {
fmt.Fprintln(os.Stderr, err)
}
return false return false
} }
ts, err := os.Stat(target) ts, err := os.Stat(target)
if err != nil { if err != nil {
fmt.Fprint(os.Stderr, err) fmt.Fprintln(os.Stderr, err)
return false return false
} }
if !ts.IsDir() { if !ts.IsDir() {
@ -242,46 +313,73 @@ func shouldTraverse(dir string, fi os.FileInfo) bool {
return true return true
} }
func loadPkgIndex() { var testHookScanDir = func(dir string) {}
pkgIndex.Lock()
pkgIndex.m = make(map[string][]pkg) func scanGoRoot() { scanGoDirs(true) }
pkgIndex.Unlock() func scanGoPath() { scanGoDirs(false) }
func scanGoDirs(goRoot bool) {
if Debug {
which := "$GOROOT"
if !goRoot {
which = "$GOPATH"
}
log.Printf("scanning " + which)
defer log.Printf("scanned " + which)
}
dirScanMu.Lock()
if dirScan == nil {
dirScan = make(map[string]*pkg)
}
dirScanMu.Unlock()
var wg sync.WaitGroup var wg sync.WaitGroup
for _, path := range build.Default.SrcDirs() { for _, path := range build.Default.SrcDirs() {
isGoroot := path == filepath.Join(build.Default.GOROOT, "src")
if isGoroot != goRoot {
continue
}
fsgate.enter() fsgate.enter()
testHookScanDir(path)
if Debug {
log.Printf("scanGoDir, open dir: %v\n", path)
}
f, err := os.Open(path) f, err := os.Open(path)
if err != nil { if err != nil {
fsgate.leave() fsgate.leave()
fmt.Fprint(os.Stderr, err) fmt.Fprintf(os.Stderr, "goimports: scanning directories: %v\n", err)
continue continue
} }
children, err := f.Readdir(-1) children, err := f.Readdir(-1)
f.Close() f.Close()
fsgate.leave() fsgate.leave()
if err != nil { if err != nil {
fmt.Fprint(os.Stderr, err) fmt.Fprintf(os.Stderr, "goimports: scanning directory entries: %v\n", err)
continue continue
} }
for _, child := range children { for _, child := range children {
if shouldTraverse(path, child) { if !shouldTraverse(path, child) {
wg.Add(1) continue
go func(path, name string) {
defer wg.Done()
loadPkg(&wg, path, name)
}(path, child.Name())
} }
wg.Add(1)
go func(path, name string) {
defer wg.Done()
scanDir(&wg, path, name)
}(path, child.Name())
} }
} }
wg.Wait() wg.Wait()
} }
func loadPkg(wg *sync.WaitGroup, root, pkgrelpath string) { func scanDir(wg *sync.WaitGroup, root, pkgrelpath string) {
importpath := filepath.ToSlash(pkgrelpath) importpath := filepath.ToSlash(pkgrelpath)
dir := filepath.Join(root, importpath) dir := filepath.Join(root, importpath)
fsgate.enter() fsgate.enter()
defer fsgate.leave() defer fsgate.leave()
if Debug {
log.Printf("scanning dir %s", dir)
}
pkgDir, err := os.Open(dir) pkgDir, err := os.Open(dir)
if err != nil { if err != nil {
return return
@ -309,60 +407,132 @@ func loadPkg(wg *sync.WaitGroup, root, pkgrelpath string) {
wg.Add(1) wg.Add(1)
go func(root, name string) { go func(root, name string) {
defer wg.Done() defer wg.Done()
loadPkg(wg, root, name) scanDir(wg, root, name)
}(root, filepath.Join(importpath, name)) }(root, filepath.Join(importpath, name))
} }
} }
if hasGo { if hasGo {
shortName := importPathToName(importpath, "") dirScanMu.Lock()
pkgIndex.Lock() dirScan[dir] = &pkg{
pkgIndex.m[shortName] = append(pkgIndex.m[shortName], pkg{ importPath: importpath,
importpath: importpath, importPathShort: vendorlessImportPath(importpath),
dir: dir, dir: dir,
}) }
pkgIndex.Unlock() dirScanMu.Unlock()
} }
} }
// loadExports returns a list exports for a package. // vendorlessImportPath returns the devendorized version of the provided import path.
var loadExports = loadExportsGoPath // e.g. "foo/bar/vendor/a/b" => "a/b"
func vendorlessImportPath(ipath string) string {
// Devendorize for use in import statement.
if i := strings.LastIndex(ipath, "/vendor/"); i >= 0 {
return ipath[i+len("/vendor/"):]
}
if strings.HasPrefix(ipath, "vendor/") {
return ipath[len("vendor/"):]
}
return ipath
}
func loadExportsGoPath(dir string) map[string]bool { // loadExports returns the set of exported symbols in the package at dir.
// It returns nil on error or if the package name in dir does not match expectPackage.
var loadExports func(expectPackage, dir string) map[string]bool = loadExportsGoPath
func loadExportsGoPath(expectPackage, dir string) map[string]bool {
if Debug {
log.Printf("loading exports in dir %s (seeking package %s)", dir, expectPackage)
}
exports := make(map[string]bool) exports := make(map[string]bool)
buildPkg, err := build.ImportDir(dir, 0)
if err != nil { ctx := build.Default
if strings.Contains(err.Error(), "no buildable Go source files in") {
return nil // ReadDir is like ioutil.ReadDir, but only returns *.go files
// and filters out _test.go files since they're not relevant
// and only slow things down.
ctx.ReadDir = func(dir string) (notTests []os.FileInfo, err error) {
all, err := ioutil.ReadDir(dir)
if err != nil {
return nil, err
} }
fmt.Fprintf(os.Stderr, "could not import %q: %v\n", dir, err) notTests = all[:0]
for _, fi := range all {
name := fi.Name()
if strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") {
notTests = append(notTests, fi)
}
}
return notTests, nil
}
files, err := ctx.ReadDir(dir)
if err != nil {
log.Print(err)
return nil return nil
} }
fset := token.NewFileSet() fset := token.NewFileSet()
for _, files := range [...][]string{buildPkg.GoFiles, buildPkg.CgoFiles} {
for _, file := range files { for _, fi := range files {
f, err := parser.ParseFile(fset, filepath.Join(dir, file), nil, 0) match, err := ctx.MatchFile(dir, fi.Name())
if err != nil { if err != nil || !match {
fmt.Fprintf(os.Stderr, "could not parse %q: %v\n", file, err) continue
continue }
fullFile := filepath.Join(dir, fi.Name())
f, err := parser.ParseFile(fset, fullFile, nil, 0)
if err != nil {
if Debug {
log.Printf("Parsing %s: %v", fullFile, err)
} }
for name := range f.Scope.Objects { return nil
if ast.IsExported(name) { }
exports[name] = true pkgName := f.Name.Name
} if pkgName == "documentation" {
// Special case from go/build.ImportDir, not
// handled by ctx.MatchFile.
continue
}
if pkgName != expectPackage {
if Debug {
log.Printf("scan of dir %v is not expected package %v (actually %v)", dir, expectPackage, pkgName)
}
return nil
}
for name := range f.Scope.Objects {
if ast.IsExported(name) {
exports[name] = true
} }
} }
} }
if Debug {
exportList := make([]string, 0, len(exports))
for k := range exports {
exportList = append(exportList, k)
}
sort.Strings(exportList)
log.Printf("scanned dir %v (package %v): exports = %v", dir, expectPackage, strings.Join(exportList, ", "))
}
return exports return exports
} }
// findImport searches for a package with the given symbols. // findImport searches for a package with the given symbols.
// If no package is found, findImport returns "". // If no package is found, findImport returns ("", false, nil)
// Declared as a variable rather than a function so goimports can be easily //
// extended by adding a file with an init function. // This is declared as a variable rather than a function so goimports
var findImport = findImportGoPath // can be easily extended by adding a file with an init function.
//
// The rename value tells goimports whether to use the package name as
// a local qualifier in an import. For example, if findImports("pkg",
// "X") returns ("foo/bar", rename=true), then goimports adds the
// import line:
// import pkg "foo/bar"
// to satisfy uses of pkg.X in the file.
var findImport func(pkgName string, symbols map[string]bool, filename string) (foundPkg string, rename bool, err error) = findImportGoPath
func findImportGoPath(pkgName string, symbols map[string]bool, filename string) (string, bool, error) { // findImportGoPath is the normal implementation of findImport. (Some
// companies have their own internally)
func findImportGoPath(pkgName string, symbols map[string]bool, filename string) (foundPkg string, rename bool, err error) {
// Fast path for the standard library. // Fast path for the standard library.
// In the common case we hopefully never have to scan the GOPATH, which can // In the common case we hopefully never have to scan the GOPATH, which can
// be slow with moving disks. // be slow with moving disks.
@ -385,57 +555,103 @@ func findImportGoPath(pkgName string, symbols map[string]bool, filename string)
// in the current Go file. Return rename=true when the other Go files // in the current Go file. Return rename=true when the other Go files
// use a renamed package that's also used in the current file. // use a renamed package that's also used in the current file.
pkgIndexOnce.Do(loadPkgIndex) scanGoRootOnce.Do(scanGoRoot)
if !strings.HasPrefix(filename, build.Default.GOROOT) {
scanGoPathOnce.Do(scanGoPath)
}
// Collect exports for packages with matching names. var candidates []*pkg
var (
wg sync.WaitGroup for _, pkg := range dirScan {
mu sync.Mutex if !strings.Contains(lastTwoComponents(pkg.importPathShort), pkgName) {
shortest string // Speed optimization to minimize disk I/O:
) // the last two components on disk must contain the
pkgIndex.Lock() // package name somewhere.
for _, pkg := range pkgIndex.m[pkgName] { //
// This permits mismatch naming like directory
// "go-foo" being package "foo", or "pkg.v3" being "pkg",
// or directory "google.golang.org/api/cloudbilling/v1"
// being package "cloudbilling", but doesn't
// permit a directory "foo" to be package
// "bar", which is strongly discouraged
// anyway. There's no reason goimports needs
// to be slow just to accomodate that.
continue
}
if !canUse(filename, pkg.dir) { if !canUse(filename, pkg.dir) {
continue continue
} }
wg.Add(1) candidates = append(candidates, pkg)
go func(importpath, dir string) { }
defer wg.Done()
exports := loadExports(dir) sort.Sort(byImportPathShortLength(candidates))
if exports == nil { if Debug {
for i, pkg := range candidates {
log.Printf("%s candidate %d/%d: %v", pkgName, i+1, len(candidates), pkg.importPathShort)
}
}
// Collect exports for packages with matching names.
done := make(chan struct{}) // closed when we find the answer
defer close(done)
rescv := make([]chan *pkg, len(candidates))
for i := range candidates {
rescv[i] = make(chan *pkg, 1)
}
const maxConcurrentPackageImport = 4
loadExportsSem := make(chan struct{}, maxConcurrentPackageImport)
go func() {
for i, pkg := range candidates {
select {
case loadExportsSem <- struct{}{}:
case <-done:
return return
} }
// If it doesn't have the right symbols, stop. pkg := pkg
for symbol := range symbols { resc := rescv[i]
if !exports[symbol] { go func() {
return defer func() { <-loadExportsSem }()
exports := loadExports(pkgName, pkg.dir)
// If it doesn't have the right
// symbols, send nil to mean no match.
for symbol := range symbols {
if !exports[symbol] {
pkg = nil
break
}
} }
} resc <- pkg
}()
// Devendorize for use in import statement. }
if i := strings.LastIndex(importpath, "/vendor/"); i >= 0 { }()
importpath = importpath[i+len("/vendor/"):] for _, resc := range rescv {
} else if strings.HasPrefix(importpath, "vendor/") { pkg := <-resc
importpath = importpath[len("vendor/"):] if pkg == nil {
} continue
}
// Save as the answer. // If the package name in the source doesn't match the import path's base,
// If there are multiple candidates, the shortest wins, // return true so the rewriter adds a name (import foo "github.com/bar/go-foo")
// to prefer "bytes" over "github.com/foo/bytes". needsRename := path.Base(pkg.importPath) != pkgName
mu.Lock() return pkg.importPathShort, needsRename, nil
if shortest == "" || len(importpath) < len(shortest) || len(importpath) == len(shortest) && importpath < shortest {
shortest = importpath
}
mu.Unlock()
}(pkg.importpath, pkg.dir)
} }
pkgIndex.Unlock() return "", false, nil
wg.Wait()
return shortest, false, nil
} }
// canUse reports whether the package in dir is usable from filename,
// respecting the Go "internal" and "vendor" visibility rules.
func canUse(filename, dir string) bool { func canUse(filename, dir string) bool {
// Fast path check, before any allocations. If it doesn't contain vendor
// or internal, it's not tricky:
// Note that this can false-negative on directories like "notinternal",
// but we check it correctly below. This is just a fast path.
if !strings.Contains(dir, "vendor") && !strings.Contains(dir, "internal") {
return true
}
dirSlash := filepath.ToSlash(dir) dirSlash := filepath.ToSlash(dir)
if !strings.Contains(dirSlash, "/vendor/") && !strings.Contains(dirSlash, "/internal/") && !strings.HasSuffix(dirSlash, "/internal") { if !strings.Contains(dirSlash, "/vendor/") && !strings.Contains(dirSlash, "/internal/") && !strings.HasSuffix(dirSlash, "/internal") {
return true return true
@ -461,6 +677,21 @@ func canUse(filename, dir string) bool {
return !strings.Contains(relSlash, "/vendor/") && !strings.Contains(relSlash, "/internal/") && !strings.HasSuffix(relSlash, "/internal") return !strings.Contains(relSlash, "/vendor/") && !strings.Contains(relSlash, "/internal/") && !strings.HasSuffix(relSlash, "/internal")
} }
// lastTwoComponents returns at most the last two path components
// of v, using either / or \ as the path separator.
func lastTwoComponents(v string) string {
nslash := 0
for i := len(v) - 1; i >= 0; i-- {
if v[i] == '/' || v[i] == '\\' {
nslash++
if nslash == 2 {
return v[i:]
}
}
}
return v
}
type visitFn func(node ast.Node) ast.Visitor type visitFn func(node ast.Node) ast.Visitor
func (fn visitFn) Visit(node ast.Node) ast.Visitor { func (fn visitFn) Visit(node ast.Node) ast.Visitor {

390
imports/fix_test.go
View File

@ -857,22 +857,17 @@ func TestImportSymlinks(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
pkgIndexOnce = &sync.Once{} withEmptyGoPath(func() {
oldGOPATH := build.Default.GOPATH build.Default.GOPATH = newGoPath
build.Default.GOPATH = newGoPath
defer func() {
build.Default.GOPATH = oldGOPATH
visitedSymlinks.m = nil
}()
input := `package p input := `package p
var ( var (
_ = fmt.Print _ = fmt.Print
_ = mypkg.Foo _ = mypkg.Foo
) )
` `
output := `package p output := `package p
import ( import (
"fmt" "fmt"
@ -884,13 +879,14 @@ var (
_ = mypkg.Foo _ = mypkg.Foo
) )
` `
buf, err := Process(newGoPath+"/src/myotherpkg/toformat.go", []byte(input), &Options{}) buf, err := Process(newGoPath+"/src/myotherpkg/toformat.go", []byte(input), &Options{})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if got := string(buf); got != output { if got := string(buf); got != output {
t.Fatalf("results differ\nGOT:\n%s\nWANT:\n%s\n", got, output) t.Fatalf("results differ\nGOT:\n%s\nWANT:\n%s\n", got, output)
} }
})
} }
// Test for correctly identifying the name of a vendored package when it // Test for correctly identifying the name of a vendored package when it
@ -902,30 +898,12 @@ func TestFixImportsVendorPackage(t *testing.T) {
if _, err := os.Stat(filepath.Join(runtime.GOROOT(), "src/vendor")); err != nil { if _, err := os.Stat(filepath.Join(runtime.GOROOT(), "src/vendor")); err != nil {
t.Skip(err) t.Skip(err)
} }
testConfig{
newGoPath, err := ioutil.TempDir("", "vendortest") gopathFiles: map[string]string{
if err != nil { "mypkg.com/outpkg/vendor/mypkg.com/mypkg.v1/f.go": "package mypkg\nvar Foo = 123\n",
t.Fatal(err) },
} }.test(t, func(t *goimportTest) {
defer os.RemoveAll(newGoPath) input := `package p
vendoredPath := newGoPath + "/src/mypkg.com/outpkg/vendor/mypkg.com/mypkg.v1"
if err := os.MkdirAll(vendoredPath, 0755); err != nil {
t.Fatal(err)
}
pkgIndexOnce = &sync.Once{}
oldGOPATH := build.Default.GOPATH
build.Default.GOPATH = newGoPath
defer func() {
build.Default.GOPATH = oldGOPATH
}()
if err := ioutil.WriteFile(vendoredPath+"/f.go", []byte("package mypkg\nvar Foo = 123\n"), 0666); err != nil {
t.Fatal(err)
}
input := `package p
import ( import (
"fmt" "fmt"
@ -938,13 +916,14 @@ var (
_ = mypkg.Foo _ = mypkg.Foo
) )
` `
buf, err := Process(newGoPath+"/src/mypkg.com/outpkg/toformat.go", []byte(input), &Options{}) buf, err := Process(filepath.Join(t.gopath, "src/mypkg.com/outpkg/toformat.go"), []byte(input), &Options{})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if got := string(buf); got != input { if got := string(buf); got != input {
t.Fatalf("results differ\nGOT:\n%s\nWANT:\n%s\n", got, input) t.Fatalf("results differ\nGOT:\n%s\nWANT:\n%s\n", got, input)
} }
})
} }
func TestFindImportGoPath(t *testing.T) { func TestFindImportGoPath(t *testing.T) {
@ -954,66 +933,69 @@ func TestFindImportGoPath(t *testing.T) {
} }
defer os.RemoveAll(goroot) defer os.RemoveAll(goroot)
pkgIndexOnce = &sync.Once{}
origStdlib := stdlib origStdlib := stdlib
defer func() { defer func() {
stdlib = origStdlib stdlib = origStdlib
}() }()
stdlib = nil stdlib = nil
// Test against imaginary bits/bytes package in std lib withEmptyGoPath(func() {
bytesDir := filepath.Join(goroot, "src", "pkg", "bits", "bytes") // Test against imaginary bits/bytes package in std lib
for _, tag := range build.Default.ReleaseTags { bytesDir := filepath.Join(goroot, "src", "pkg", "bits", "bytes")
// Go 1.4 rearranged the GOROOT tree to remove the "pkg" path component. for _, tag := range build.Default.ReleaseTags {
if tag == "go1.4" { // Go 1.4 rearranged the GOROOT tree to remove the "pkg" path component.
bytesDir = filepath.Join(goroot, "src", "bits", "bytes") if tag == "go1.4" {
bytesDir = filepath.Join(goroot, "src", "bits", "bytes")
}
} }
} if err := os.MkdirAll(bytesDir, 0755); err != nil {
if err := os.MkdirAll(bytesDir, 0755); err != nil { t.Fatal(err)
t.Fatal(err) }
} bytesSrcPath := filepath.Join(bytesDir, "bytes.go")
bytesSrcPath := filepath.Join(bytesDir, "bytes.go") bytesPkgPath := "bits/bytes"
bytesPkgPath := "bits/bytes" bytesSrc := []byte(`package bytes
bytesSrc := []byte(`package bytes
type Buffer2 struct {} type Buffer2 struct {}
`) `)
if err := ioutil.WriteFile(bytesSrcPath, bytesSrc, 0775); err != nil { if err := ioutil.WriteFile(bytesSrcPath, bytesSrc, 0775); err != nil {
t.Fatal(err) t.Fatal(err)
} }
oldGOROOT := build.Default.GOROOT build.Default.GOROOT = goroot
oldGOPATH := build.Default.GOPATH
build.Default.GOROOT = goroot
build.Default.GOPATH = ""
defer func() {
build.Default.GOROOT = oldGOROOT
build.Default.GOPATH = oldGOPATH
}()
got, rename, err := findImportGoPath("bytes", map[string]bool{"Buffer2": true}, "x.go") got, rename, err := findImportGoPath("bytes", map[string]bool{"Buffer2": true}, "x.go")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if got != bytesPkgPath || rename { if got != bytesPkgPath || rename {
t.Errorf(`findImportGoPath("bytes", Buffer2 ...)=%q, %t, want "%s", false`, got, rename, bytesPkgPath) t.Errorf(`findImportGoPath("bytes", Buffer2 ...)=%q, %t, want "%s", false`, got, rename, bytesPkgPath)
} }
got, rename, err = findImportGoPath("bytes", map[string]bool{"Missing": true}, "x.go") got, rename, err = findImportGoPath("bytes", map[string]bool{"Missing": true}, "x.go")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if got != "" || rename { if got != "" || rename {
t.Errorf(`findImportGoPath("bytes", Missing ...)=%q, %t, want "", false`, got, rename) t.Errorf(`findImportGoPath("bytes", Missing ...)=%q, %t, want "", false`, got, rename)
} }
})
} }
func withEmptyGoPath(fn func()) { func withEmptyGoPath(fn func()) {
pkgIndexOnce = &sync.Once{} dirScanMu.Lock()
scanGoRootOnce = &sync.Once{}
scanGoPathOnce = &sync.Once{}
dirScan = nil
dirScanMu.Unlock()
oldGOPATH := build.Default.GOPATH oldGOPATH := build.Default.GOPATH
oldGOROOT := build.Default.GOROOT
build.Default.GOPATH = "" build.Default.GOPATH = ""
visitedSymlinks.m = nil
testHookScanDir = func(string) {}
defer func() { defer func() {
testHookScanDir = func(string) {}
build.Default.GOPATH = oldGOPATH build.Default.GOPATH = oldGOPATH
build.Default.GOROOT = oldGOROOT
}() }()
fn() fn()
} }
@ -1033,7 +1015,7 @@ func TestFindImportInternal(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if got != "internal/race" || rename { if got != "internal/race" || rename {
t.Errorf(`findImportGoPath("race", Acquire ...)=%q, %t, want "internal/race", false`, got, rename) t.Errorf(`findImportGoPath("race", Acquire ...) = %q, %t; want "internal/race", false`, got, rename)
} }
// should not be able to use internal from outside that tree // should not be able to use internal from outside that tree
@ -1090,65 +1072,45 @@ func TestFindImportRandRead(t *testing.T) {
} }
func TestFindImportVendor(t *testing.T) { func TestFindImportVendor(t *testing.T) {
pkgIndexOnce = &sync.Once{} testConfig{
oldGOPATH := build.Default.GOPATH gorootFiles: map[string]string{
build.Default.GOPATH = "" "vendor/golang.org/x/net/http2/hpack/huffman.go": "package hpack\nfunc HuffmanDecode() { }\n",
defer func() { },
build.Default.GOPATH = oldGOPATH }.test(t, func(t *goimportTest) {
}() got, rename, err := findImportGoPath("hpack", map[string]bool{"HuffmanDecode": true}, filepath.Join(t.goroot, "src/math/x.go"))
if err != nil {
_, err := os.Stat(filepath.Join(runtime.GOROOT(), "src/vendor")) t.Fatal(err)
if err != nil { }
t.Skip(err) want := "golang.org/x/net/http2/hpack"
} if got != want || rename {
t.Errorf(`findImportGoPath("hpack", HuffmanDecode ...) = %q, %t; want %q, false`, got, rename, want)
got, rename, err := findImportGoPath("hpack", map[string]bool{"HuffmanDecode": true}, filepath.Join(runtime.GOROOT(), "src/math/x.go")) }
if err != nil { })
t.Fatal(err)
}
want := "golang.org/x/net/http2/hpack"
// Pre-1.7, we temporarily had this package under "internal" - adjust want accordingly.
_, err = os.Stat(filepath.Join(runtime.GOROOT(), "src/vendor", want))
if err != nil {
want = filepath.Join("internal", want)
}
if got != want || rename {
t.Errorf(`findImportGoPath("hpack", HuffmanDecode ...)=%q, %t, want %q, false`, got, rename, want)
}
// should not be able to use vendor from outside that tree
got, rename, err = findImportGoPath("hpack", map[string]bool{"HuffmanDecode": true}, filepath.Join(runtime.GOROOT(), "x.go"))
if err != nil {
t.Fatal(err)
}
if got != "" || rename {
t.Errorf(`findImportGoPath("hpack", HuffmanDecode ...)=%q, %t, want "", false`, got, rename)
}
} }
func TestProcessVendor(t *testing.T) { func TestProcessVendor(t *testing.T) {
pkgIndexOnce = &sync.Once{} withEmptyGoPath(func() {
oldGOPATH := build.Default.GOPATH _, err := os.Stat(filepath.Join(runtime.GOROOT(), "src/vendor"))
build.Default.GOPATH = "" if err != nil {
defer func() { t.Skip(err)
build.Default.GOPATH = oldGOPATH }
}()
_, err := os.Stat(filepath.Join(runtime.GOROOT(), "src/vendor")) target := filepath.Join(runtime.GOROOT(), "src/math/x.go")
if err != nil { out, err := Process(target, []byte("package http\nimport \"bytes\"\nfunc f() { strings.NewReader(); hpack.HuffmanDecode() }\n"), nil)
t.Skip(err)
}
target := filepath.Join(runtime.GOROOT(), "src/math/x.go") if err != nil {
out, err := Process(target, []byte("package http\nimport \"bytes\"\nfunc f() { strings.NewReader(); hpack.HuffmanDecode() }\n"), nil) t.Fatal(err)
}
if err != nil { want := "golang_org/x/net/http2/hpack"
t.Fatal(err) if _, err := os.Stat(filepath.Join(runtime.GOROOT(), "src/vendor", want)); os.IsNotExist(err) {
} want = "golang.org/x/net/http2/hpack"
want := "golang.org/x/net/http2/hpack" }
if !bytes.Contains(out, []byte(want)) {
t.Fatalf("Process(%q) did not add expected hpack import:\n%s", target, out) if !bytes.Contains(out, []byte(want)) {
} t.Fatalf("Process(%q) did not add expected hpack import %q; got:\n%s", target, want, out)
}
})
} }
func TestFindImportStdlib(t *testing.T) { func TestFindImportStdlib(t *testing.T) {
@ -1174,6 +1136,150 @@ func TestFindImportStdlib(t *testing.T) {
} }
} }
type testConfig struct {
// gorootFiles optionally specifies the complete contents of GOROOT to use,
// If nil, the normal current $GOROOT is used.
gorootFiles map[string]string // paths relative to $GOROOT/src to contents
// gopathFiles is like gorootFiles, but for $GOPATH.
// If nil, there is no GOPATH, though.
gopathFiles map[string]string // paths relative to $GOPATH/src to contents
}
func mustTempDir(t *testing.T, prefix string) string {
dir, err := ioutil.TempDir("", prefix)
if err != nil {
t.Fatal(err)
}
return dir
}
func mapToDir(destDir string, files map[string]string) error {
for path, contents := range files {
file := filepath.Join(destDir, "src", path)
if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
return err
}
if err := ioutil.WriteFile(file, []byte(contents), 0644); err != nil {
return err
}
}
return nil
}
func (c testConfig) test(t *testing.T, fn func(*goimportTest)) {
var goroot string
var gopath string
if c.gorootFiles != nil {
goroot = mustTempDir(t, "goroot-")
defer os.RemoveAll(goroot)
if err := mapToDir(goroot, c.gorootFiles); err != nil {
t.Fatal(err)
}
}
if c.gopathFiles != nil {
gopath = mustTempDir(t, "gopath-")
defer os.RemoveAll(gopath)
if err := mapToDir(gopath, c.gopathFiles); err != nil {
t.Fatal(err)
}
}
withEmptyGoPath(func() {
if goroot != "" {
build.Default.GOROOT = goroot
}
build.Default.GOPATH = gopath
it := &goimportTest{
T: t,
goroot: build.Default.GOROOT,
gopath: gopath,
ctx: &build.Default,
}
fn(it)
})
}
type goimportTest struct {
*testing.T
ctx *build.Context
goroot string
gopath string
}
// Tests that added imports are renamed when the import path's base doesn't
// match its package name. For example, we want to generate:
//
// import cloudbilling "google.golang.org/api/cloudbilling/v1"
func TestRenameWhenPackageNameMismatch(t *testing.T) {
testConfig{
gopathFiles: map[string]string{
"foo/bar/v1/x.go": "package bar \n const X = 1",
},
}.test(t, func(t *goimportTest) {
buf, err := Process(t.gopath+"/src/test/t.go", []byte("package main \n const Y = bar.X"), &Options{})
if err != nil {
t.Fatal(err)
}
const want = `package main
import bar "foo/bar/v1"
const Y = bar.X
`
if string(buf) != want {
t.Errorf("Got:\n%s\nWant:\n%s", buf, want)
}
})
}
// Tests that running goimport on files in GOROOT (for people hacking
// on Go itself) don't cause the GOPATH to be scanned (which might be
// much bigger).
func TestOptimizationWhenInGoroot(t *testing.T) {
testConfig{
gopathFiles: map[string]string{
"foo/foo.go": "package foo\nconst X = 1\n",
},
}.test(t, func(t *goimportTest) {
testHookScanDir = func(dir string) {
if dir != filepath.Join(build.Default.GOROOT, "src") {
t.Errorf("unexpected dir scan of %s", dir)
}
}
const in = "package foo\n\nconst Y = bar.X\n"
buf, err := Process(t.goroot+"/src/foo/foo.go", []byte(in), nil)
if err != nil {
t.Fatal(err)
}
if string(buf) != in {
t.Errorf("got:\n%q\nwant unchanged:\n%q\n", in, buf)
}
})
}
// Tests that "package documentation" files are ignored.
func TestIgnoreDocumentationPackage(t *testing.T) {
testConfig{
gopathFiles: map[string]string{
"foo/foo.go": "package foo\nconst X = 1\n",
"foo/doc.go": "package documentation \n // just to confuse things\n",
},
}.test(t, func(t *goimportTest) {
const in = "package x\n\nconst Y = foo.X\n"
const want = "package x\n\nimport \"foo\"\n\nconst Y = foo.X\n"
buf, err := Process(t.gopath+"/src/x/x.go", []byte(in), nil)
if err != nil {
t.Fatal(err)
}
if string(buf) != want {
t.Errorf("wrong output.\ngot:\n%q\nwant:\n%q\n", in, want)
}
})
}
func strSet(ss []string) map[string]bool { func strSet(ss []string) map[string]bool {
m := make(map[string]bool) m := make(map[string]bool)
for _, s := range ss { for _, s := range ss {

4
refactor/rename/mvpkg_test.go
View File

@ -12,6 +12,7 @@ import (
"path/filepath" "path/filepath"
"reflect" "reflect"
"regexp" "regexp"
"runtime"
"strings" "strings"
"testing" "testing"
@ -113,6 +114,9 @@ var _ foo.T
} }
func TestMoves(t *testing.T) { func TestMoves(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("broken on Windows; see golang.org/issue/16384")
}
tests := []struct { tests := []struct {
ctxt *build.Context ctxt *build.Context
from, to string from, to string