cybercyst/go
1
0
Fork 0
mirror of https://github.com/golang/go.git synced 2025-05-05 15:43:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

go/packages: add name= query

Browse Source 
Add an implementation of name= for go list. It will be used to
implement goimports and godoc-like lookups by package name.

Imported a copy of the semver package from the stdlib to do version
comparison, and tweaked the gopathwalk API to include a hint about what
kind of source directory is being traversed.

Note that the tests, despite my best efforts, are not hermetic: go list
insists on doing version lookups in situations where it seems to me like
it shouldn't need to.

I think this implementation is ready for serious use. The one thing I'm
nervous about is that it currently does a substring match when looking
for a package name, so if you look up a package named "a" you will get
a huge number of results. This matches goimports' behavior but I don't
know if it's suitable for general use.

Change-Id: I2b7f823b74571fe30d3bd9c7dfafb4e6a40df5d3
Reviewed-on: https://go-review.googlesource.com/c/138878
Run-TryBot: Heschi Kreinick <heschi@google.com>
Reviewed-by: Michael Matloob <matloob@golang.org>
This commit is contained in:
Heschi Kreinick 2018-09-27 18:21:00 -04:00
parent bf9c22dffd
commit 63d31665e3
43 changed files with 1056 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

301
go/packages/golist.go
View File

@ -8,11 +8,16 @@ import (
"bytes"
"encoding/json"
"fmt"
"golang.org/x/tools/internal/gopathwalk"
"golang.org/x/tools/internal/semver"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"sync"
)
// A goTooOldError reports that the go command
@ -27,6 +32,7 @@ type goTooOldError struct {
func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) {
// Determine files requested in contains patterns
var containFiles []string
var packagesNamed []string
restPatterns := make([]string, 0, len(patterns))
// Extract file= and other [querytype]= patterns. Report an error if querytype
// doesn't exist.
@ -41,7 +47,9 @@ extractQueries:
case "file":
containFiles = append(containFiles, value)
case "pattern":
restPatterns = append(containFiles, value)
restPatterns = append(restPatterns, value)
case "name":
packagesNamed = append(packagesNamed, value)
case "": // not a reserved query
restPatterns = append(restPatterns, pattern)
default:
@ -69,15 +77,14 @@ extractQueries:
}
}
containFiles = absJoin(cfg.Dir, containFiles)
patterns = restPatterns
// TODO(matloob): Remove the definition of listfunc and just use golistPackages once go1.12 is released.
var listfunc driver
listfunc = func(cfg *Config, words ...string) (*driverResponse, error) {
response, err := golistDriverCurrent(cfg, patterns...)
response, err := golistDriverCurrent(cfg, words...)
if _, ok := err.(goTooOldError); ok {
listfunc = golistDriverFallback
return listfunc(cfg, patterns...)
return listfunc(cfg, words...)
}
listfunc = golistDriverCurrent
return response, err
@ -87,8 +94,8 @@ extractQueries:
var err error
// see if we have any patterns to pass through to go list.
if len(patterns) > 0 {
response, err = listfunc(cfg, patterns...)
if len(restPatterns) > 0 {
response, err = listfunc(cfg, restPatterns...)
if err != nil {
return nil, err
}
@ -96,18 +103,44 @@ extractQueries:
response = &driverResponse{}
}
// Run go list for contains: patterns.
if len(containFiles) == 0 && len(packagesNamed) == 0 {
return response, nil
}
seenPkgs := make(map[string]*Package) // for deduplication. different containing queries could produce same packages
if len(containFiles) > 0 {
for _, pkg := range response.Packages {
seenPkgs[pkg.ID] = pkg
}
addPkg := func(p *Package) {
if _, ok := seenPkgs[p.ID]; ok {
return
}
for _, f := range containFiles {
seenPkgs[p.ID] = p
response.Packages = append(response.Packages, p)
}
containsResults, err := runContainsQueries(cfg, listfunc, addPkg, containFiles)
if err != nil {
return nil, err
}
response.Roots = append(response.Roots, containsResults...)
namedResults, err := runNamedQueries(cfg, listfunc, addPkg, packagesNamed)
if err != nil {
return nil, err
}
response.Roots = append(response.Roots, namedResults...)
return response, nil
}
func runContainsQueries(cfg *Config, driver driver, addPkg func(*Package), queries []string) ([]string, error) {
var results []string
for _, query := range queries {
// TODO(matloob): Do only one query per directory.
fdir := filepath.Dir(f)
fdir := filepath.Dir(query)
cfg.Dir = fdir
dirResponse, err := listfunc(cfg, ".")
dirResponse, err := driver(cfg, ".")
if err != nil {
return nil, err
}
@ -120,24 +153,241 @@ extractQueries:
// We don't bother to filter packages that will be dropped by the changes of roots,
// that will happen anyway during graph construction outside this function.
// Over-reporting packages is not a problem.
if _, ok := seenPkgs[pkg.ID]; !ok {
// it is a new package, just add it
seenPkgs[pkg.ID] = pkg
response.Packages = append(response.Packages, pkg)
}
addPkg(pkg)
// if the package was not a root one, it cannot have the file
if !isRoot[pkg.ID] {
continue
}
for _, pkgFile := range pkg.GoFiles {
if filepath.Base(f) == filepath.Base(pkgFile) {
response.Roots = append(response.Roots, pkg.ID)
if filepath.Base(query) == filepath.Base(pkgFile) {
results = append(results, pkg.ID)
break
}
}
}
}
return response, nil
return results, nil
}
// modCacheRegexp splits a path in a module cache into module, module version, and package.
var modCacheRegexp = regexp.MustCompile(`(.*)@([^/\\]*)(.*)`)
func runNamedQueries(cfg *Config, driver driver, addPkg func(*Package), queries []string) ([]string, error) {
// Determine which directories are relevant to scan.
roots, modulesEnabled, err := roots(cfg)
if err != nil {
return nil, err
}
// Scan the selected directories. Simple matches, from GOPATH/GOROOT
// or the local module, can simply be "go list"ed. Matches from the
// module cache need special treatment.
var matchesMu sync.Mutex
var simpleMatches, modCacheMatches []string
add := func(root gopathwalk.Root, dir string) {
// Walk calls this concurrently; protect the result slices.
matchesMu.Lock()
defer matchesMu.Unlock()
path := dir[len(root.Path)+1:]
if pathMatchesQueries(path, queries) {
switch root.Type {
case gopathwalk.RootModuleCache:
modCacheMatches = append(modCacheMatches, path)
case gopathwalk.RootCurrentModule:
// We'd need to read go.mod to find the full
// import path. Relative's easier.
rel, err := filepath.Rel(cfg.Dir, dir)
if err != nil {
// This ought to be impossible, since
// we found dir in the current module.
panic(err)
}
simpleMatches = append(simpleMatches, "./"+rel)
case gopathwalk.RootGOPATH, gopathwalk.RootGOROOT:
simpleMatches = append(simpleMatches, path)
}
}
}
gopathwalk.Walk(roots, add, gopathwalk.Options{ModulesEnabled: modulesEnabled})
var results []string
addResponse := func(r *driverResponse) {
for _, pkg := range r.Packages {
addPkg(pkg)
for _, name := range queries {
if pkg.Name == name {
results = append(results, pkg.ID)
break
}
}
}
}
if len(simpleMatches) != 0 {
resp, err := driver(cfg, simpleMatches...)
if err != nil {
return nil, err
}
addResponse(resp)
}
// Module cache matches are tricky. We want to avoid downloading new
// versions of things, so we need to use the ones present in the cache.
// go list doesn't accept version specifiers, so we have to write out a
// temporary module, and do the list in that module.
if len(modCacheMatches) != 0 {
// Collect all the matches, deduplicating by major version
// and preferring the newest.
type modInfo struct {
mod string
major string
}
mods := make(map[modInfo]string)
var imports []string
for _, modPath := range modCacheMatches {
matches := modCacheRegexp.FindStringSubmatch(modPath)
mod, ver := filepath.ToSlash(matches[1]), matches[2]
importPath := filepath.ToSlash(filepath.Join(matches[1], matches[3]))
major := semver.Major(ver)
if prevVer, ok := mods[modInfo{mod, major}]; !ok || semver.Compare(ver, prevVer) > 0 {
mods[modInfo{mod, major}] = ver
}
imports = append(imports, importPath)
}
// Build the temporary module.
var gomod bytes.Buffer
gomod.WriteString("module modquery\nrequire (\n")
for mod, version := range mods {
gomod.WriteString("\t" + mod.mod + " " + version + "\n")
}
gomod.WriteString(")\n")
tmpCfg := *cfg
var err error
tmpCfg.Dir, err = ioutil.TempDir("", "gopackages-modquery")
if err != nil {
return nil, err
}
defer os.RemoveAll(tmpCfg.Dir)
if err := ioutil.WriteFile(filepath.Join(tmpCfg.Dir, "go.mod"), gomod.Bytes(), 0777); err != nil {
return nil, fmt.Errorf("writing go.mod for module cache query: %v", err)
}
// Run the query, using the import paths calculated from the matches above.
resp, err := driver(&tmpCfg, imports...)
if err != nil {
return nil, fmt.Errorf("querying module cache matches: %v", err)
}
addResponse(resp)
}
return results, nil
}
// roots selects the appropriate paths to walk based on the passed-in configuration,
// particularly the environment and the presence of a go.mod in cfg.Dir's parents.
func roots(cfg *Config) ([]gopathwalk.Root, bool, error) {
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
cmd := exec.CommandContext(cfg.Context, "go", "env", "GOROOT", "GOPATH", "GOMOD")
cmd.Stdout = stdout
cmd.Stderr = stderr
cmd.Dir = cfg.Dir
cmd.Env = cfg.Env
if err := cmd.Run(); err != nil {
return nil, false, fmt.Errorf("running go env: %v (stderr: %q)", err, stderr.Bytes())
}
fields := strings.Split(string(stdout.Bytes()), "\n")
if len(fields) != 4 || len(fields[3]) != 0 {
return nil, false, fmt.Errorf("go env returned unexpected output: %q (stderr: %q)", stdout.Bytes(), stderr.Bytes())
}
goroot, gopath, gomod := fields[0], filepath.SplitList(fields[1]), fields[2]
modsEnabled := gomod != ""
var roots []gopathwalk.Root
// Always add GOROOT.
roots = append(roots, gopathwalk.Root{filepath.Join(goroot, "/src"), gopathwalk.RootGOROOT})
// If modules are enabled, scan the module dir.
if modsEnabled {
roots = append(roots, gopathwalk.Root{filepath.Dir(gomod), gopathwalk.RootCurrentModule})
}
// Add either GOPATH/src or GOPATH/pkg/mod, depending on module mode.
for _, p := range gopath {
if modsEnabled {
roots = append(roots, gopathwalk.Root{filepath.Join(p, "/pkg/mod"), gopathwalk.RootModuleCache})
} else {
roots = append(roots, gopathwalk.Root{filepath.Join(p, "/src"), gopathwalk.RootGOPATH})
}
}
return roots, modsEnabled, nil
}
// These functions were copied from goimports. See further documentation there.
// pathMatchesQueries is adapted from pkgIsCandidate.
// TODO: is it reasonable to do Contains here, rather than an exact match on a path component?
func pathMatchesQueries(path string, queries []string) bool {
lastTwo := lastTwoComponents(path)
for _, query := range queries {
if strings.Contains(lastTwo, query) {
return true
}
if hasHyphenOrUpperASCII(lastTwo) && !hasHyphenOrUpperASCII(query) {
lastTwo = lowerASCIIAndRemoveHyphen(lastTwo)
if strings.Contains(lastTwo, query) {
return true
}
}
}
return false
}
// lastTwoComponents returns at most the last two path components
// of v, using either / or \ as the path separator.
func lastTwoComponents(v string) string {
nslash := 0
for i := len(v) - 1; i >= 0; i-- {
if v[i] == '/' || v[i] == '\\' {
nslash++
if nslash == 2 {
return v[i:]
}
}
}
return v
}
func hasHyphenOrUpperASCII(s string) bool {
for i := 0; i < len(s); i++ {
b := s[i]
if b == '-' || ('A' <= b && b <= 'Z') {
return true
}
}
return false
}
func lowerASCIIAndRemoveHyphen(s string) (ret string) {
buf := make([]byte, 0, len(s))
for i := 0; i < len(s); i++ {
b := s[i]
switch {
case b == '-':
continue
case 'A' <= b && b <= 'Z':
buf = append(buf, b+('a'-'A'))
default:
buf = append(buf, b)
}
}
return string(buf)
}
// Fields must match go list;
@ -325,12 +575,13 @@ func golistargs(cfg *Config, words []string) []string {
// golist returns the JSON-encoded result of a "go list args..." query.
func golist(cfg *Config, args []string) (*bytes.Buffer, error) {
out := new(bytes.Buffer)
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
cmd := exec.CommandContext(cfg.Context, "go", args...)
cmd.Env = cfg.Env
cmd.Dir = cfg.Dir
cmd.Stdout = out
cmd.Stderr = new(bytes.Buffer)
cmd.Stdout = stdout
cmd.Stderr = stderr
if err := cmd.Run(); err != nil {
exitErr, ok := err.(*exec.ExitError)
if !ok {
@ -362,14 +613,14 @@ func golist(cfg *Config, args []string) (*bytes.Buffer, error) {
// If so, then we should continue to print stderr as go list
// will be silent unless something unexpected happened.
// If not, perhaps we should suppress it to reduce noise.
if stderr := fmt.Sprint(cmd.Stderr); stderr != "" {
if len(stderr.Bytes()) != 0 {
fmt.Fprintf(os.Stderr, "go list stderr <<%s>>\n", stderr)
}
// debugging
if false {
fmt.Fprintln(os.Stderr, out)
fmt.Fprintln(os.Stderr, stdout)
}
return out, nil
return stdout, nil
}

100
go/packages/packages_test.go
View File

@ -1107,6 +1107,106 @@ func TestContains_FallbackSticks(t *testing.T) {
}
}
func TestName(t *testing.T) {
tmp, cleanup := makeTree(t, map[string]string{
"src/a/needle/needle.go": `package needle; import "c"`,
"src/b/needle/needle.go": `package needle;`,
"src/c/c.go": `package c;`,
"src/irrelevant/irrelevant.go": `package irrelevant;`,
})
defer cleanup()
cfg := &packages.Config{
Mode: packages.LoadImports,
Dir: tmp,
Env: append(os.Environ(), "GOPATH="+tmp, "GO111MODULE=off"),
}
initial, err := packages.Load(cfg, "name=needle")
if err != nil {
t.Fatal(err)
}
graph, _ := importGraph(initial)
wantGraph := `
* a/needle
* b/needle
c
a/needle -> c
`[1:]
if graph != wantGraph {
t.Errorf("wrong import graph: got <<%s>>, want <<%s>>", graph, wantGraph)
}
}
func TestName_Modules(t *testing.T) {
tmp, cleanup := makeTree(t, map[string]string{
"src/localmod/go.mod": `module test`,
"src/localmod/pkg/pkg.go": `package pkg;`,
})
defer cleanup()
wd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
// testdata/TestNamed_Modules contains:
// - pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/pkg
// - pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.0/pkg
// - src/b/pkg
cfg := &packages.Config{
Mode: packages.LoadImports,
Dir: filepath.Join(tmp, "src/localmod"),
Env: append(os.Environ(), "GOPATH="+wd+"/testdata/TestName_Modules", "GO111MODULE=on"),
}
initial, err := packages.Load(cfg, "name=pkg")
if err != nil {
t.Fatal(err)
}
graph, _ := importGraph(initial)
wantGraph := `
* github.com/heschik/tools-testrepo/pkg
* github.com/heschik/tools-testrepo/v2/pkg
* test/pkg
`[1:]
if graph != wantGraph {
t.Errorf("wrong import graph: got <<%s>>, want <<%s>>", graph, wantGraph)
}
}
func TestName_ModulesDedup(t *testing.T) {
tmp, cleanup := makeTree(t, map[string]string{
"src/localmod/go.mod": `module test`,
})
defer cleanup()
wd, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
// testdata/TestNamed_ModulesDedup contains:
// - pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.2/pkg/pkg.go
// - pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.1/pkg/pkg.go
// - pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/pkg/pkg.go
// but, inexplicably, not v2.0.0. Nobody knows why.
cfg := &packages.Config{
Mode: packages.LoadImports,
Dir: filepath.Join(tmp, "src/localmod"),
Env: append(os.Environ(), "GOPATH="+wd+"/testdata/TestName_ModulesDedup", "GO111MODULE=on"),
}
initial, err := packages.Load(cfg, "name=pkg")
if err != nil {
t.Fatal(err)
}
for _, pkg := range initial {
if strings.Contains(pkg.PkgPath, "v2") {
if strings.Contains(pkg.GoFiles[0], "v2.0.2") {
return
}
}
}
t.Errorf("didn't find v2.0.2 of pkg in Load results: %v", initial)
}
func TestJSON(t *testing.T) {
//TODO: add in some errors
tmp, cleanup := makeTree(t, map[string]string{

3
go/packages/testdata/README vendored Normal file
View File

@ -0,0 +1,3 @@
Test data directories here were created by running go commands with GOPATH set as such:
GOPATH=......./testdata/TestNamed_ModulesDedup go get github.com/heschik/tools-testrepo/v2@v2.0.1
and then removing the vcs cache directories, which appear to be unnecessary.

1
go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/list vendored Normal file
View File

@ -0,0 +1 @@
v1.0.0

1
go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.info vendored Normal file
View File

@ -0,0 +1 @@
{"Version":"v1.0.0","Time":"2018-09-28T22:09:08Z"}

1
go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.mod vendored Normal file
View File

@ -0,0 +1 @@
module github.com/heschik/tools-testrepo

BIN
go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.zip vendored Normal file
View File

Binary file not shown.

1
go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.ziphash vendored Normal file
View File

@ -0,0 +1 @@
h1:D2qc+R2eCTCyoT8WAYoExXhPBThJWmlYSfB4coWbfBE=

1
go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/list vendored Normal file
View File

@ -0,0 +1 @@
v2.0.0

1
go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.0.info vendored Normal file
View File

@ -0,0 +1 @@
{"Version":"v2.0.0","Time":"2018-09-28T22:12:08Z"}

1
go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.0.mod vendored Normal file
View File

@ -0,0 +1 @@
module github.com/heschik/tools-testrepo/v2

BIN
go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.0.zip vendored Normal file
View File

Binary file not shown.

1
go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.0.ziphash vendored Normal file
View File

@ -0,0 +1 @@
h1:Ll4Bx8ZD8zg8lD4idX7CAhx/jh16o9dWC2m9SnT1qu0=

1
go/packages/testdata/TestName_Modules/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.0/go.mod vendored Normal file
View File

@ -0,0 +1 @@
module github.com/heschik/tools-testrepo/v2

1
go/packages/testdata/TestName_Modules/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.0/pkg/pkg.go vendored Normal file
View File

@ -0,0 +1 @@
package pkg

1
go/packages/testdata/TestName_Modules/pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/go.mod vendored Normal file
View File

@ -0,0 +1 @@
module github.com/heschik/tools-testrepo

1
go/packages/testdata/TestName_Modules/pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/pkg/pkg.go vendored Normal file
View File

@ -0,0 +1 @@
package pkg

1
go/packages/testdata/TestName_Modules/src/b/pkg/pkg.go vendored Normal file
View File

@ -0,0 +1 @@
package pkg

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/list vendored Normal file
View File

@ -0,0 +1 @@
v1.0.0

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.info vendored Normal file
View File

@ -0,0 +1 @@
{"Version":"v1.0.0","Time":"2018-09-28T22:09:08Z"}

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.mod vendored Normal file
View File

@ -0,0 +1 @@
module github.com/heschik/tools-testrepo

BIN
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.zip vendored Normal file
View File

Binary file not shown.

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.ziphash vendored Normal file
View File

@ -0,0 +1 @@
h1:D2qc+R2eCTCyoT8WAYoExXhPBThJWmlYSfB4coWbfBE=

2
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/list vendored Normal file
View File

@ -0,0 +1,2 @@
v2.0.1
v2.0.2

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.1.info vendored Normal file
View File

@ -0,0 +1 @@
{"Version":"v2.0.1","Time":"2018-09-28T22:12:08Z"}

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.1.mod vendored Normal file
View File

@ -0,0 +1 @@
module github.com/heschik/tools-testrepo/v2

BIN
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.1.zip vendored Normal file
View File

Binary file not shown.

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.1.ziphash vendored Normal file
View File

@ -0,0 +1 @@
h1:efPBVdJ45IMcA/KXBOWyOZLo1TETKCXvzrZgfY+gqZk=

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.2.info vendored Normal file
View File

@ -0,0 +1 @@
{"Version":"v2.0.2","Time":"2018-09-28T22:12:08Z"}

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.2.mod vendored Normal file
View File

@ -0,0 +1 @@
module github.com/heschik/tools-testrepo/v2

BIN
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.2.zip vendored Normal file
View File

Binary file not shown.

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.2.ziphash vendored Normal file
View File

@ -0,0 +1 @@
h1:vUnR/JOkfEQt/wvMqbT9G2gODHVgVD1saTJ8x2ngAck=

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.1/go.mod vendored Normal file
View File

@ -0,0 +1 @@
module github.com/heschik/tools-testrepo/v2

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.1/pkg/pkg.go vendored Normal file
View File

@ -0,0 +1 @@
package pkg

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.2/go.mod vendored Normal file
View File

@ -0,0 +1 @@
module github.com/heschik/tools-testrepo/v2

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.2/pkg/pkg.go vendored Normal file
View File

@ -0,0 +1 @@
package pkg

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/go.mod vendored Normal file
View File

@ -0,0 +1 @@
module github.com/heschik/tools-testrepo

1
go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/pkg/pkg.go vendored Normal file
View File

@ -0,0 +1 @@
package pkg

6
imports/fix.go
View File

@ -526,21 +526,21 @@ func scanGoDirs() map[string]*pkg {
result := make(map[string]*pkg)
var mu sync.Mutex
add := func(srcDir, dir string) {
add := func(root gopathwalk.Root, dir string) {
mu.Lock()
defer mu.Unlock()
if _, dup := result[dir]; dup {
return
}
importpath := filepath.ToSlash(dir[len(srcDir)+len("/"):])
importpath := filepath.ToSlash(dir[len(root.Path)+len("/"):])
result[dir] = &pkg{
importPath: importpath,
importPathShort: VendorlessPath(importpath),
dir: dir,
}
}
gopathwalk.Walk(add, gopathwalk.Options{Debug: Debug, ModulesEnabled: false})
gopathwalk.Walk(gopathwalk.SrcDirsRoots(), add, gopathwalk.Options{Debug: Debug, ModulesEnabled: false})
return result
}

95
internal/gopathwalk/walk.go
View File

@ -25,41 +25,66 @@ type Options struct {
ModulesEnabled bool // Search module caches. Also disables legacy goimports ignore rules.
}
// RootType indicates the type of a Root.
type RootType int
const (
RootUnknown RootType = iota
RootGOROOT
RootGOPATH
RootCurrentModule
RootModuleCache
)
// A Root is a starting point for a Walk.
type Root struct {
Path string
Type RootType
}
// SrcDirsRoots returns the roots from build.Default.SrcDirs(). Not modules-compatible.
func SrcDirsRoots() []Root {
var roots []Root
roots = append(roots, Root{filepath.Join(build.Default.GOROOT, "src"), RootGOROOT})
for _, p := range filepath.SplitList(build.Default.GOPATH) {
roots = append(roots, Root{filepath.Join(p, "src"), RootGOPATH})
}
return roots
}
// Walk walks Go source directories ($GOROOT, $GOPATH, etc) to find packages.
// For each package found, add will be called (concurrently) with the absolute
// paths of the containing source directory and the package directory.
func Walk(add func(srcDir string, dir string), opts Options) {
for _, srcDir := range build.Default.SrcDirs() {
walkDir(srcDir, add, opts)
// add will be called concurrently.
func Walk(roots []Root, add func(root Root, dir string), opts Options) {
for _, root := range roots {
walkDir(root, add, opts)
}
}
func walkDir(srcDir string, add func(string, string), opts Options) {
func walkDir(root Root, add func(Root, string), opts Options) {
if opts.Debug {
log.Printf("scanning %s", srcDir)
log.Printf("scanning %s", root.Path)
}
w := &walker{
srcDir: srcDir,
srcV: filepath.Join(srcDir, "v"),
srcMod: filepath.Join(srcDir, "mod"),
root: root,
add: add,
opts: opts,
}
w.init()
if err := fastwalk.Walk(srcDir, w.walk); err != nil {
log.Printf("goimports: scanning directory %v: %v", srcDir, err)
if err := fastwalk.Walk(root.Path, w.walk); err != nil {
log.Printf("goimports: scanning directory %v: %v", root.Path, err)
}
if opts.Debug {
defer log.Printf("scanned %s", srcDir)
defer log.Printf("scanned %s", root.Path)
}
}
// walker is the callback for fastwalk.Walk.
type walker struct {
srcDir string // The source directory to scan.
srcV, srcMod string // vgo-style module cache dirs. Optional.
add func(string, string) // The callback that will be invoked for every possible Go package dir.
root Root // The source directory to scan.
add func(Root, string) // The callback that will be invoked for every possible Go package dir.
opts Options // Options passed to Walk by the user.
ignoredDirs []os.FileInfo // The ignored directories, loaded from .goimportsignore files.
@ -67,15 +92,32 @@ type walker struct {
// init initializes the walker based on its Options.
func (w *walker) init() {
if !w.opts.ModulesEnabled {
w.ignoredDirs = w.getIgnoredDirs(w.srcDir)
var ignoredPaths []string
if w.root.Type == RootModuleCache {
ignoredPaths = []string{"cache"}
}
if !w.opts.ModulesEnabled && w.root.Type == RootGOPATH {
ignoredPaths = w.getIgnoredDirs(w.root.Path)
ignoredPaths = append(ignoredPaths, "v", "mod")
}
for _, p := range ignoredPaths {
full := filepath.Join(w.root.Path, p)
if fi, err := os.Stat(full); err == nil {
w.ignoredDirs = append(w.ignoredDirs, fi)
if w.opts.Debug {
log.Printf("Directory added to ignore list: %s", full)
}
} else if w.opts.Debug {
log.Printf("Error statting ignored directory: %v", err)
}
}
}
// getIgnoredDirs reads an optional config file at <path>/.goimportsignore
// of relative directories to ignore when scanning for go files.
// The provided path is one of the $GOPATH entries with "src" appended.
func (w *walker) getIgnoredDirs(path string) []os.FileInfo {
func (w *walker) getIgnoredDirs(path string) []string {
file := filepath.Join(path, ".goimportsignore")
slurp, err := ioutil.ReadFile(file)
if w.opts.Debug {
@ -89,22 +131,14 @@ func (w *walker) getIgnoredDirs(path string) []os.FileInfo {
return nil
}
var ignoredDirs []os.FileInfo
var ignoredDirs []string
bs := bufio.NewScanner(bytes.NewReader(slurp))
for bs.Scan() {
line := strings.TrimSpace(bs.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
full := filepath.Join(path, line)
if fi, err := os.Stat(full); err == nil {
ignoredDirs = append(ignoredDirs, fi)
if w.opts.Debug {
log.Printf("Directory added to ignore list: %s", full)
}
} else if w.opts.Debug {
log.Printf("Error statting entry in .goimportsignore: %v", err)
}
ignoredDirs = append(ignoredDirs, line)
}
return ignoredDirs
}
@ -119,12 +153,9 @@ func (w *walker) shouldSkipDir(fi os.FileInfo) bool {
}
func (w *walker) walk(path string, typ os.FileMode) error {
if !w.opts.ModulesEnabled && (path == w.srcV || path == w.srcMod) {
return filepath.SkipDir
}
dir := filepath.Dir(path)
if typ.IsRegular() {
if dir == w.srcDir {
if dir == w.root.Path {
// Doesn't make sense to have regular files
// directly in your $GOPATH/src or $GOROOT/src.
return fastwalk.SkipFiles
@ -133,7 +164,7 @@ func (w *walker) walk(path string, typ os.FileMode) error {
return nil
}
w.add(w.srcDir, dir)
w.add(w.root, dir)
return fastwalk.SkipFiles
}
if typ == os.ModeDir {

6
internal/gopathwalk/walk_test.go
View File

@ -107,9 +107,9 @@ func TestSkip(t *testing.T) {
}
var found []string
walkDir(filepath.Join(dir, "src"), func(srcDir string, dir string) {
found = append(found, dir[len(srcDir)+1:])
}, Options{ModulesEnabled: false})
walkDir(Root{filepath.Join(dir, "src"), RootGOPATH}, func(root Root, dir string) {
found = append(found, dir[len(root.Path)+1:])
}, Options{ModulesEnabled: false, Debug: true})
if want := []string{"shouldfind"}; !reflect.DeepEqual(found, want) {
t.Errorf("expected to find only %v, got %v", want, found)
}

388
internal/semver/semver.go Normal file
View File

@ -0,0 +1,388 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package semver implements comparison of semantic version strings.
// In this package, semantic version strings must begin with a leading "v",
// as in "v1.0.0".
//
// The general form of a semantic version string accepted by this package is
//
// vMAJOR[.MINOR[.PATCH[-PRERELEASE][+BUILD]]]
//
// where square brackets indicate optional parts of the syntax;
// MAJOR, MINOR, and PATCH are decimal integers without extra leading zeros;
// PRERELEASE and BUILD are each a series of non-empty dot-separated identifiers
// using only alphanumeric characters and hyphens; and
// all-numeric PRERELEASE identifiers must not have leading zeros.
//
// This package follows Semantic Versioning 2.0.0 (see semver.org)
// with two exceptions. First, it requires the "v" prefix. Second, it recognizes
// vMAJOR and vMAJOR.MINOR (with no prerelease or build suffixes)
// as shorthands for vMAJOR.0.0 and vMAJOR.MINOR.0.
package semver
// parsed returns the parsed form of a semantic version string.
type parsed struct {
major string
minor string
patch string
short string
prerelease string
build string
err string
}
// IsValid reports whether v is a valid semantic version string.
func IsValid(v string) bool {
_, ok := parse(v)
return ok
}
// Canonical returns the canonical formatting of the semantic version v.
// It fills in any missing .MINOR or .PATCH and discards build metadata.
// Two semantic versions compare equal only if their canonical formattings
// are identical strings.
// The canonical invalid semantic version is the empty string.
func Canonical(v string) string {
p, ok := parse(v)
if !ok {
return ""
}
if p.build != "" {
return v[:len(v)-len(p.build)]
}
if p.short != "" {
return v + p.short
}
return v
}
// Major returns the major version prefix of the semantic version v.
// For example, Major("v2.1.0") == "v2".
// If v is an invalid semantic version string, Major returns the empty string.
func Major(v string) string {
pv, ok := parse(v)
if !ok {
return ""
}
return v[:1+len(pv.major)]
}
// MajorMinor returns the major.minor version prefix of the semantic version v.
// For example, MajorMinor("v2.1.0") == "v2.1".
// If v is an invalid semantic version string, MajorMinor returns the empty string.
func MajorMinor(v string) string {
pv, ok := parse(v)
if !ok {
return ""
}
i := 1 + len(pv.major)
if j := i + 1 + len(pv.minor); j <= len(v) && v[i] == '.' && v[i+1:j] == pv.minor {
return v[:j]
}
return v[:i] + "." + pv.minor
}
// Prerelease returns the prerelease suffix of the semantic version v.
// For example, Prerelease("v2.1.0-pre+meta") == "-pre".
// If v is an invalid semantic version string, Prerelease returns the empty string.
func Prerelease(v string) string {
pv, ok := parse(v)
if !ok {
return ""
}
return pv.prerelease
}
// Build returns the build suffix of the semantic version v.
// For example, Build("v2.1.0+meta") == "+meta".
// If v is an invalid semantic version string, Build returns the empty string.
func Build(v string) string {
pv, ok := parse(v)
if !ok {
return ""
}
return pv.build
}
// Compare returns an integer comparing two versions according to
// according to semantic version precedence.
// The result will be 0 if v == w, -1 if v < w, or +1 if v > w.
//
// An invalid semantic version string is considered less than a valid one.
// All invalid semantic version strings compare equal to each other.
func Compare(v, w string) int {
pv, ok1 := parse(v)
pw, ok2 := parse(w)
if !ok1 && !ok2 {
return 0
}
if !ok1 {
return -1
}
if !ok2 {
return +1
}
if c := compareInt(pv.major, pw.major); c != 0 {
return c
}
if c := compareInt(pv.minor, pw.minor); c != 0 {
return c
}
if c := compareInt(pv.patch, pw.patch); c != 0 {
return c
}
return comparePrerelease(pv.prerelease, pw.prerelease)
}
// Max canonicalizes its arguments and then returns the version string
// that compares greater.
func Max(v, w string) string {
v = Canonical(v)
w = Canonical(w)
if Compare(v, w) > 0 {
return v
}
return w
}
func parse(v string) (p parsed, ok bool) {
if v == "" || v[0] != 'v' {
p.err = "missing v prefix"
return
}
p.major, v, ok = parseInt(v[1:])
if !ok {
p.err = "bad major version"
return
}
if v == "" {
p.minor = "0"
p.patch = "0"
p.short = ".0.0"
return
}
if v[0] != '.' {
p.err = "bad minor prefix"
ok = false
return
}
p.minor, v, ok = parseInt(v[1:])
if !ok {
p.err = "bad minor version"
return
}
if v == "" {
p.patch = "0"
p.short = ".0"
return
}
if v[0] != '.' {
p.err = "bad patch prefix"
ok = false
return
}
p.patch, v, ok = parseInt(v[1:])
if !ok {
p.err = "bad patch version"
return
}
if len(v) > 0 && v[0] == '-' {
p.prerelease, v, ok = parsePrerelease(v)
if !ok {
p.err = "bad prerelease"
return
}
}
if len(v) > 0 && v[0] == '+' {
p.build, v, ok = parseBuild(v)
if !ok {
p.err = "bad build"
return
}
}
if v != "" {
p.err = "junk on end"
ok = false
return
}
ok = true
return
}
func parseInt(v string) (t, rest string, ok bool) {
if v == "" {
return
}
if v[0] < '0' || '9' < v[0] {
return
}
i := 1
for i < len(v) && '0' <= v[i] && v[i] <= '9' {
i++
}
if v[0] == '0' && i != 1 {
return
}
return v[:i], v[i:], true
}
func parsePrerelease(v string) (t, rest string, ok bool) {
// "A pre-release version MAY be denoted by appending a hyphen and
// a series of dot separated identifiers immediately following the patch version.
// Identifiers MUST comprise only ASCII alphanumerics and hyphen [0-9A-Za-z-].
// Identifiers MUST NOT be empty. Numeric identifiers MUST NOT include leading zeroes."
if v == "" || v[0] != '-' {
return
}
i := 1
start := 1
for i < len(v) && v[i] != '+' {
if !isIdentChar(v[i]) && v[i] != '.' {
return
}
if v[i] == '.' {
if start == i || isBadNum(v[start:i]) {
return
}
start = i + 1
}
i++
}
if start == i || isBadNum(v[start:i]) {
return
}
return v[:i], v[i:], true
}
func parseBuild(v string) (t, rest string, ok bool) {
if v == "" || v[0] != '+' {
return
}
i := 1
start := 1
for i < len(v) {
if !isIdentChar(v[i]) {
return
}
if v[i] == '.' {
if start == i {
return
}
start = i + 1
}
i++
}
if start == i {
return
}
return v[:i], v[i:], true
}
func isIdentChar(c byte) bool {
return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '-'
}
func isBadNum(v string) bool {
i := 0
for i < len(v) && '0' <= v[i] && v[i] <= '9' {
i++
}
return i == len(v) && i > 1 && v[0] == '0'
}
func isNum(v string) bool {
i := 0
for i < len(v) && '0' <= v[i] && v[i] <= '9' {
i++
}
return i == len(v)
}
func compareInt(x, y string) int {
if x == y {
return 0
}
if len(x) < len(y) {
return -1
}
if len(x) > len(y) {
return +1
}
if x < y {
return -1
} else {
return +1
}
}
func comparePrerelease(x, y string) int {
// "When major, minor, and patch are equal, a pre-release version has
// lower precedence than a normal version.
// Example: 1.0.0-alpha < 1.0.0.
// Precedence for two pre-release versions with the same major, minor,
// and patch version MUST be determined by comparing each dot separated
// identifier from left to right until a difference is found as follows:
// identifiers consisting of only digits are compared numerically and
// identifiers with letters or hyphens are compared lexically in ASCII
// sort order. Numeric identifiers always have lower precedence than
// non-numeric identifiers. A larger set of pre-release fields has a
// higher precedence than a smaller set, if all of the preceding
// identifiers are equal.
// Example: 1.0.0-alpha < 1.0.0-alpha.1 < 1.0.0-alpha.beta <
// 1.0.0-beta < 1.0.0-beta.2 < 1.0.0-beta.11 < 1.0.0-rc.1 < 1.0.0."
if x == y {
return 0
}
if x == "" {
return +1
}
if y == "" {
return -1
}
for x != "" && y != "" {
x = x[1:] // skip - or .
y = y[1:] // skip - or .
var dx, dy string
dx, x = nextIdent(x)
dy, y = nextIdent(y)
if dx != dy {
ix := isNum(dx)
iy := isNum(dy)
if ix != iy {
if ix {
return -1
} else {
return +1
}
}
if ix {
if len(dx) < len(dy) {
return -1
}
if len(dx) > len(dy) {
return +1
}
}
if dx < dy {
return -1
} else {
return +1
}
}
}
if x == "" {
return -1
} else {
return +1
}
}
func nextIdent(x string) (dx, rest string) {
i := 0
for i < len(x) && x[i] != '.' {
i++
}
return x[:i], x[i:]
}

182
internal/semver/semver_test.go Normal file
View File

@ -0,0 +1,182 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package semver
import (
"strings"
"testing"
)
var tests = []struct {
in string
out string
}{
{"bad", ""},
{"v1-alpha.beta.gamma", ""},
{"v1-pre", ""},
{"v1+meta", ""},
{"v1-pre+meta", ""},
{"v1.2-pre", ""},
{"v1.2+meta", ""},
{"v1.2-pre+meta", ""},
{"v1.0.0-alpha", "v1.0.0-alpha"},
{"v1.0.0-alpha.1", "v1.0.0-alpha.1"},
{"v1.0.0-alpha.beta", "v1.0.0-alpha.beta"},
{"v1.0.0-beta", "v1.0.0-beta"},
{"v1.0.0-beta.2", "v1.0.0-beta.2"},
{"v1.0.0-beta.11", "v1.0.0-beta.11"},
{"v1.0.0-rc.1", "v1.0.0-rc.1"},
{"v1", "v1.0.0"},
{"v1.0", "v1.0.0"},
{"v1.0.0", "v1.0.0"},
{"v1.2", "v1.2.0"},
{"v1.2.0", "v1.2.0"},
{"v1.2.3-456", "v1.2.3-456"},
{"v1.2.3-456.789", "v1.2.3-456.789"},
{"v1.2.3-456-789", "v1.2.3-456-789"},
{"v1.2.3-456a", "v1.2.3-456a"},
{"v1.2.3-pre", "v1.2.3-pre"},
{"v1.2.3-pre+meta", "v1.2.3-pre"},
{"v1.2.3-pre.1", "v1.2.3-pre.1"},
{"v1.2.3-zzz", "v1.2.3-zzz"},
{"v1.2.3", "v1.2.3"},
{"v1.2.3+meta", "v1.2.3"},
{"v1.2.3+meta-pre", "v1.2.3"},
}
func TestIsValid(t *testing.T) {
for _, tt := range tests {
ok := IsValid(tt.in)
if ok != (tt.out != "") {
t.Errorf("IsValid(%q) = %v, want %v", tt.in, ok, !ok)
}
}
}
func TestCanonical(t *testing.T) {
for _, tt := range tests {
out := Canonical(tt.in)
if out != tt.out {
t.Errorf("Canonical(%q) = %q, want %q", tt.in, out, tt.out)
}
}
}
func TestMajor(t *testing.T) {
for _, tt := range tests {
out := Major(tt.in)
want := ""
if i := strings.Index(tt.out, "."); i >= 0 {
want = tt.out[:i]
}
if out != want {
t.Errorf("Major(%q) = %q, want %q", tt.in, out, want)
}
}
}
func TestMajorMinor(t *testing.T) {
for _, tt := range tests {
out := MajorMinor(tt.in)
var want string
if tt.out != "" {
want = tt.in
if i := strings.Index(want, "+"); i >= 0 {
want = want[:i]
}
if i := strings.Index(want, "-"); i >= 0 {
want = want[:i]
}
switch strings.Count(want, ".") {
case 0:
want += ".0"
case 1:
// ok
case 2:
want = want[:strings.LastIndex(want, ".")]
}
}
if out != want {
t.Errorf("MajorMinor(%q) = %q, want %q", tt.in, out, want)
}
}
}
func TestPrerelease(t *testing.T) {
for _, tt := range tests {
pre := Prerelease(tt.in)
var want string
if tt.out != "" {
if i := strings.Index(tt.out, "-"); i >= 0 {
want = tt.out[i:]
}
}
if pre != want {
t.Errorf("Prerelease(%q) = %q, want %q", tt.in, pre, want)
}
}
}
func TestBuild(t *testing.T) {
for _, tt := range tests {
build := Build(tt.in)
var want string
if tt.out != "" {
if i := strings.Index(tt.in, "+"); i >= 0 {
want = tt.in[i:]
}
}
if build != want {
t.Errorf("Build(%q) = %q, want %q", tt.in, build, want)
}
}
}
func TestCompare(t *testing.T) {
for i, ti := range tests {
for j, tj := range tests {
cmp := Compare(ti.in, tj.in)
var want int
if ti.out == tj.out {
want = 0
} else if i < j {
want = -1
} else {
want = +1
}
if cmp != want {
t.Errorf("Compare(%q, %q) = %d, want %d", ti.in, tj.in, cmp, want)
}
}
}
}
func TestMax(t *testing.T) {
for i, ti := range tests {
for j, tj := range tests {
max := Max(ti.in, tj.in)
want := Canonical(ti.in)
if i < j {
want = Canonical(tj.in)
}
if max != want {
t.Errorf("Max(%q, %q) = %q, want %q", ti.in, tj.in, max, want)
}
}
}
}
var (
v1 = "v1.0.0+metadata-dash"
v2 = "v1.0.0+metadata-dash1"
)
func BenchmarkCompare(b *testing.B) {
for i := 0; i < b.N; i++ {
if Compare(v1, v2) != 0 {
b.Fatalf("bad compare")
}
}
}