diff --git a/go/packages/golist.go b/go/packages/golist.go index a5717cc06c..7ca5ebe459 100644 --- a/go/packages/golist.go +++ b/go/packages/golist.go @@ -283,7 +283,10 @@ func runNamedQueries(cfg *Config, driver driver, response *responseDeduper, quer matchesMu.Lock() defer matchesMu.Unlock() - path := dir[len(root.Path)+1:] + path := dir + if dir != root.Path { + path = dir[len(root.Path)+1:] + } if pathMatchesQueries(path, queries) { switch root.Type { case gopathwalk.RootModuleCache: diff --git a/go/packages/packagestest/modules.go b/go/packages/packagestest/modules.go index 58137bc6b0..0bcdbf5c84 100644 --- a/go/packages/packagestest/modules.go +++ b/go/packages/packagestest/modules.go @@ -170,6 +170,9 @@ func writeModuleProxy(dir, module string, files map[string]string) error { if err := z.Close(); err != nil { return err } + if err := f.Close(); err != nil { + return err + } return nil } diff --git a/imports/fix.go b/imports/fix.go index 085d8aa269..f18c413514 100644 --- a/imports/fix.go +++ b/imports/fix.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" "sync" + "time" "golang.org/x/tools/go/ast/astutil" "golang.org/x/tools/go/packages" @@ -142,8 +143,8 @@ func addGlobals(f *ast.File, globals map[string]bool) { // collectReferences builds a map of selector expressions, from // left hand side (X) to a set of right hand sides (Sel). -func collectReferences(f *ast.File) map[string]map[string]bool { - refs := map[string]map[string]bool{} +func collectReferences(f *ast.File) references { + refs := references{} var visitor visitFn visitor = func(node ast.Node) ast.Visitor { @@ -226,6 +227,11 @@ func (p *pass) findMissingImport(pkg string, syms map[string]bool) *importInfo { return nil } +// references is set of references found in a Go file. The first map key is the +// left hand side of a selector expression, the second key is the right hand +// side, and the value should always be true. +type references map[string]map[string]bool + // A pass contains all the inputs and state necessary to fix a file's imports. // It can be modified in some ways during use; see comments below. type pass struct { @@ -239,8 +245,8 @@ type pass struct { // Intermediate state, generated by load. existingImports map[string]*importInfo - allRefs map[string]map[string]bool - missingRefs map[string]map[string]bool + allRefs references + missingRefs references // Inputs to fix. These can be augmented between successive fix calls. lastTry bool // indicates that this is the last call and fix should clean up as best it can. @@ -258,28 +264,14 @@ func (p *pass) loadPackageNames(imports []*importInfo) error { unknown = append(unknown, imp.importPath) } - if !p.fixEnv.shouldUseGoPackages() { - for _, path := range unknown { - name := importPathToName(p.fixEnv, path, p.srcDir) - if name == "" { - continue - } - p.knownPackages[path] = &packageInfo{ - name: name, - exports: map[string]bool{}, - } - } - return nil - } - - cfg := p.fixEnv.newPackagesConfig(packages.LoadFiles) - pkgs, err := packages.Load(cfg, unknown...) + names, err := p.fixEnv.getResolver().loadPackageNames(unknown, p.srcDir) if err != nil { return err } - for _, pkg := range pkgs { - p.knownPackages[VendorlessPath(pkg.PkgPath)] = &packageInfo{ - name: pkg.Name, + + for path, name := range names { + p.knownPackages[path] = &packageInfo{ + name: name, exports: map[string]bool{}, } } @@ -305,7 +297,7 @@ func (p *pass) importIdentifier(imp *importInfo) string { // file's missing symbols, if any, or removes unused imports if not. func (p *pass) load() bool { p.knownPackages = map[string]*packageInfo{} - p.missingRefs = map[string]map[string]bool{} + p.missingRefs = references{} p.existingImports = map[string]*importInfo{} // Load basic information about the file in question. @@ -328,8 +320,12 @@ func (p *pass) load() bool { // f's imports by the identifier they introduce. imports := collectImports(p.f) if p.loadRealPackageNames { - if err := p.loadPackageNames(append(imports, p.candidates...)); err != nil { - panic(err) + err := p.loadPackageNames(append(imports, p.candidates...)) + if err != nil { + if Debug { + log.Printf("loading package names: %v", err) + } + return false } } for _, imp := range imports { @@ -464,7 +460,7 @@ func fixImportsDefault(fset *token.FileSet, f *ast.File, filename string, env *f // derive package names from import paths, see if the file is already // complete. We can't add any imports yet, because we don't know // if missing references are actually package vars. - p := &pass{fset: fset, f: f, srcDir: srcDir, fixEnv: env} + p := &pass{fset: fset, f: f, srcDir: srcDir} if p.load() { return nil } @@ -473,7 +469,6 @@ func fixImportsDefault(fset *token.FileSet, f *ast.File, filename string, env *f // Second pass: add information from other files in the same package, // like their package vars and imports. - p = &pass{fset: fset, f: f, srcDir: srcDir, fixEnv: env} p.otherFiles = otherFiles if p.load() { return nil @@ -487,7 +482,8 @@ func fixImportsDefault(fset *token.FileSet, f *ast.File, filename string, env *f } // Third pass: get real package names where we had previously used - // the naive algorithm. + // the naive algorithm. This is the first step that will use the + // environment, so we provide it here for the first time. p = &pass{fset: fset, f: f, srcDir: srcDir, fixEnv: env} p.loadRealPackageNames = true p.otherFiles = otherFiles @@ -517,14 +513,13 @@ func fixImportsDefault(fset *token.FileSet, f *ast.File, filename string, env *f type fixEnv struct { // If non-empty, these will be used instead of the // process-wide values. - GOPATH, GOROOT, GO111MODULE string - WorkingDir string + GOPATH, GOROOT, GO111MODULE, GOPROXY, GOFLAGS string + WorkingDir string // If true, use go/packages regardless of the environment. ForceGoPackages bool - ranGoEnv bool - gomod string + resolver resolver } func (e *fixEnv) env() []string { @@ -537,26 +532,27 @@ func (e *fixEnv) env() []string { add("GOPATH", e.GOPATH) add("GOROOT", e.GOROOT) add("GO111MODULE", e.GO111MODULE) + add("GOPROXY", e.GOPROXY) + add("GOFLAGS", e.GOFLAGS) + if e.WorkingDir != "" { + add("PWD", e.WorkingDir) + } return env } -func (e *fixEnv) shouldUseGoPackages() bool { +func (e *fixEnv) getResolver() resolver { + if e.resolver != nil { + return e.resolver + } if e.ForceGoPackages { - return true + return &goPackagesResolver{env: e} } - if !e.ranGoEnv { - e.ranGoEnv = true - cmd := exec.Command("go", "env", "GOMOD") - cmd.Dir = e.WorkingDir - cmd.Env = e.env() - out, err := cmd.Output() - if err != nil { - return false - } - e.gomod = string(bytes.TrimSpace(out)) + out, err := e.invokeGo("env", "GOMOD") + if err != nil || len(bytes.TrimSpace(out.Bytes())) == 0 { + return &gopathResolver{env: e} } - return e.gomod != "" + return &moduleResolver{env: e} } func (e *fixEnv) newPackagesConfig(mode packages.LoadMode) *packages.Config { @@ -574,7 +570,36 @@ func (e *fixEnv) buildContext() *build.Context { return &ctx } -func addStdlibCandidates(pass *pass, refs map[string]map[string]bool) { +func (e *fixEnv) invokeGo(args ...string) (*bytes.Buffer, error) { + cmd := exec.Command("go", args...) + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + cmd.Stdout = stdout + cmd.Stderr = stderr + cmd.Env = e.env() + cmd.Dir = e.WorkingDir + + if Debug { + defer func(start time.Time) { log.Printf("%s for %v", time.Since(start), cmdDebugStr(cmd)) }(time.Now()) + } + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("running go: %v (stderr:\n%s)", err, stderr) + } + return stdout, nil +} + +func cmdDebugStr(cmd *exec.Cmd) string { + env := make(map[string]string) + for _, kv := range cmd.Env { + split := strings.Split(kv, "=") + k, v := split[0], split[1] + env[k] = v + } + + return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v GOPROXY=%v PWD=%v go %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["GOPROXY"], env["PWD"], cmd.Args) +} + +func addStdlibCandidates(pass *pass, refs references) { add := func(pkg string) { pass.addCandidate( &importInfo{importPath: pkg}, @@ -595,13 +620,47 @@ func addStdlibCandidates(pass *pass, refs map[string]map[string]bool) { } } -func scanGoPackages(env *fixEnv, refs map[string]map[string]bool) ([]*pkg, error) { +// A resolver does the build-system-specific parts of goimports. +type resolver interface { + // loadPackageNames loads the package names in importPaths. + loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) + // scan finds (at least) the packages satisfying refs. The returned slice is unordered. + scan(refs references) ([]*pkg, error) +} + +// gopathResolver implements resolver for GOPATH and module workspaces using go/packages. +type goPackagesResolver struct { + env *fixEnv +} + +func (r *goPackagesResolver) loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) { + cfg := r.env.newPackagesConfig(packages.LoadFiles) + pkgs, err := packages.Load(cfg, importPaths...) + if err != nil { + return nil, err + } + names := map[string]string{} + for _, pkg := range pkgs { + names[VendorlessPath(pkg.PkgPath)] = pkg.Name + } + // We may not have found all the packages. Guess the rest. + for _, path := range importPaths { + if _, ok := names[path]; ok { + continue + } + names[path] = importPathToNameBasic(path, srcDir) + } + return names, nil + +} + +func (r *goPackagesResolver) scan(refs references) ([]*pkg, error) { var loadQueries []string for pkgName := range refs { loadQueries = append(loadQueries, "name="+pkgName) } sort.Strings(loadQueries) - cfg := env.newPackagesConfig(packages.LoadFiles) + cfg := r.env.newPackagesConfig(packages.LoadFiles) goPackages, err := packages.Load(cfg, loadQueries...) if err != nil { return nil, err @@ -618,18 +677,10 @@ func scanGoPackages(env *fixEnv, refs map[string]map[string]bool) ([]*pkg, error return scan, nil } -var addExternalCandidates = addExternalCandidatesDefault - -func addExternalCandidatesDefault(pass *pass, refs map[string]map[string]bool, filename string) error { - var dirScan []*pkg - if pass.fixEnv.shouldUseGoPackages() { - var err error - dirScan, err = scanGoPackages(pass.fixEnv, refs) - if err != nil { - return err - } - } else { - dirScan = scanGoDirs(pass.fixEnv) +func addExternalCandidates(pass *pass, refs references, filename string) error { + dirScan, err := pass.fixEnv.getResolver().scan(refs) + if err != nil { + return err } // Search for imports matching potential package references. @@ -705,6 +756,19 @@ func importPathToNameBasic(importPath, srcDir string) (packageName string) { return base } +// gopathResolver implements resolver for GOPATH workspaces. +type gopathResolver struct { + env *fixEnv +} + +func (r *gopathResolver) loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) { + names := map[string]string{} + for _, path := range importPaths { + names[path] = importPathToName(r.env, path, srcDir) + } + return names, nil +} + // importPathToNameGoPath finds out the actual package name, as declared in its .go files. // If there's a problem, it returns "". func importPathToName(env *fixEnv, importPath, srcDir string) (packageName string) { @@ -713,26 +777,23 @@ func importPathToName(env *fixEnv, importPath, srcDir string) (packageName strin return path.Base(importPath) // stdlib packages always match their paths. } - pkgName, err := importPathToNameGoPathParse(env, importPath, srcDir) - if Debug { - log.Printf("importPathToNameGoPathParse(%q, srcDir=%q) = %q, %v", importPath, srcDir, pkgName, err) + buildPkg, err := env.buildContext().Import(importPath, srcDir, build.FindOnly) + if err != nil { + return "" } + pkgName, err := packageDirToName(buildPkg.Dir) if err != nil { return "" } return pkgName } -// importPathToNameGoPathParse is a faster version of build.Import if +// packageDirToName is a faster version of build.Import if // the only thing desired is the package name. It uses build.FindOnly // to find the directory and then only parses one file in the package, // trusting that the files in the directory are consistent. -func importPathToNameGoPathParse(env *fixEnv, importPath, srcDir string) (packageName string, err error) { - buildPkg, err := env.buildContext().Import(importPath, srcDir, build.FindOnly) - if err != nil { - return "", err - } - d, err := os.Open(buildPkg.Dir) +func packageDirToName(dir string) (packageName string, err error) { + d, err := os.Open(dir) if err != nil { return "", err } @@ -752,7 +813,7 @@ func importPathToNameGoPathParse(env *fixEnv, importPath, srcDir string) (packag continue } nfile++ - fullFile := filepath.Join(buildPkg.Dir, name) + fullFile := filepath.Join(dir, name) fset := token.NewFileSet() f, err := parser.ParseFile(fset, fullFile, nil, parser.PackageClauseOnly) @@ -826,8 +887,7 @@ func distance(basepath, targetpath string) int { return strings.Count(p, string(filepath.Separator)) + 1 } -// scanGoDirs populates the dirScan map for GOPATH and GOROOT. -func scanGoDirs(env *fixEnv) []*pkg { +func (r *gopathResolver) scan(_ references) ([]*pkg, error) { dupCheck := make(map[string]bool) var result []*pkg @@ -847,8 +907,8 @@ func scanGoDirs(env *fixEnv) []*pkg { dir: dir, }) } - gopathwalk.Walk(gopathwalk.SrcDirsRoots(env.buildContext()), add, gopathwalk.Options{Debug: Debug, ModulesEnabled: false}) - return result + gopathwalk.Walk(gopathwalk.SrcDirsRoots(r.env.buildContext()), add, gopathwalk.Options{Debug: Debug, ModulesEnabled: false}) + return result, nil } // VendorlessPath returns the devendorized version of the import path ipath. diff --git a/imports/fix_test.go b/imports/fix_test.go index dd9fe45184..d475e714d8 100644 --- a/imports/fix_test.go +++ b/imports/fix_test.go @@ -1511,9 +1511,10 @@ func (c testConfig) test(t *testing.T, fn func(*goimportTest)) { c.modules = []packagestest.Module{c.module} } - kinds := []string{"GOPATH_GoPackages"} + var kinds []string for _, exporter := range packagestest.All { kinds = append(kinds, exporter.Name()) + kinds = append(kinds, exporter.Name()+"_GoPackages") } for _, kind := range kinds { t.Run(kind, func(t *testing.T) { @@ -1532,6 +1533,12 @@ func (c testConfig) test(t *testing.T, fn func(*goimportTest)) { t.Skip("test marked GOPATH-only") } exporter = packagestest.Modules + case "Modules_GoPackages": + if c.gopathOnly { + t.Skip("test marked GOPATH-only") + } + exporter = packagestest.Modules + forceGoPackages = true default: panic("unknown test type") } @@ -1792,7 +1799,6 @@ const Y = foo.X // never make it that far). func TestImportPathToNameGoPathParse(t *testing.T) { testConfig{ - gopathOnly: true, module: packagestest.Module{ Name: "example.net/pkg", Files: fm{ @@ -1803,13 +1809,18 @@ func TestImportPathToNameGoPathParse(t *testing.T) { }, }, }.test(t, func(t *goimportTest) { - got, err := importPathToNameGoPathParse(t.fixEnv, "example.net/pkg", filepath.Join(t.fixEnv.GOPATH, "src", "other.net")) + if strings.Contains(t.Name(), "GoPackages") { + t.Skip("go/packages does not ignore package main") + } + r := t.fixEnv.getResolver() + srcDir := filepath.Dir(t.exported.File("example.net/pkg", "z.go")) + names, err := r.loadPackageNames([]string{"example.net/pkg"}, srcDir) if err != nil { t.Fatal(err) } const want = "the_pkg_name_to_find" - if got != want { - t.Errorf("importPathToNameGoPathParse(..) = %q; want %q", got, want) + if got := names["example.net/pkg"]; got != want { + t.Errorf("loadPackageNames(..) = %q; want %q", got, want) } }) } diff --git a/imports/mod.go b/imports/mod.go new file mode 100644 index 0000000000..ec769145cc --- /dev/null +++ b/imports/mod.go @@ -0,0 +1,351 @@ +package imports + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/tools/internal/gopathwalk" + "golang.org/x/tools/internal/module" +) + +// moduleResolver implements resolver for modules using the go command as little +// as feasible. +type moduleResolver struct { + env *fixEnv + + main *moduleJSON + modsByModPath []*moduleJSON // All modules, ordered by # of path components in module Path... + modsByDir []*moduleJSON // ...or Dir. +} + +type moduleJSON struct { + Path string // module path + Version string // module version + Versions []string // available module versions (with -versions) + Replace *moduleJSON // replaced by this module + Time *time.Time // time version was created + Update *moduleJSON // available update, if any (with -u) + Main bool // is this the main module? + Indirect bool // is this module only an indirect dependency of main module? + Dir string // directory holding files for this module, if any + GoMod string // path to go.mod file for this module, if any + Error *moduleErrorJSON // error loading module +} + +type moduleErrorJSON struct { + Err string // the error itself +} + +func (r *moduleResolver) init() error { + if r.main != nil { + return nil + } + stdout, err := r.env.invokeGo("list", "-m", "-json", "...") + if err != nil { + return err + } + for dec := json.NewDecoder(stdout); dec.More(); { + mod := &moduleJSON{} + if err := dec.Decode(mod); err != nil { + return err + } + if mod.Dir == "" { + if Debug { + log.Printf("module %v has not been downloaded and will be ignored", mod.Path) + } + // Can't do anything with a module that's not downloaded. + continue + } + r.modsByModPath = append(r.modsByModPath, mod) + r.modsByDir = append(r.modsByDir, mod) + if mod.Main { + r.main = mod + } + } + + sort.Slice(r.modsByModPath, func(i, j int) bool { + count := func(x int) int { + return strings.Count(r.modsByModPath[x].Path, "/") + } + return count(j) < count(i) // descending order + }) + sort.Slice(r.modsByDir, func(i, j int) bool { + count := func(x int) int { + return strings.Count(r.modsByDir[x].Dir, "/") + } + return count(j) < count(i) // descending order + }) + + return nil +} + +// findPackage returns the module and directory that contains the package at +// the given import path, or returns nil, "" if no module is in scope. +func (r *moduleResolver) findPackage(importPath string) (*moduleJSON, string) { + for _, m := range r.modsByModPath { + if !strings.HasPrefix(importPath, m.Path) { + continue + } + pathInModule := importPath[len(m.Path):] + pkgDir := filepath.Join(m.Dir, pathInModule) + if dirIsNestedModule(pkgDir, m) { + continue + } + + pkgFiles, err := ioutil.ReadDir(pkgDir) + if err != nil { + continue + } + + // A module only contains a package if it has buildable go + // files in that directory. If not, it could be provided by an + // outer module. See #29736. + for _, fi := range pkgFiles { + if ok, _ := r.env.buildContext().MatchFile(pkgDir, fi.Name()); ok { + return m, pkgDir + } + } + } + return nil, "" +} + +// findModuleByDir returns the module that contains dir, or nil if no such +// module is in scope. +func (r *moduleResolver) findModuleByDir(dir string) *moduleJSON { + // This is quite tricky and may not be correct. dir could be: + // - a package in the main module. + // - a replace target underneath the main module's directory. + // - a nested module in the above. + // - a replace target somewhere totally random. + // - a nested module in the above. + // - in the mod cache. + // - in /vendor/ in -mod=vendor mode. + // - nested module? Dunno. + // Rumor has it that replace targets cannot contain other replace targets. + for _, m := range r.modsByDir { + if !strings.HasPrefix(dir, m.Dir) { + continue + } + + if dirIsNestedModule(dir, m) { + continue + } + + return m + } + return nil +} + +// dirIsNestedModule reports if dir is contained in a nested module underneath +// mod, not actually in mod. +func dirIsNestedModule(dir string, mod *moduleJSON) bool { + if !strings.HasPrefix(dir, mod.Dir) { + return false + } + mf := findModFile(dir) + if mf == "" { + return false + } + return filepath.Dir(mf) != mod.Dir +} + +func findModFile(dir string) string { + for { + f := filepath.Join(dir, "go.mod") + info, err := os.Stat(f) + if err == nil && !info.IsDir() { + return f + } + d := filepath.Dir(dir) + if len(d) >= len(dir) { + return "" // reached top of file system, no go.mod + } + dir = d + } +} + +func (r *moduleResolver) loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) { + if err := r.init(); err != nil { + return nil, err + } + names := map[string]string{} + for _, path := range importPaths { + _, packageDir := r.findPackage(path) + if packageDir == "" { + continue + } + name, err := packageDirToName(packageDir) + if err != nil { + continue + } + names[path] = name + } + return names, nil +} + +func (r *moduleResolver) scan(_ references) ([]*pkg, error) { + if err := r.init(); err != nil { + return nil, err + } + + // Walk GOROOT, GOPATH/pkg/mod, and the main module. + roots := []gopathwalk.Root{ + {filepath.Join(r.env.GOROOT, "/src"), gopathwalk.RootGOROOT}, + {r.main.Dir, gopathwalk.RootCurrentModule}, + } + for _, p := range filepath.SplitList(r.env.GOPATH) { + roots = append(roots, gopathwalk.Root{filepath.Join(p, "/pkg/mod"), gopathwalk.RootModuleCache}) + } + + // Walk replace targets, just in case they're not in any of the above. + for _, mod := range r.modsByModPath { + if mod.Replace != nil { + roots = append(roots, gopathwalk.Root{mod.Dir, gopathwalk.RootOther}) + } + } + + var result []*pkg + dupCheck := make(map[string]bool) + var mu sync.Mutex + + gopathwalk.Walk(roots, func(root gopathwalk.Root, dir string) { + mu.Lock() + defer mu.Unlock() + + if _, dup := dupCheck[dir]; dup { + return + } + + dupCheck[dir] = true + + subdir := "" + if dir != root.Path { + subdir = dir[len(root.Path)+len("/"):] + } + importPath := filepath.ToSlash(subdir) + if strings.HasPrefix(importPath, "vendor/") { + // Ignore vendor dirs. If -mod=vendor is on, then things + // should mostly just work, but when it's not vendor/ + // is a mess. There's no easy way to tell if it's on. + // We can still find things in the mod cache and + // map them into /vendor when -mod=vendor is on. + return + } + switch root.Type { + case gopathwalk.RootCurrentModule: + importPath = path.Join(r.main.Path, filepath.ToSlash(subdir)) + case gopathwalk.RootModuleCache: + matches := modCacheRegexp.FindStringSubmatch(subdir) + modPath, err := module.DecodePath(filepath.ToSlash(matches[1])) + if err != nil { + if Debug { + log.Printf("decoding module cache path %q: %v", subdir, err) + } + return + } + importPath = path.Join(modPath, filepath.ToSlash(matches[3])) + case gopathwalk.RootGOROOT: + importPath = subdir + } + + // Check if the directory is underneath a module that's in scope. + if mod := r.findModuleByDir(dir); mod != nil { + // It is. If dir is the target of a replace directive, + // our guessed import path is wrong. Use the real one. + if mod.Dir == dir { + importPath = mod.Path + } else { + dirInMod := dir[len(mod.Dir)+len("/"):] + importPath = path.Join(mod.Path, filepath.ToSlash(dirInMod)) + } + } else { + // The package is in an unknown module. Check that it's + // not obviously impossible to import. + var modFile string + switch root.Type { + case gopathwalk.RootModuleCache: + matches := modCacheRegexp.FindStringSubmatch(subdir) + modFile = filepath.Join(matches[1], "@", matches[2], "go.mod") + default: + modFile = findModFile(dir) + } + + modBytes, err := ioutil.ReadFile(modFile) + if err == nil && !strings.HasPrefix(importPath, modulePath(modBytes)) { + // The module's declared path does not match + // its expected path. It probably needs a + // replace directive we don't have. + return + } + } + // We may have discovered a package that has a different version + // in scope already. Canonicalize to that one if possible. + if _, canonicalDir := r.findPackage(importPath); canonicalDir != "" { + dir = canonicalDir + } + + result = append(result, &pkg{ + importPathShort: VendorlessPath(importPath), + dir: dir, + }) + }, gopathwalk.Options{Debug: Debug, ModulesEnabled: true}) + return result, nil +} + +// modCacheRegexp splits a path in a module cache into module, module version, and package. +var modCacheRegexp = regexp.MustCompile(`(.*)@([^/\\]*)(.*)`) + +var ( + slashSlash = []byte("//") + moduleStr = []byte("module") +) + +// modulePath returns the module path from the gomod file text. +// If it cannot find a module path, it returns an empty string. +// It is tolerant of unrelated problems in the go.mod file. +// +// Copied from cmd/go/internal/modfile. +func modulePath(mod []byte) string { + for len(mod) > 0 { + line := mod + mod = nil + if i := bytes.IndexByte(line, '\n'); i >= 0 { + line, mod = line[:i], line[i+1:] + } + if i := bytes.Index(line, slashSlash); i >= 0 { + line = line[:i] + } + line = bytes.TrimSpace(line) + if !bytes.HasPrefix(line, moduleStr) { + continue + } + line = line[len(moduleStr):] + n := len(line) + line = bytes.TrimSpace(line) + if len(line) == n || len(line) == 0 { + continue + } + + if line[0] == '"' || line[0] == '`' { + p, err := strconv.Unquote(string(line)) + if err != nil { + return "" // malformed quoted string or multiline module path + } + return p + } + + return string(line) + } + return "" // missing module path +} diff --git a/imports/mod_test.go b/imports/mod_test.go new file mode 100644 index 0000000000..bda2c06167 --- /dev/null +++ b/imports/mod_test.go @@ -0,0 +1,643 @@ +// +build go1.11 + +package imports + +import ( + "archive/zip" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "testing" + + "golang.org/x/tools/internal/module" + "golang.org/x/tools/internal/txtar" +) + +// Tests that we handle a nested module. This is different from other tests +// where the module is in scope -- here we have to figure out the import path +// without any help from go list. +func TestScanOutOfScopeNestedModule(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module x + +-- x.go -- +package x + +-- v2/go.mod -- +module x + +-- v2/x.go -- +package x`, "") + defer mt.cleanup() + + pkg := mt.assertScanFinds("x/v2", "x") + if pkg != nil && !strings.HasSuffix(filepath.ToSlash(pkg.dir), "main/v2") { + t.Errorf("x/v2 was found in %v, wanted .../main/v2", pkg.dir) + } + // We can't load the package name from the import path, but that should + // be okay -- if we end up adding this result, we'll add it with a name + // if necessary. +} + +// Tests that we don't find a nested module contained in a local replace target. +// The code for this case is too annoying to write, so it's just ignored. +func TestScanNestedModuleInLocalReplace(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module x + +require y v0.0.0 +replace y => ./y + +-- x.go -- +package x + +-- y/go.mod -- +module y + +-- y/y.go -- +package y + +-- y/z/go.mod -- +module y/z + +-- y/z/z.go -- +package z +`, "") + defer mt.cleanup() + + mt.assertFound("y", "y") + + scan, err := mt.resolver.scan(nil) + if err != nil { + t.Fatal(err) + } + for _, pkg := range scan { + if strings.HasSuffix(filepath.ToSlash(pkg.dir), "main/y/z") { + t.Errorf("scan found a package %v in dir main/y/z, wanted none", pkg.importPathShort) + } + } +} + +// Tests that path encoding is handled correctly. Adapted from mod_case.txt. +func TestModCase(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module x + +require rsc.io/QUOTE v1.5.2 + +-- x.go -- +package x + +import _ "rsc.io/QUOTE/QUOTE" +`, "") + defer mt.cleanup() + mt.assertFound("rsc.io/QUOTE/QUOTE", "QUOTE") +} + +// Not obviously relevant to goimports. Adapted from mod_domain_root.txt anyway. +func TestModDomainRoot(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module x + +require example.com v1.0.0 + +-- x.go -- +package x +import _ "example.com" +`, "") + defer mt.cleanup() + mt.assertFound("example.com", "x") +} + +// Tests that -mod=vendor sort of works. Adapted from mod_getmode_vendor.txt. +func TestModeGetmodeVendor(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module x + +require rsc.io/quote v1.5.2 +-- x.go -- +package x +import _ "rsc.io/quote" +`, "") + defer mt.cleanup() + + if _, err := mt.env.invokeGo("mod", "vendor"); err != nil { + t.Fatal(err) + } + + mt.env.GOFLAGS = "-mod=vendor" + mt.assertModuleFoundInDir("rsc.io/quote", "quote", `/vendor/`) + + mt.env.GOFLAGS = "" + // Clear out the resolver's cache, since we've changed the environment. + mt.resolver = &moduleResolver{env: mt.env} + mt.assertModuleFoundInDir("rsc.io/quote", "quote", `pkg.*mod.*/quote@.*$`) +} + +// Tests that a module replace works. Adapted from mod_list.txt. We start with +// go.mod2; the first part of the test is irrelevant. +func TestModList(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module x +require rsc.io/quote v1.5.1 +replace rsc.io/sampler v1.3.0 => rsc.io/sampler v1.3.1 + +-- x.go -- +package x +import _ "rsc.io/quote" +`, "") + defer mt.cleanup() + + mt.assertModuleFoundInDir("rsc.io/sampler", "sampler", `pkg.mod.*/sampler@v1.3.1$`) +} + +// Tests that a local replace works. Adapted from mod_local_replace.txt. +func TestModLocalReplace(t *testing.T) { + mt := setup(t, ` +-- x/y/go.mod -- +module x/y +require zz v1.0.0 +replace zz v1.0.0 => ../z + +-- x/y/y.go -- +package y +import _ "zz" + +-- x/z/go.mod -- +module x/z + +-- x/z/z.go -- +package z +`, "x/y") + defer mt.cleanup() + + mt.assertFound("zz", "z") +} + +// Tests that the package at the root of the main module can be found. +// Adapted from the first part of mod_multirepo.txt. +func TestModMultirepo1(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module rsc.io/quote + +-- x.go -- +package quote +`, "") + defer mt.cleanup() + + mt.assertModuleFoundInDir("rsc.io/quote", "quote", `/main`) +} + +// Tests that a simple module dependency is found. Adapted from the third part +// of mod_multirepo.txt (We skip the case where it doesn't have a go.mod +// entry -- we just don't work in that case.) +func TestModMultirepo3(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module rsc.io/quote + +require rsc.io/quote/v2 v2.0.1 +-- x.go -- +package quote + +import _ "rsc.io/quote/v2" +`, "") + defer mt.cleanup() + + mt.assertModuleFoundInDir("rsc.io/quote", "quote", `/main`) + mt.assertModuleFoundInDir("rsc.io/quote/v2", "quote", `pkg.mod.*/v2@v2.0.1$`) +} + +// Tests that a nested module is found in the module cache, even though +// it's checked out. Adapted from the fourth part of mod_multirepo.txt. +func TestModMultirepo4(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module rsc.io/quote +require rsc.io/quote/v2 v2.0.1 + +-- x.go -- +package quote +import _ "rsc.io/quote/v2" + +-- v2/go.mod -- +package rsc.io/quote/v2 + +-- v2/x.go -- +package quote +import _ "rsc.io/quote/v2" +`, "") + defer mt.cleanup() + + mt.assertModuleFoundInDir("rsc.io/quote", "quote", `/main`) + mt.assertModuleFoundInDir("rsc.io/quote/v2", "quote", `pkg.mod.*/v2@v2.0.1$`) +} + +// Tests a simple module dependency. Adapted from the first part of mod_replace.txt. +func TestModReplace1(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module quoter + +require rsc.io/quote/v3 v3.0.0 + +-- main.go -- + +package main +`, "") + defer mt.cleanup() + mt.assertFound("rsc.io/quote/v3", "quote") +} + +// Tests a local replace. Adapted from the second part of mod_replace.txt. +func TestModReplace2(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module quoter + +require rsc.io/quote/v3 v3.0.0 +replace rsc.io/quote/v3 => ./local/rsc.io/quote/v3 +-- main.go -- +package main + +-- local/rsc.io/quote/v3/go.mod -- +module rsc.io/quote/v3 + +require rsc.io/sampler v1.3.0 + +-- local/rsc.io/quote/v3/quote.go -- +package quote + +import "rsc.io/sampler" +`, "") + defer mt.cleanup() + mt.assertModuleFoundInDir("rsc.io/quote/v3", "quote", `/local/rsc.io/quote/v3`) +} + +// Tests that a module can be replaced by a different module path. Adapted +// from the third part of mod_replace.txt. +func TestModReplace3(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module quoter + +require not-rsc.io/quote/v3 v3.1.0 +replace not-rsc.io/quote/v3 v3.1.0 => ./local/rsc.io/quote/v3 + +-- usenewmodule/main.go -- +package main + +-- local/rsc.io/quote/v3/go.mod -- +module rsc.io/quote/v3 + +require rsc.io/sampler v1.3.0 + +-- local/rsc.io/quote/v3/quote.go -- +package quote + +-- local/not-rsc.io/quote/v3/go.mod -- +module not-rsc.io/quote/v3 + +-- local/not-rsc.io/quote/v3/quote.go -- +package quote +`, "") + defer mt.cleanup() + mt.assertModuleFoundInDir("not-rsc.io/quote/v3", "quote", "local/rsc.io/quote/v3") +} + +// Tests more local replaces, notably the case where an outer module provides +// a package that could also be provided by an inner module. Adapted from +// mod_replace_import.txt, with example.com/v changed to /vv because Go 1.11 +// thinks /v is an invalid major version. +func TestModReplaceImport(t *testing.T) { + mt := setup(t, ` +-- go.mod -- +module example.com/m + +replace ( + example.com/a => ./a + example.com/a/b => ./b +) + +replace ( + example.com/x => ./x + example.com/x/v3 => ./v3 +) + +replace ( + example.com/y/z/w => ./w + example.com/y => ./y +) + +replace ( + example.com/vv v1.11.0 => ./v11 + example.com/vv v1.12.0 => ./v12 + example.com/vv => ./vv +) + +require ( + example.com/a/b v0.0.0 + example.com/x/v3 v3.0.0 + example.com/y v0.0.0 + example.com/y/z/w v0.0.0 + example.com/vv v1.12.0 +) +-- m.go -- +package main +import ( + _ "example.com/a/b" + _ "example.com/x/v3" + _ "example.com/y/z/w" + _ "example.com/vv" +) +func main() {} + +-- a/go.mod -- +module a.localhost +-- a/a.go -- +package a +-- a/b/b.go-- +package b + +-- b/go.mod -- +module a.localhost/b +-- b/b.go -- +package b + +-- x/go.mod -- +module x.localhost +-- x/x.go -- +package x +-- x/v3.go -- +package v3 +import _ "x.localhost/v3" + +-- v3/go.mod -- +module x.localhost/v3 +-- v3/x.go -- +package x + +-- w/go.mod -- +module w.localhost +-- w/skip/skip.go -- +// Package skip is nested below nonexistent package w. +package skip + +-- y/go.mod -- +module y.localhost +-- y/z/w/w.go -- +package w + +-- v12/go.mod -- +module v.localhost +-- v12/v.go -- +package v + +-- v11/go.mod -- +module v.localhost +-- v11/v.go -- +package v + +-- vv/go.mod -- +module v.localhost +-- vv/v.go -- +package v +`, "") + defer mt.cleanup() + + mt.assertModuleFoundInDir("example.com/a/b", "b", `main/b$`) + mt.assertModuleFoundInDir("example.com/x/v3", "x", `main/v3$`) + mt.assertModuleFoundInDir("example.com/y/z/w", "w", `main/y/z/w$`) + mt.assertModuleFoundInDir("example.com/vv", "v", `main/v12$`) +} + +// assertFound asserts that the package at importPath is found to have pkgName, +// and that scanning for pkgName finds it at importPath. +func (t *modTest) assertFound(importPath, pkgName string) (string, *pkg) { + t.Helper() + + names, err := t.resolver.loadPackageNames([]string{importPath}, t.env.WorkingDir) + if err != nil { + t.Errorf("loading package name for %v: %v", importPath, err) + } + if names[importPath] != pkgName { + t.Errorf("package name for %v = %v, want %v", importPath, names[importPath], pkgName) + } + pkg := t.assertScanFinds(importPath, pkgName) + + _, foundDir := t.resolver.findPackage(importPath) + return foundDir, pkg +} + +func (t *modTest) assertScanFinds(importPath, pkgName string) *pkg { + t.Helper() + scan, err := t.resolver.scan(nil) + if err != nil { + t.Errorf("scan failed: %v", err) + } + for _, pkg := range scan { + if pkg.importPathShort == importPath { + return pkg + } + } + t.Errorf("scanning for %v did not find %v", pkgName, importPath) + return nil +} + +// assertModuleFoundInDir is the same as assertFound, but also checks that the +// package was found in an active module whose Dir matches dirRE. +func (t *modTest) assertModuleFoundInDir(importPath, pkgName, dirRE string) { + t.Helper() + dir, pkg := t.assertFound(importPath, pkgName) + re, err := regexp.Compile(dirRE) + if err != nil { + t.Fatal(err) + } + + if dir == "" { + t.Errorf("import path %v not found in active modules", importPath) + } else { + if !re.MatchString(filepath.ToSlash(dir)) { + t.Errorf("finding dir for %s: dir = %q did not match regex %q", importPath, dir, dirRE) + } + } + if pkg != nil { + if !re.MatchString(filepath.ToSlash(pkg.dir)) { + t.Errorf("scanning for %s: dir = %q did not match regex %q", pkgName, pkg.dir, dirRE) + } + } +} + +var proxyOnce sync.Once +var proxyDir string + +type modTest struct { + *testing.T + env *fixEnv + resolver *moduleResolver + cleanup func() +} + +// setup builds a test enviroment from a txtar and supporting modules +// in testdata/mod, along the lines of TestScript in cmd/go. +func setup(t *testing.T, main, wd string) *modTest { + t.Helper() + proxyOnce.Do(func() { + var err error + proxyDir, err = ioutil.TempDir("", "proxy-") + if err != nil { + t.Fatal(err) + } + if err := writeProxy(proxyDir, "testdata/mod"); err != nil { + t.Fatal(err) + } + }) + + dir, err := ioutil.TempDir("", t.Name()) + if err != nil { + t.Fatal(err) + } + + mainDir := filepath.Join(dir, "main") + if err := writeModule(mainDir, main); err != nil { + t.Fatal(err) + } + + env := &fixEnv{ + GOPATH: filepath.Join(dir, "gopath"), + GO111MODULE: "on", + GOPROXY: "file://" + filepath.ToSlash(proxyDir), + WorkingDir: filepath.Join(mainDir, wd), + } + + // go mod tidy instead of download because tidy will notice dependencies + // in code, not just in go.mod files. + if _, err := env.invokeGo("mod", "download"); err != nil { + t.Fatal(err) + } + + return &modTest{ + T: t, + env: env, + resolver: &moduleResolver{env: env}, + cleanup: func() { + _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.IsDir() { + _ = os.Chmod(path, 0777) + } + return nil + }) + _ = os.RemoveAll(dir) // ignore errors + }, + } +} + +// writeModule writes the module in the ar, a txtar, to dir. +func writeModule(dir, ar string) error { + a := txtar.Parse([]byte(ar)) + + for _, f := range a.Files { + fpath := filepath.Join(dir, f.Name) + if err := os.MkdirAll(filepath.Dir(fpath), 0755); err != nil { + return err + } + + if err := ioutil.WriteFile(fpath, f.Data, 0644); err != nil { + return err + } + } + return nil +} + +// writeProxy writes all the txtar-formatted modules in arDir to a proxy +// directory in dir. +func writeProxy(dir, arDir string) error { + files, err := ioutil.ReadDir(arDir) + if err != nil { + return err + } + + for _, fi := range files { + if err := writeProxyModule(dir, filepath.Join(arDir, fi.Name())); err != nil { + return err + } + } + return nil +} + +// writeProxyModule writes a txtar-formatted module at arPath to the module +// proxy in base. +func writeProxyModule(base, arPath string) error { + arName := filepath.Base(arPath) + i := strings.LastIndex(arName, "_v") + ver := strings.TrimSuffix(arName[i+1:], ".txt") + modDir := strings.Replace(arName[:i], "_", "/", -1) + modPath, err := module.DecodePath(modDir) + if err != nil { + return err + } + + dir := filepath.Join(base, modDir, "@v") + a, err := txtar.ParseFile(arPath) + + if err != nil { + return err + } + + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + + f, err := os.OpenFile(filepath.Join(dir, ver+".zip"), os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + z := zip.NewWriter(f) + for _, f := range a.Files { + if f.Name[0] == '.' { + if err := ioutil.WriteFile(filepath.Join(dir, ver+f.Name), f.Data, 0644); err != nil { + return err + } + } else { + zf, err := z.Create(modPath + "@" + ver + "/" + f.Name) + if err != nil { + return err + } + if _, err := zf.Write(f.Data); err != nil { + return err + } + } + } + if err := z.Close(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + + list, err := os.OpenFile(filepath.Join(dir, "list"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + if _, err := fmt.Fprintf(list, "%s\n", ver); err != nil { + return err + } + if err := list.Close(); err != nil { + return err + } + return nil +} diff --git a/imports/testdata/mod/example.com_v1.0.0.txt b/imports/testdata/mod/example.com_v1.0.0.txt new file mode 100644 index 0000000000..263287d9e2 --- /dev/null +++ b/imports/testdata/mod/example.com_v1.0.0.txt @@ -0,0 +1,9 @@ +Written by hand. +Test case for module at root of domain. + +-- .mod -- +module example.com +-- .info -- +{"Version": "v1.0.0"} +-- x.go -- +package x diff --git a/imports/testdata/mod/golang.org_x_text_v0.0.0-20170915032832-14c0d48ead0c.txt b/imports/testdata/mod/golang.org_x_text_v0.0.0-20170915032832-14c0d48ead0c.txt new file mode 100644 index 0000000000..f4f50cdedb --- /dev/null +++ b/imports/testdata/mod/golang.org_x_text_v0.0.0-20170915032832-14c0d48ead0c.txt @@ -0,0 +1,47 @@ +written by hand - just enough to compile rsc.io/sampler, rsc.io/quote + +-- .mod -- +module golang.org/x/text +-- .info -- +{"Version":"v0.0.0-20170915032832-14c0d48ead0c","Name":"v0.0.0-20170915032832-14c0d48ead0c","Short":"14c0d48ead0c","Time":"2017-09-15T03:28:32Z"} +-- go.mod -- +module golang.org/x/text +-- unused/unused.go -- +package unused +-- language/lang.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This is a tiny version of golang.org/x/text. + +package language + +import "strings" + +type Tag string + +func Make(s string) Tag { return Tag(s) } + +func (t Tag) String() string { return string(t) } + +func NewMatcher(tags []Tag) Matcher { return &matcher{tags} } + +type Matcher interface { + Match(...Tag) (Tag, int, int) +} + +type matcher struct { + tags []Tag +} + +func (m *matcher) Match(prefs ...Tag) (Tag, int, int) { + for _, pref := range prefs { + for _, tag := range m.tags { + if tag == pref || strings.HasPrefix(string(pref), string(tag+"-")) || strings.HasPrefix(string(tag), string(pref+"-")) { + return tag, 0, 0 + } + } + } + return m.tags[0], 0, 0 +} diff --git a/imports/testdata/mod/rsc.io_!q!u!o!t!e_v1.5.2.txt b/imports/testdata/mod/rsc.io_!q!u!o!t!e_v1.5.2.txt new file mode 100644 index 0000000000..21185c39f3 --- /dev/null +++ b/imports/testdata/mod/rsc.io_!q!u!o!t!e_v1.5.2.txt @@ -0,0 +1,88 @@ +rsc.io/QUOTE v1.5.2 + +-- .mod -- +module rsc.io/QUOTE + +require rsc.io/quote v1.5.2 +-- .info -- +{"Version":"v1.5.2","Name":"","Short":"","Time":"2018-07-15T16:25:34Z"} +-- go.mod -- +module rsc.io/QUOTE + +require rsc.io/quote v1.5.2 +-- QUOTE/quote.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// PACKAGE QUOTE COLLECTS LOUD SAYINGS. +package QUOTE + +import ( + "strings" + + "rsc.io/quote" +) + +// HELLO RETURNS A GREETING. +func HELLO() string { + return strings.ToUpper(quote.Hello()) +} + +// GLASS RETURNS A USEFUL PHRASE FOR WORLD TRAVELERS. +func GLASS() string { + return strings.ToUpper(quote.GLASS()) +} + +// GO RETURNS A GO PROVERB. +func GO() string { + return strings.ToUpper(quote.GO()) +} + +// OPT RETURNS AN OPTIMIZATION TRUTH. +func OPT() string { + return strings.ToUpper(quote.OPT()) +} +-- QUOTE/quote_test.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package QUOTE + +import ( + "os" + "testing" +) + +func init() { + os.Setenv("LC_ALL", "en") +} + +func TestHELLO(t *testing.T) { + hello := "HELLO, WORLD" + if out := HELLO(); out != hello { + t.Errorf("HELLO() = %q, want %q", out, hello) + } +} + +func TestGLASS(t *testing.T) { + glass := "I CAN EAT GLASS AND IT DOESN'T HURT ME." + if out := GLASS(); out != glass { + t.Errorf("GLASS() = %q, want %q", out, glass) + } +} + +func TestGO(t *testing.T) { + go1 := "DON'T COMMUNICATE BY SHARING MEMORY, SHARE MEMORY BY COMMUNICATING." + if out := GO(); out != go1 { + t.Errorf("GO() = %q, want %q", out, go1) + } +} + +func TestOPT(t *testing.T) { + opt := "IF A PROGRAM IS TOO SLOW, IT MUST HAVE A LOOP." + if out := OPT(); out != opt { + t.Errorf("OPT() = %q, want %q", out, opt) + } +} diff --git a/imports/testdata/mod/rsc.io_!q!u!o!t!e_v1.5.3-!p!r!e.txt b/imports/testdata/mod/rsc.io_!q!u!o!t!e_v1.5.3-!p!r!e.txt new file mode 100644 index 0000000000..54bac2df7b --- /dev/null +++ b/imports/testdata/mod/rsc.io_!q!u!o!t!e_v1.5.3-!p!r!e.txt @@ -0,0 +1,88 @@ +rsc.io/QUOTE v1.5.3-PRE (sigh) + +-- .mod -- +module rsc.io/QUOTE + +require rsc.io/quote v1.5.2 +-- .info -- +{"Version":"v1.5.3-PRE","Name":"","Short":"","Time":"2018-07-15T16:25:34Z"} +-- go.mod -- +module rsc.io/QUOTE + +require rsc.io/quote v1.5.2 +-- QUOTE/quote.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// PACKAGE QUOTE COLLECTS LOUD SAYINGS. +package QUOTE + +import ( + "strings" + + "rsc.io/quote" +) + +// HELLO RETURNS A GREETING. +func HELLO() string { + return strings.ToUpper(quote.Hello()) +} + +// GLASS RETURNS A USEFUL PHRASE FOR WORLD TRAVELERS. +func GLASS() string { + return strings.ToUpper(quote.GLASS()) +} + +// GO RETURNS A GO PROVERB. +func GO() string { + return strings.ToUpper(quote.GO()) +} + +// OPT RETURNS AN OPTIMIZATION TRUTH. +func OPT() string { + return strings.ToUpper(quote.OPT()) +} +-- QUOTE/quote_test.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package QUOTE + +import ( + "os" + "testing" +) + +func init() { + os.Setenv("LC_ALL", "en") +} + +func TestHELLO(t *testing.T) { + hello := "HELLO, WORLD" + if out := HELLO(); out != hello { + t.Errorf("HELLO() = %q, want %q", out, hello) + } +} + +func TestGLASS(t *testing.T) { + glass := "I CAN EAT GLASS AND IT DOESN'T HURT ME." + if out := GLASS(); out != glass { + t.Errorf("GLASS() = %q, want %q", out, glass) + } +} + +func TestGO(t *testing.T) { + go1 := "DON'T COMMUNICATE BY SHARING MEMORY, SHARE MEMORY BY COMMUNICATING." + if out := GO(); out != go1 { + t.Errorf("GO() = %q, want %q", out, go1) + } +} + +func TestOPT(t *testing.T) { + opt := "IF A PROGRAM IS TOO SLOW, IT MUST HAVE A LOOP." + if out := OPT(); out != opt { + t.Errorf("OPT() = %q, want %q", out, opt) + } +} diff --git a/imports/testdata/mod/rsc.io_quote_v1.5.1.txt b/imports/testdata/mod/rsc.io_quote_v1.5.1.txt new file mode 100644 index 0000000000..eed051bea0 --- /dev/null +++ b/imports/testdata/mod/rsc.io_quote_v1.5.1.txt @@ -0,0 +1,86 @@ +rsc.io/quote@23179ee8a569 + +-- .mod -- +module "rsc.io/quote" + +require "rsc.io/sampler" v1.3.0 +-- .info -- +{"Version":"v1.5.1","Name":"23179ee8a569bb05d896ae05c6503ec69a19f99f","Short":"23179ee8a569","Time":"2018-02-14T00:58:40Z"} +-- go.mod -- +module "rsc.io/quote" + +require "rsc.io/sampler" v1.3.0 +-- quote.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package quote collects pithy sayings. +package quote // import "rsc.io/quote" + +import "rsc.io/sampler" + +// Hello returns a greeting. +func Hello() string { + return sampler.Hello() +} + +// Glass returns a useful phrase for world travelers. +func Glass() string { + // See http://www.oocities.org/nodotus/hbglass.html. + return "I can eat glass and it doesn't hurt me." +} + +// Go returns a Go proverb. +func Go() string { + return "Don't communicate by sharing memory, share memory by communicating." +} + +// Opt returns an optimization truth. +func Opt() string { + // Wisdom from ken. + return "If a program is too slow, it must have a loop." +} +-- quote_test.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package quote + +import ( + "os" + "testing" +) + +func init() { + os.Setenv("LC_ALL", "en") +} + +func TestHello(t *testing.T) { + hello := "Hello, world." + if out := Hello(); out != hello { + t.Errorf("Hello() = %q, want %q", out, hello) + } +} + +func TestGlass(t *testing.T) { + glass := "I can eat glass and it doesn't hurt me." + if out := Glass(); out != glass { + t.Errorf("Glass() = %q, want %q", out, glass) + } +} + +func TestGo(t *testing.T) { + go1 := "Don't communicate by sharing memory, share memory by communicating." + if out := Go(); out != go1 { + t.Errorf("Go() = %q, want %q", out, go1) + } +} + +func TestOpt(t *testing.T) { + opt := "If a program is too slow, it must have a loop." + if out := Opt(); out != opt { + t.Errorf("Opt() = %q, want %q", out, opt) + } +} diff --git a/imports/testdata/mod/rsc.io_quote_v1.5.2.txt b/imports/testdata/mod/rsc.io_quote_v1.5.2.txt new file mode 100644 index 0000000000..8671f6fe77 --- /dev/null +++ b/imports/testdata/mod/rsc.io_quote_v1.5.2.txt @@ -0,0 +1,98 @@ +rsc.io/quote@v1.5.2 + +-- .mod -- +module "rsc.io/quote" + +require "rsc.io/sampler" v1.3.0 +-- .info -- +{"Version":"v1.5.2","Name":"c4d4236f92427c64bfbcf1cc3f8142ab18f30b22","Short":"c4d4236f9242","Time":"2018-02-14T15:44:20Z"} +-- buggy/buggy_test.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buggy + +import "testing" + +func Test(t *testing.T) { + t.Fatal("buggy!") +} +-- go.mod -- +module "rsc.io/quote" + +require "rsc.io/sampler" v1.3.0 +-- quote.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package quote collects pithy sayings. +package quote // import "rsc.io/quote" + +import "rsc.io/sampler" + +// Hello returns a greeting. +func Hello() string { + return sampler.Hello() +} + +// Glass returns a useful phrase for world travelers. +func Glass() string { + // See http://www.oocities.org/nodotus/hbglass.html. + return "I can eat glass and it doesn't hurt me." +} + +// Go returns a Go proverb. +func Go() string { + return "Don't communicate by sharing memory, share memory by communicating." +} + +// Opt returns an optimization truth. +func Opt() string { + // Wisdom from ken. + return "If a program is too slow, it must have a loop." +} +-- quote_test.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package quote + +import ( + "os" + "testing" +) + +func init() { + os.Setenv("LC_ALL", "en") +} + +func TestHello(t *testing.T) { + hello := "Hello, world." + if out := Hello(); out != hello { + t.Errorf("Hello() = %q, want %q", out, hello) + } +} + +func TestGlass(t *testing.T) { + glass := "I can eat glass and it doesn't hurt me." + if out := Glass(); out != glass { + t.Errorf("Glass() = %q, want %q", out, glass) + } +} + +func TestGo(t *testing.T) { + go1 := "Don't communicate by sharing memory, share memory by communicating." + if out := Go(); out != go1 { + t.Errorf("Go() = %q, want %q", out, go1) + } +} + +func TestOpt(t *testing.T) { + opt := "If a program is too slow, it must have a loop." + if out := Opt(); out != opt { + t.Errorf("Opt() = %q, want %q", out, opt) + } +} diff --git a/imports/testdata/mod/rsc.io_quote_v2_v2.0.1.txt b/imports/testdata/mod/rsc.io_quote_v2_v2.0.1.txt new file mode 100644 index 0000000000..d51128c46b --- /dev/null +++ b/imports/testdata/mod/rsc.io_quote_v2_v2.0.1.txt @@ -0,0 +1,86 @@ +rsc.io/quote/v2@v2.0.1 + +-- .mod -- +module rsc.io/quote/v2 + +require rsc.io/sampler v1.3.0 +-- .info -- +{"Version":"v2.0.1","Name":"754f68430672776c84704e2d10209a6ec700cd64","Short":"754f68430672","Time":"2018-07-09T16:25:34Z"} +-- go.mod -- +module rsc.io/quote/v2 + +require rsc.io/sampler v1.3.0 +-- quote.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package quote collects pithy sayings. +package quote // import "rsc.io/quote" + +import "rsc.io/sampler" + +// Hello returns a greeting. +func HelloV2() string { + return sampler.Hello() +} + +// Glass returns a useful phrase for world travelers. +func GlassV2() string { + // See http://www.oocities.org/nodotus/hbglass.html. + return "I can eat glass and it doesn't hurt me." +} + +// Go returns a Go proverb. +func GoV2() string { + return "Don't communicate by sharing memory, share memory by communicating." +} + +// Opt returns an optimization truth. +func OptV2() string { + // Wisdom from ken. + return "If a program is too slow, it must have a loop." +} +-- quote_test.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package quote + +import ( + "os" + "testing" +) + +func init() { + os.Setenv("LC_ALL", "en") +} + +func TestHello(t *testing.T) { + hello := "Hello, world." + if out := Hello(); out != hello { + t.Errorf("Hello() = %q, want %q", out, hello) + } +} + +func TestGlass(t *testing.T) { + glass := "I can eat glass and it doesn't hurt me." + if out := Glass(); out != glass { + t.Errorf("Glass() = %q, want %q", out, glass) + } +} + +func TestGo(t *testing.T) { + go1 := "Don't communicate by sharing memory, share memory by communicating." + if out := Go(); out != go1 { + t.Errorf("Go() = %q, want %q", out, go1) + } +} + +func TestOpt(t *testing.T) { + opt := "If a program is too slow, it must have a loop." + if out := Opt(); out != opt { + t.Errorf("Opt() = %q, want %q", out, opt) + } +} diff --git a/imports/testdata/mod/rsc.io_quote_v3_v3.0.0.txt b/imports/testdata/mod/rsc.io_quote_v3_v3.0.0.txt new file mode 100644 index 0000000000..0afe1f0519 --- /dev/null +++ b/imports/testdata/mod/rsc.io_quote_v3_v3.0.0.txt @@ -0,0 +1,45 @@ +rsc.io/quote/v3@v3.0.0 + +-- .mod -- +module rsc.io/quote/v3 + +require rsc.io/sampler v1.3.0 + +-- .info -- +{"Version":"v3.0.0","Name":"d88915d7e77ed0fd35d0a022a2f244e2202fd8c8","Short":"d88915d7e77e","Time":"2018-07-09T15:34:46Z"} +-- go.mod -- +module rsc.io/quote/v3 + +require rsc.io/sampler v1.3.0 + +-- quote.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package quote collects pithy sayings. +package quote // import "rsc.io/quote" + +import "rsc.io/sampler" + +// Hello returns a greeting. +func HelloV3() string { + return sampler.Hello() +} + +// Glass returns a useful phrase for world travelers. +func GlassV3() string { + // See http://www.oocities.org/nodotus/hbglass.html. + return "I can eat glass and it doesn't hurt me." +} + +// Go returns a Go proverb. +func GoV3() string { + return "Don't communicate by sharing memory, share memory by communicating." +} + +// Opt returns an optimization truth. +func OptV3() string { + // Wisdom from ken. + return "If a program is too slow, it must have a loop." +} diff --git a/imports/testdata/mod/rsc.io_sampler_v1.3.0.txt b/imports/testdata/mod/rsc.io_sampler_v1.3.0.txt new file mode 100644 index 0000000000..febe51fd9a --- /dev/null +++ b/imports/testdata/mod/rsc.io_sampler_v1.3.0.txt @@ -0,0 +1,202 @@ +rsc.io/sampler@v1.3.0 + +-- .mod -- +module "rsc.io/sampler" + +require "golang.org/x/text" v0.0.0-20170915032832-14c0d48ead0c +-- .info -- +{"Version":"v1.3.0","Name":"0cc034b51e57ed7832d4c67d526f75a900996e5c","Short":"0cc034b51e57","Time":"2018-02-13T19:05:03Z"} +-- glass.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Translations from Frank da Cruz, Ethan Mollick, and many others. +// See http://kermitproject.org/utf8.html. +// http://www.oocities.org/nodotus/hbglass.html +// https://en.wikipedia.org/wiki/I_Can_Eat_Glass + +package sampler + +var glass = newText(` + +English: en: I can eat glass and it doesn't hurt me. +French: fr: Je peux manger du verre, ça ne me fait pas mal. +Spanish: es: Puedo comer vidrio, no me hace daño. + +`) +-- glass_test.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sampler + +import ( + "testing" + + "golang.org/x/text/language" + _ "rsc.io/testonly" +) + +var glassTests = []struct { + prefs []language.Tag + text string +}{ + { + []language.Tag{language.Make("en-US"), language.Make("fr")}, + "I can eat glass and it doesn't hurt me.", + }, + { + []language.Tag{language.Make("fr"), language.Make("en-US")}, + "Je peux manger du verre, ça ne me fait pas mal.", + }, +} + +func TestGlass(t *testing.T) { + for _, tt := range glassTests { + text := Glass(tt.prefs...) + if text != tt.text { + t.Errorf("Glass(%v) = %q, want %q", tt.prefs, text, tt.text) + } + } +} +-- go.mod -- +module "rsc.io/sampler" + +require "golang.org/x/text" v0.0.0-20170915032832-14c0d48ead0c +-- hello.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Translations by Google Translate. + +package sampler + +var hello = newText(` + +English: en: Hello, world. +French: fr: Bonjour le monde. +Spanish: es: Hola Mundo. + +`) +-- hello_test.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sampler + +import ( + "testing" + + "golang.org/x/text/language" +) + +var helloTests = []struct { + prefs []language.Tag + text string +}{ + { + []language.Tag{language.Make("en-US"), language.Make("fr")}, + "Hello, world.", + }, + { + []language.Tag{language.Make("fr"), language.Make("en-US")}, + "Bonjour le monde.", + }, +} + +func TestHello(t *testing.T) { + for _, tt := range helloTests { + text := Hello(tt.prefs...) + if text != tt.text { + t.Errorf("Hello(%v) = %q, want %q", tt.prefs, text, tt.text) + } + } +} +-- sampler.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sampler shows simple texts. +package sampler // import "rsc.io/sampler" + +import ( + "os" + "strings" + + "golang.org/x/text/language" +) + +// DefaultUserPrefs returns the default user language preferences. +// It consults the $LC_ALL, $LC_MESSAGES, and $LANG environment +// variables, in that order. +func DefaultUserPrefs() []language.Tag { + var prefs []language.Tag + for _, k := range []string{"LC_ALL", "LC_MESSAGES", "LANG"} { + if env := os.Getenv(k); env != "" { + prefs = append(prefs, language.Make(env)) + } + } + return prefs +} + +// Hello returns a localized greeting. +// If no prefs are given, Hello uses DefaultUserPrefs. +func Hello(prefs ...language.Tag) string { + if len(prefs) == 0 { + prefs = DefaultUserPrefs() + } + return hello.find(prefs) +} + +// Glass returns a localized silly phrase. +// If no prefs are given, Glass uses DefaultUserPrefs. +func Glass(prefs ...language.Tag) string { + if len(prefs) == 0 { + prefs = DefaultUserPrefs() + } + return glass.find(prefs) +} + +// A text is a localized text. +type text struct { + byTag map[string]string + matcher language.Matcher +} + +// newText creates a new localized text, given a list of translations. +func newText(s string) *text { + t := &text{ + byTag: make(map[string]string), + } + var tags []language.Tag + for _, line := range strings.Split(s, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + f := strings.Split(line, ": ") + if len(f) != 3 { + continue + } + tag := language.Make(f[1]) + tags = append(tags, tag) + t.byTag[tag.String()] = f[2] + } + t.matcher = language.NewMatcher(tags) + return t +} + +// find finds the text to use for the given language tag preferences. +func (t *text) find(prefs []language.Tag) string { + tag, _, _ := t.matcher.Match(prefs...) + s := t.byTag[tag.String()] + if strings.HasPrefix(s, "RTL ") { + s = "\u200F" + strings.TrimPrefix(s, "RTL ") + "\u200E" + } + return s +} diff --git a/imports/testdata/mod/rsc.io_sampler_v1.3.1.txt b/imports/testdata/mod/rsc.io_sampler_v1.3.1.txt new file mode 100644 index 0000000000..a293f10869 --- /dev/null +++ b/imports/testdata/mod/rsc.io_sampler_v1.3.1.txt @@ -0,0 +1,201 @@ +rsc.io/sampler@v1.3.1 + +-- .mod -- +module "rsc.io/sampler" + +require "golang.org/x/text" v0.0.0-20170915032832-14c0d48ead0c +-- .info -- +{"Version":"v1.3.1","Name":"f545d0289d06e2add4556ea6a15fc4938014bf87","Short":"f545d0289d06","Time":"2018-02-14T16:34:12Z"} +-- glass.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Translations from Frank da Cruz, Ethan Mollick, and many others. +// See http://kermitproject.org/utf8.html. +// http://www.oocities.org/nodotus/hbglass.html +// https://en.wikipedia.org/wiki/I_Can_Eat_Glass + +package sampler + +var glass = newText(` + +English: en: I can eat glass and it doesn't hurt me. +French: fr: Je peux manger du verre, ça ne me fait pas mal. +Spanish: es: Puedo comer vidrio, no me hace daño. + +`) +-- glass_test.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sampler + +import ( + "testing" + + "golang.org/x/text/language" +) + +var glassTests = []struct { + prefs []language.Tag + text string +}{ + { + []language.Tag{language.Make("en-US"), language.Make("fr")}, + "I can eat glass and it doesn't hurt me.", + }, + { + []language.Tag{language.Make("fr"), language.Make("en-US")}, + "Je peux manger du verre, ça ne me fait pas mal.", + }, +} + +func TestGlass(t *testing.T) { + for _, tt := range glassTests { + text := Glass(tt.prefs...) + if text != tt.text { + t.Errorf("Glass(%v) = %q, want %q", tt.prefs, text, tt.text) + } + } +} +-- go.mod -- +module "rsc.io/sampler" + +require "golang.org/x/text" v0.0.0-20170915032832-14c0d48ead0c +-- hello.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Translations by Google Translate. + +package sampler + +var hello = newText(` + +English: en: Hello, world. +French: fr: Bonjour le monde. +Spanish: es: Hola Mundo. + +`) +-- hello_test.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sampler + +import ( + "testing" + + "golang.org/x/text/language" +) + +var helloTests = []struct { + prefs []language.Tag + text string +}{ + { + []language.Tag{language.Make("en-US"), language.Make("fr")}, + "Hello, world.", + }, + { + []language.Tag{language.Make("fr"), language.Make("en-US")}, + "Bonjour le monde.", + }, +} + +func TestHello(t *testing.T) { + for _, tt := range helloTests { + text := Hello(tt.prefs...) + if text != tt.text { + t.Errorf("Hello(%v) = %q, want %q", tt.prefs, text, tt.text) + } + } +} +-- sampler.go -- +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package sampler shows simple texts in a variety of languages. +package sampler // import "rsc.io/sampler" + +import ( + "os" + "strings" + + "golang.org/x/text/language" +) + +// DefaultUserPrefs returns the default user language preferences. +// It consults the $LC_ALL, $LC_MESSAGES, and $LANG environment +// variables, in that order. +func DefaultUserPrefs() []language.Tag { + var prefs []language.Tag + for _, k := range []string{"LC_ALL", "LC_MESSAGES", "LANG"} { + if env := os.Getenv(k); env != "" { + prefs = append(prefs, language.Make(env)) + } + } + return prefs +} + +// Hello returns a localized greeting. +// If no prefs are given, Hello uses DefaultUserPrefs. +func Hello(prefs ...language.Tag) string { + if len(prefs) == 0 { + prefs = DefaultUserPrefs() + } + return hello.find(prefs) +} + +// Glass returns a localized silly phrase. +// If no prefs are given, Glass uses DefaultUserPrefs. +func Glass(prefs ...language.Tag) string { + if len(prefs) == 0 { + prefs = DefaultUserPrefs() + } + return glass.find(prefs) +} + +// A text is a localized text. +type text struct { + byTag map[string]string + matcher language.Matcher +} + +// newText creates a new localized text, given a list of translations. +func newText(s string) *text { + t := &text{ + byTag: make(map[string]string), + } + var tags []language.Tag + for _, line := range strings.Split(s, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + f := strings.Split(line, ": ") + if len(f) != 3 { + continue + } + tag := language.Make(f[1]) + tags = append(tags, tag) + t.byTag[tag.String()] = f[2] + } + t.matcher = language.NewMatcher(tags) + return t +} + +// find finds the text to use for the given language tag preferences. +func (t *text) find(prefs []language.Tag) string { + tag, _, _ := t.matcher.Match(prefs...) + s := t.byTag[tag.String()] + if strings.HasPrefix(s, "RTL ") { + s = "\u200F" + strings.TrimPrefix(s, "RTL ") + "\u200E" + } + return s +} diff --git a/internal/gopathwalk/walk.go b/internal/gopathwalk/walk.go index 488088b573..04bb96a362 100644 --- a/internal/gopathwalk/walk.go +++ b/internal/gopathwalk/walk.go @@ -35,6 +35,7 @@ const ( RootGOPATH RootCurrentModule RootModuleCache + RootOther ) // A Root is a starting point for a Walk. @@ -162,7 +163,7 @@ func (w *walker) shouldSkipDir(fi os.FileInfo) bool { func (w *walker) walk(path string, typ os.FileMode) error { dir := filepath.Dir(path) if typ.IsRegular() { - if dir == w.root.Path { + if dir == w.root.Path && (w.root.Type == RootGOROOT || w.root.Type == RootGOPATH) { // Doesn't make sense to have regular files // directly in your $GOPATH/src or $GOROOT/src. return fastwalk.SkipFiles diff --git a/internal/module/module.go b/internal/module/module.go new file mode 100644 index 0000000000..9a4edb9dec --- /dev/null +++ b/internal/module/module.go @@ -0,0 +1,540 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package module defines the module.Version type +// along with support code. +package module + +// IMPORTANT NOTE +// +// This file essentially defines the set of valid import paths for the go command. +// There are many subtle considerations, including Unicode ambiguity, +// security, network, and file system representations. +// +// This file also defines the set of valid module path and version combinations, +// another topic with many subtle considerations. +// +// Changes to the semantics in this file require approval from rsc. + +import ( + "fmt" + "sort" + "strings" + "unicode" + "unicode/utf8" + + "golang.org/x/tools/internal/semver" +) + +// A Version is defined by a module path and version pair. +type Version struct { + Path string + + // Version is usually a semantic version in canonical form. + // There are two exceptions to this general rule. + // First, the top-level target of a build has no specific version + // and uses Version = "". + // Second, during MVS calculations the version "none" is used + // to represent the decision to take no version of a given module. + Version string `json:",omitempty"` +} + +// Check checks that a given module path, version pair is valid. +// In addition to the path being a valid module path +// and the version being a valid semantic version, +// the two must correspond. +// For example, the path "yaml/v2" only corresponds to +// semantic versions beginning with "v2.". +func Check(path, version string) error { + if err := CheckPath(path); err != nil { + return err + } + if !semver.IsValid(version) { + return fmt.Errorf("malformed semantic version %v", version) + } + _, pathMajor, _ := SplitPathVersion(path) + if !MatchPathMajor(version, pathMajor) { + if pathMajor == "" { + pathMajor = "v0 or v1" + } + if pathMajor[0] == '.' { // .v1 + pathMajor = pathMajor[1:] + } + return fmt.Errorf("mismatched module path %v and version %v (want %v)", path, version, pathMajor) + } + return nil +} + +// firstPathOK reports whether r can appear in the first element of a module path. +// The first element of the path must be an LDH domain name, at least for now. +// To avoid case ambiguity, the domain name must be entirely lower case. +func firstPathOK(r rune) bool { + return r == '-' || r == '.' || + '0' <= r && r <= '9' || + 'a' <= r && r <= 'z' +} + +// pathOK reports whether r can appear in an import path element. +// Paths can be ASCII letters, ASCII digits, and limited ASCII punctuation: + - . _ and ~. +// This matches what "go get" has historically recognized in import paths. +// TODO(rsc): We would like to allow Unicode letters, but that requires additional +// care in the safe encoding (see note below). +func pathOK(r rune) bool { + if r < utf8.RuneSelf { + return r == '+' || r == '-' || r == '.' || r == '_' || r == '~' || + '0' <= r && r <= '9' || + 'A' <= r && r <= 'Z' || + 'a' <= r && r <= 'z' + } + return false +} + +// fileNameOK reports whether r can appear in a file name. +// For now we allow all Unicode letters but otherwise limit to pathOK plus a few more punctuation characters. +// If we expand the set of allowed characters here, we have to +// work harder at detecting potential case-folding and normalization collisions. +// See note about "safe encoding" below. +func fileNameOK(r rune) bool { + if r < utf8.RuneSelf { + // Entire set of ASCII punctuation, from which we remove characters: + // ! " # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \ ] ^ _ ` { | } ~ + // We disallow some shell special characters: " ' * < > ? ` | + // (Note that some of those are disallowed by the Windows file system as well.) + // We also disallow path separators / : and \ (fileNameOK is only called on path element characters). + // We allow spaces (U+0020) in file names. + const allowed = "!#$%&()+,-.=@[]^_{}~ " + if '0' <= r && r <= '9' || 'A' <= r && r <= 'Z' || 'a' <= r && r <= 'z' { + return true + } + for i := 0; i < len(allowed); i++ { + if rune(allowed[i]) == r { + return true + } + } + return false + } + // It may be OK to add more ASCII punctuation here, but only carefully. + // For example Windows disallows < > \, and macOS disallows :, so we must not allow those. + return unicode.IsLetter(r) +} + +// CheckPath checks that a module path is valid. +func CheckPath(path string) error { + if err := checkPath(path, false); err != nil { + return fmt.Errorf("malformed module path %q: %v", path, err) + } + i := strings.Index(path, "/") + if i < 0 { + i = len(path) + } + if i == 0 { + return fmt.Errorf("malformed module path %q: leading slash", path) + } + if !strings.Contains(path[:i], ".") { + return fmt.Errorf("malformed module path %q: missing dot in first path element", path) + } + if path[0] == '-' { + return fmt.Errorf("malformed module path %q: leading dash in first path element", path) + } + for _, r := range path[:i] { + if !firstPathOK(r) { + return fmt.Errorf("malformed module path %q: invalid char %q in first path element", path, r) + } + } + if _, _, ok := SplitPathVersion(path); !ok { + return fmt.Errorf("malformed module path %q: invalid version", path) + } + return nil +} + +// CheckImportPath checks that an import path is valid. +func CheckImportPath(path string) error { + if err := checkPath(path, false); err != nil { + return fmt.Errorf("malformed import path %q: %v", path, err) + } + return nil +} + +// checkPath checks that a general path is valid. +// It returns an error describing why but not mentioning path. +// Because these checks apply to both module paths and import paths, +// the caller is expected to add the "malformed ___ path %q: " prefix. +// fileName indicates whether the final element of the path is a file name +// (as opposed to a directory name). +func checkPath(path string, fileName bool) error { + if !utf8.ValidString(path) { + return fmt.Errorf("invalid UTF-8") + } + if path == "" { + return fmt.Errorf("empty string") + } + if strings.Contains(path, "..") { + return fmt.Errorf("double dot") + } + if strings.Contains(path, "//") { + return fmt.Errorf("double slash") + } + if path[len(path)-1] == '/' { + return fmt.Errorf("trailing slash") + } + elemStart := 0 + for i, r := range path { + if r == '/' { + if err := checkElem(path[elemStart:i], fileName); err != nil { + return err + } + elemStart = i + 1 + } + } + if err := checkElem(path[elemStart:], fileName); err != nil { + return err + } + return nil +} + +// checkElem checks whether an individual path element is valid. +// fileName indicates whether the element is a file name (not a directory name). +func checkElem(elem string, fileName bool) error { + if elem == "" { + return fmt.Errorf("empty path element") + } + if strings.Count(elem, ".") == len(elem) { + return fmt.Errorf("invalid path element %q", elem) + } + if elem[0] == '.' && !fileName { + return fmt.Errorf("leading dot in path element") + } + if elem[len(elem)-1] == '.' { + return fmt.Errorf("trailing dot in path element") + } + charOK := pathOK + if fileName { + charOK = fileNameOK + } + for _, r := range elem { + if !charOK(r) { + return fmt.Errorf("invalid char %q", r) + } + } + + // Windows disallows a bunch of path elements, sadly. + // See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file + short := elem + if i := strings.Index(short, "."); i >= 0 { + short = short[:i] + } + for _, bad := range badWindowsNames { + if strings.EqualFold(bad, short) { + return fmt.Errorf("disallowed path element %q", elem) + } + } + return nil +} + +// CheckFilePath checks whether a slash-separated file path is valid. +func CheckFilePath(path string) error { + if err := checkPath(path, true); err != nil { + return fmt.Errorf("malformed file path %q: %v", path, err) + } + return nil +} + +// badWindowsNames are the reserved file path elements on Windows. +// See https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file +var badWindowsNames = []string{ + "CON", + "PRN", + "AUX", + "NUL", + "COM1", + "COM2", + "COM3", + "COM4", + "COM5", + "COM6", + "COM7", + "COM8", + "COM9", + "LPT1", + "LPT2", + "LPT3", + "LPT4", + "LPT5", + "LPT6", + "LPT7", + "LPT8", + "LPT9", +} + +// SplitPathVersion returns prefix and major version such that prefix+pathMajor == path +// and version is either empty or "/vN" for N >= 2. +// As a special case, gopkg.in paths are recognized directly; +// they require ".vN" instead of "/vN", and for all N, not just N >= 2. +func SplitPathVersion(path string) (prefix, pathMajor string, ok bool) { + if strings.HasPrefix(path, "gopkg.in/") { + return splitGopkgIn(path) + } + + i := len(path) + dot := false + for i > 0 && ('0' <= path[i-1] && path[i-1] <= '9' || path[i-1] == '.') { + if path[i-1] == '.' { + dot = true + } + i-- + } + if i <= 1 || i == len(path) || path[i-1] != 'v' || path[i-2] != '/' { + return path, "", true + } + prefix, pathMajor = path[:i-2], path[i-2:] + if dot || len(pathMajor) <= 2 || pathMajor[2] == '0' || pathMajor == "/v1" { + return path, "", false + } + return prefix, pathMajor, true +} + +// splitGopkgIn is like SplitPathVersion but only for gopkg.in paths. +func splitGopkgIn(path string) (prefix, pathMajor string, ok bool) { + if !strings.HasPrefix(path, "gopkg.in/") { + return path, "", false + } + i := len(path) + if strings.HasSuffix(path, "-unstable") { + i -= len("-unstable") + } + for i > 0 && ('0' <= path[i-1] && path[i-1] <= '9') { + i-- + } + if i <= 1 || path[i-1] != 'v' || path[i-2] != '.' { + // All gopkg.in paths must end in vN for some N. + return path, "", false + } + prefix, pathMajor = path[:i-2], path[i-2:] + if len(pathMajor) <= 2 || pathMajor[2] == '0' && pathMajor != ".v0" { + return path, "", false + } + return prefix, pathMajor, true +} + +// MatchPathMajor reports whether the semantic version v +// matches the path major version pathMajor. +func MatchPathMajor(v, pathMajor string) bool { + if strings.HasPrefix(pathMajor, ".v") && strings.HasSuffix(pathMajor, "-unstable") { + pathMajor = strings.TrimSuffix(pathMajor, "-unstable") + } + if strings.HasPrefix(v, "v0.0.0-") && pathMajor == ".v1" { + // Allow old bug in pseudo-versions that generated v0.0.0- pseudoversion for gopkg .v1. + // For example, gopkg.in/yaml.v2@v2.2.1's go.mod requires gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405. + return true + } + m := semver.Major(v) + if pathMajor == "" { + return m == "v0" || m == "v1" || semver.Build(v) == "+incompatible" + } + return (pathMajor[0] == '/' || pathMajor[0] == '.') && m == pathMajor[1:] +} + +// CanonicalVersion returns the canonical form of the version string v. +// It is the same as semver.Canonical(v) except that it preserves the special build suffix "+incompatible". +func CanonicalVersion(v string) string { + cv := semver.Canonical(v) + if semver.Build(v) == "+incompatible" { + cv += "+incompatible" + } + return cv +} + +// Sort sorts the list by Path, breaking ties by comparing Versions. +func Sort(list []Version) { + sort.Slice(list, func(i, j int) bool { + mi := list[i] + mj := list[j] + if mi.Path != mj.Path { + return mi.Path < mj.Path + } + // To help go.sum formatting, allow version/file. + // Compare semver prefix by semver rules, + // file by string order. + vi := mi.Version + vj := mj.Version + var fi, fj string + if k := strings.Index(vi, "/"); k >= 0 { + vi, fi = vi[:k], vi[k:] + } + if k := strings.Index(vj, "/"); k >= 0 { + vj, fj = vj[:k], vj[k:] + } + if vi != vj { + return semver.Compare(vi, vj) < 0 + } + return fi < fj + }) +} + +// Safe encodings +// +// Module paths appear as substrings of file system paths +// (in the download cache) and of web server URLs in the proxy protocol. +// In general we cannot rely on file systems to be case-sensitive, +// nor can we rely on web servers, since they read from file systems. +// That is, we cannot rely on the file system to keep rsc.io/QUOTE +// and rsc.io/quote separate. Windows and macOS don't. +// Instead, we must never require two different casings of a file path. +// Because we want the download cache to match the proxy protocol, +// and because we want the proxy protocol to be possible to serve +// from a tree of static files (which might be stored on a case-insensitive +// file system), the proxy protocol must never require two different casings +// of a URL path either. +// +// One possibility would be to make the safe encoding be the lowercase +// hexadecimal encoding of the actual path bytes. This would avoid ever +// needing different casings of a file path, but it would be fairly illegible +// to most programmers when those paths appeared in the file system +// (including in file paths in compiler errors and stack traces) +// in web server logs, and so on. Instead, we want a safe encoding that +// leaves most paths unaltered. +// +// The safe encoding is this: +// replace every uppercase letter with an exclamation mark +// followed by the letter's lowercase equivalent. +// +// For example, +// github.com/Azure/azure-sdk-for-go -> github.com/!azure/azure-sdk-for-go. +// github.com/GoogleCloudPlatform/cloudsql-proxy -> github.com/!google!cloud!platform/cloudsql-proxy +// github.com/Sirupsen/logrus -> github.com/!sirupsen/logrus. +// +// Import paths that avoid upper-case letters are left unchanged. +// Note that because import paths are ASCII-only and avoid various +// problematic punctuation (like : < and >), the safe encoding is also ASCII-only +// and avoids the same problematic punctuation. +// +// Import paths have never allowed exclamation marks, so there is no +// need to define how to encode a literal !. +// +// Although paths are disallowed from using Unicode (see pathOK above), +// the eventual plan is to allow Unicode letters as well, to assume that +// file systems and URLs are Unicode-safe (storing UTF-8), and apply +// the !-for-uppercase convention. Note however that not all runes that +// are different but case-fold equivalent are an upper/lower pair. +// For example, U+004B ('K'), U+006B ('k'), and U+212A ('K' for Kelvin) +// are considered to case-fold to each other. When we do add Unicode +// letters, we must not assume that upper/lower are the only case-equivalent pairs. +// Perhaps the Kelvin symbol would be disallowed entirely, for example. +// Or perhaps it would encode as "!!k", or perhaps as "(212A)". +// +// Also, it would be nice to allow Unicode marks as well as letters, +// but marks include combining marks, and then we must deal not +// only with case folding but also normalization: both U+00E9 ('é') +// and U+0065 U+0301 ('e' followed by combining acute accent) +// look the same on the page and are treated by some file systems +// as the same path. If we do allow Unicode marks in paths, there +// must be some kind of normalization to allow only one canonical +// encoding of any character used in an import path. + +// EncodePath returns the safe encoding of the given module path. +// It fails if the module path is invalid. +func EncodePath(path string) (encoding string, err error) { + if err := CheckPath(path); err != nil { + return "", err + } + + return encodeString(path) +} + +// EncodeVersion returns the safe encoding of the given module version. +// Versions are allowed to be in non-semver form but must be valid file names +// and not contain exclamation marks. +func EncodeVersion(v string) (encoding string, err error) { + if err := checkElem(v, true); err != nil || strings.Contains(v, "!") { + return "", fmt.Errorf("disallowed version string %q", v) + } + return encodeString(v) +} + +func encodeString(s string) (encoding string, err error) { + haveUpper := false + for _, r := range s { + if r == '!' || r >= utf8.RuneSelf { + // This should be disallowed by CheckPath, but diagnose anyway. + // The correctness of the encoding loop below depends on it. + return "", fmt.Errorf("internal error: inconsistency in EncodePath") + } + if 'A' <= r && r <= 'Z' { + haveUpper = true + } + } + + if !haveUpper { + return s, nil + } + + var buf []byte + for _, r := range s { + if 'A' <= r && r <= 'Z' { + buf = append(buf, '!', byte(r+'a'-'A')) + } else { + buf = append(buf, byte(r)) + } + } + return string(buf), nil +} + +// DecodePath returns the module path of the given safe encoding. +// It fails if the encoding is invalid or encodes an invalid path. +func DecodePath(encoding string) (path string, err error) { + path, ok := decodeString(encoding) + if !ok { + return "", fmt.Errorf("invalid module path encoding %q", encoding) + } + if err := CheckPath(path); err != nil { + return "", fmt.Errorf("invalid module path encoding %q: %v", encoding, err) + } + return path, nil +} + +// DecodeVersion returns the version string for the given safe encoding. +// It fails if the encoding is invalid or encodes an invalid version. +// Versions are allowed to be in non-semver form but must be valid file names +// and not contain exclamation marks. +func DecodeVersion(encoding string) (v string, err error) { + v, ok := decodeString(encoding) + if !ok { + return "", fmt.Errorf("invalid version encoding %q", encoding) + } + if err := checkElem(v, true); err != nil { + return "", fmt.Errorf("disallowed version string %q", v) + } + return v, nil +} + +func decodeString(encoding string) (string, bool) { + var buf []byte + + bang := false + for _, r := range encoding { + if r >= utf8.RuneSelf { + return "", false + } + if bang { + bang = false + if r < 'a' || 'z' < r { + return "", false + } + buf = append(buf, byte(r+'A'-'a')) + continue + } + if r == '!' { + bang = true + continue + } + if 'A' <= r && r <= 'Z' { + return "", false + } + buf = append(buf, byte(r)) + } + if bang { + return "", false + } + return string(buf), true +} diff --git a/internal/module/module_test.go b/internal/module/module_test.go new file mode 100644 index 0000000000..b40bd03dfa --- /dev/null +++ b/internal/module/module_test.go @@ -0,0 +1,319 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package module + +import "testing" + +var checkTests = []struct { + path string + version string + ok bool +}{ + {"rsc.io/quote", "0.1.0", false}, + {"rsc io/quote", "v1.0.0", false}, + + {"github.com/go-yaml/yaml", "v0.8.0", true}, + {"github.com/go-yaml/yaml", "v1.0.0", true}, + {"github.com/go-yaml/yaml", "v2.0.0", false}, + {"github.com/go-yaml/yaml", "v2.1.5", false}, + {"github.com/go-yaml/yaml", "v3.0.0", false}, + + {"github.com/go-yaml/yaml/v2", "v1.0.0", false}, + {"github.com/go-yaml/yaml/v2", "v2.0.0", true}, + {"github.com/go-yaml/yaml/v2", "v2.1.5", true}, + {"github.com/go-yaml/yaml/v2", "v3.0.0", false}, + + {"gopkg.in/yaml.v0", "v0.8.0", true}, + {"gopkg.in/yaml.v0", "v1.0.0", false}, + {"gopkg.in/yaml.v0", "v2.0.0", false}, + {"gopkg.in/yaml.v0", "v2.1.5", false}, + {"gopkg.in/yaml.v0", "v3.0.0", false}, + + {"gopkg.in/yaml.v1", "v0.8.0", false}, + {"gopkg.in/yaml.v1", "v1.0.0", true}, + {"gopkg.in/yaml.v1", "v2.0.0", false}, + {"gopkg.in/yaml.v1", "v2.1.5", false}, + {"gopkg.in/yaml.v1", "v3.0.0", false}, + + // For gopkg.in, .v1 means v1 only (not v0). + // But early versions of vgo still generated v0 pseudo-versions for it. + // Even though now we'd generate those as v1 pseudo-versions, + // we accept the old pseudo-versions to avoid breaking existing go.mod files. + // For example gopkg.in/yaml.v2@v2.2.1's go.mod requires check.v1 at a v0 pseudo-version. + {"gopkg.in/check.v1", "v0.0.0", false}, + {"gopkg.in/check.v1", "v0.0.0-20160102150405-abcdef123456", true}, + + {"gopkg.in/yaml.v2", "v1.0.0", false}, + {"gopkg.in/yaml.v2", "v2.0.0", true}, + {"gopkg.in/yaml.v2", "v2.1.5", true}, + {"gopkg.in/yaml.v2", "v3.0.0", false}, + + {"rsc.io/quote", "v17.0.0", false}, + {"rsc.io/quote", "v17.0.0+incompatible", true}, +} + +func TestCheck(t *testing.T) { + for _, tt := range checkTests { + err := Check(tt.path, tt.version) + if tt.ok && err != nil { + t.Errorf("Check(%q, %q) = %v, wanted nil error", tt.path, tt.version, err) + } else if !tt.ok && err == nil { + t.Errorf("Check(%q, %q) succeeded, wanted error", tt.path, tt.version) + } + } +} + +var checkPathTests = []struct { + path string + ok bool + importOK bool + fileOK bool +}{ + {"x.y/z", true, true, true}, + {"x.y", true, true, true}, + + {"", false, false, false}, + {"x.y/\xFFz", false, false, false}, + {"/x.y/z", false, false, false}, + {"x./z", false, false, false}, + {".x/z", false, false, true}, + {"-x/z", false, true, true}, + {"x..y/z", false, false, false}, + {"x.y/z/../../w", false, false, false}, + {"x.y//z", false, false, false}, + {"x.y/z//w", false, false, false}, + {"x.y/z/", false, false, false}, + + {"x.y/z/v0", false, true, true}, + {"x.y/z/v1", false, true, true}, + {"x.y/z/v2", true, true, true}, + {"x.y/z/v2.0", false, true, true}, + {"X.y/z", false, true, true}, + + {"!x.y/z", false, false, true}, + {"_x.y/z", false, true, true}, + {"x.y!/z", false, false, true}, + {"x.y\"/z", false, false, false}, + {"x.y#/z", false, false, true}, + {"x.y$/z", false, false, true}, + {"x.y%/z", false, false, true}, + {"x.y&/z", false, false, true}, + {"x.y'/z", false, false, false}, + {"x.y(/z", false, false, true}, + {"x.y)/z", false, false, true}, + {"x.y*/z", false, false, false}, + {"x.y+/z", false, true, true}, + {"x.y,/z", false, false, true}, + {"x.y-/z", true, true, true}, + {"x.y./zt", false, false, false}, + {"x.y:/z", false, false, false}, + {"x.y;/z", false, false, false}, + {"x.y/z", false, false, false}, + {"x.y?/z", false, false, false}, + {"x.y@/z", false, false, true}, + {"x.y[/z", false, false, true}, + {"x.y\\/z", false, false, false}, + {"x.y]/z", false, false, true}, + {"x.y^/z", false, false, true}, + {"x.y_/z", false, true, true}, + {"x.y`/z", false, false, false}, + {"x.y{/z", false, false, true}, + {"x.y}/z", false, false, true}, + {"x.y~/z", false, true, true}, + {"x.y/z!", false, false, true}, + {"x.y/z\"", false, false, false}, + {"x.y/z#", false, false, true}, + {"x.y/z$", false, false, true}, + {"x.y/z%", false, false, true}, + {"x.y/z&", false, false, true}, + {"x.y/z'", false, false, false}, + {"x.y/z(", false, false, true}, + {"x.y/z)", false, false, true}, + {"x.y/z*", false, false, false}, + {"x.y/z+", true, true, true}, + {"x.y/z,", false, false, true}, + {"x.y/z-", true, true, true}, + {"x.y/z.t", true, true, true}, + {"x.y/z/t", true, true, true}, + {"x.y/z:", false, false, false}, + {"x.y/z;", false, false, false}, + {"x.y/z<", false, false, false}, + {"x.y/z=", false, false, true}, + {"x.y/z>", false, false, false}, + {"x.y/z?", false, false, false}, + {"x.y/z@", false, false, true}, + {"x.y/z[", false, false, true}, + {"x.y/z\\", false, false, false}, + {"x.y/z]", false, false, true}, + {"x.y/z^", false, false, true}, + {"x.y/z_", true, true, true}, + {"x.y/z`", false, false, false}, + {"x.y/z{", false, false, true}, + {"x.y/z}", false, false, true}, + {"x.y/z~", true, true, true}, + {"x.y/x.foo", true, true, true}, + {"x.y/aux.foo", false, false, false}, + {"x.y/prn", false, false, false}, + {"x.y/prn2", true, true, true}, + {"x.y/com", true, true, true}, + {"x.y/com1", false, false, false}, + {"x.y/com1.txt", false, false, false}, + {"x.y/calm1", true, true, true}, + {"github.com/!123/logrus", false, false, true}, + + // TODO: CL 41822 allowed Unicode letters in old "go get" + // without due consideration of the implications, and only on github.com (!). + // For now, we disallow non-ASCII characters in module mode, + // in both module paths and general import paths, + // until we can get the implications right. + // When we do, we'll enable them everywhere, not just for GitHub. + {"github.com/user/unicode/испытание", false, false, true}, + + {"../x", false, false, false}, + {"./y", false, false, false}, + {"x:y", false, false, false}, + {`\temp\foo`, false, false, false}, + {".gitignore", false, false, true}, + {".github/ISSUE_TEMPLATE", false, false, true}, + {"x☺y", false, false, false}, +} + +func TestCheckPath(t *testing.T) { + for _, tt := range checkPathTests { + err := CheckPath(tt.path) + if tt.ok && err != nil { + t.Errorf("CheckPath(%q) = %v, wanted nil error", tt.path, err) + } else if !tt.ok && err == nil { + t.Errorf("CheckPath(%q) succeeded, wanted error", tt.path) + } + + err = CheckImportPath(tt.path) + if tt.importOK && err != nil { + t.Errorf("CheckImportPath(%q) = %v, wanted nil error", tt.path, err) + } else if !tt.importOK && err == nil { + t.Errorf("CheckImportPath(%q) succeeded, wanted error", tt.path) + } + + err = CheckFilePath(tt.path) + if tt.fileOK && err != nil { + t.Errorf("CheckFilePath(%q) = %v, wanted nil error", tt.path, err) + } else if !tt.fileOK && err == nil { + t.Errorf("CheckFilePath(%q) succeeded, wanted error", tt.path) + } + } +} + +var splitPathVersionTests = []struct { + pathPrefix string + version string +}{ + {"x.y/z", ""}, + {"x.y/z", "/v2"}, + {"x.y/z", "/v3"}, + {"x.y/v", ""}, + {"gopkg.in/yaml", ".v0"}, + {"gopkg.in/yaml", ".v1"}, + {"gopkg.in/yaml", ".v2"}, + {"gopkg.in/yaml", ".v3"}, +} + +func TestSplitPathVersion(t *testing.T) { + for _, tt := range splitPathVersionTests { + pathPrefix, version, ok := SplitPathVersion(tt.pathPrefix + tt.version) + if pathPrefix != tt.pathPrefix || version != tt.version || !ok { + t.Errorf("SplitPathVersion(%q) = %q, %q, %v, want %q, %q, true", tt.pathPrefix+tt.version, pathPrefix, version, ok, tt.pathPrefix, tt.version) + } + } + + for _, tt := range checkPathTests { + pathPrefix, version, ok := SplitPathVersion(tt.path) + if pathPrefix+version != tt.path { + t.Errorf("SplitPathVersion(%q) = %q, %q, %v, doesn't add to input", tt.path, pathPrefix, version, ok) + } + } +} + +var encodeTests = []struct { + path string + enc string // empty means same as path +}{ + {path: "ascii.com/abcdefghijklmnopqrstuvwxyz.-+/~_0123456789"}, + {path: "github.com/GoogleCloudPlatform/omega", enc: "github.com/!google!cloud!platform/omega"}, +} + +func TestEncodePath(t *testing.T) { + // Check invalid paths. + for _, tt := range checkPathTests { + if !tt.ok { + _, err := EncodePath(tt.path) + if err == nil { + t.Errorf("EncodePath(%q): succeeded, want error (invalid path)", tt.path) + } + } + } + + // Check encodings. + for _, tt := range encodeTests { + enc, err := EncodePath(tt.path) + if err != nil { + t.Errorf("EncodePath(%q): unexpected error: %v", tt.path, err) + continue + } + want := tt.enc + if want == "" { + want = tt.path + } + if enc != want { + t.Errorf("EncodePath(%q) = %q, want %q", tt.path, enc, want) + } + } +} + +var badDecode = []string{ + "github.com/GoogleCloudPlatform/omega", + "github.com/!google!cloud!platform!/omega", + "github.com/!0google!cloud!platform/omega", + "github.com/!_google!cloud!platform/omega", + "github.com/!!google!cloud!platform/omega", + "", +} + +func TestDecodePath(t *testing.T) { + // Check invalid decodings. + for _, bad := range badDecode { + _, err := DecodePath(bad) + if err == nil { + t.Errorf("DecodePath(%q): succeeded, want error (invalid decoding)", bad) + } + } + + // Check invalid paths (or maybe decodings). + for _, tt := range checkPathTests { + if !tt.ok { + path, err := DecodePath(tt.path) + if err == nil { + t.Errorf("DecodePath(%q) = %q, want error (invalid path)", tt.path, path) + } + } + } + + // Check encodings. + for _, tt := range encodeTests { + enc := tt.enc + if enc == "" { + enc = tt.path + } + path, err := DecodePath(enc) + if err != nil { + t.Errorf("DecodePath(%q): unexpected error: %v", enc, err) + continue + } + if path != tt.path { + t.Errorf("DecodePath(%q) = %q, want %q", enc, path, tt.path) + } + } +} diff --git a/internal/txtar/archive.go b/internal/txtar/archive.go new file mode 100644 index 0000000000..c384f33bdf --- /dev/null +++ b/internal/txtar/archive.go @@ -0,0 +1,140 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package txtar implements a trivial text-based file archive format. +// +// The goals for the format are: +// +// - be trivial enough to create and edit by hand. +// - be able to store trees of text files describing go command test cases. +// - diff nicely in git history and code reviews. +// +// Non-goals include being a completely general archive format, +// storing binary data, storing file modes, storing special files like +// symbolic links, and so on. +// +// Txtar format +// +// A txtar archive is zero or more comment lines and then a sequence of file entries. +// Each file entry begins with a file marker line of the form "-- FILENAME --" +// and is followed by zero or more file content lines making up the file data. +// The comment or file content ends at the next file marker line. +// The file marker line must begin with the three-byte sequence "-- " +// and end with the three-byte sequence " --", but the enclosed +// file name can be surrounding by additional white space, +// all of which is stripped. +// +// If the txtar file is missing a trailing newline on the final line, +// parsers should consider a final newline to be present anyway. +// +// There are no possible syntax errors in a txtar archive. +package txtar + +import ( + "bytes" + "fmt" + "io/ioutil" + "strings" +) + +// An Archive is a collection of files. +type Archive struct { + Comment []byte + Files []File +} + +// A File is a single file in an archive. +type File struct { + Name string // name of file ("foo/bar.txt") + Data []byte // text content of file +} + +// Format returns the serialized form of an Archive. +// It is assumed that the Archive data structure is well-formed: +// a.Comment and all a.File[i].Data contain no file marker lines, +// and all a.File[i].Name is non-empty. +func Format(a *Archive) []byte { + var buf bytes.Buffer + buf.Write(fixNL(a.Comment)) + for _, f := range a.Files { + fmt.Fprintf(&buf, "-- %s --\n", f.Name) + buf.Write(fixNL(f.Data)) + } + return buf.Bytes() +} + +// ParseFile parses the named file as an archive. +func ParseFile(file string) (*Archive, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + return Parse(data), nil +} + +// Parse parses the serialized form of an Archive. +// The returned Archive holds slices of data. +func Parse(data []byte) *Archive { + a := new(Archive) + var name string + a.Comment, name, data = findFileMarker(data) + for name != "" { + f := File{name, nil} + f.Data, name, data = findFileMarker(data) + a.Files = append(a.Files, f) + } + return a +} + +var ( + newlineMarker = []byte("\n-- ") + marker = []byte("-- ") + markerEnd = []byte(" --") +) + +// findFileMarker finds the next file marker in data, +// extracts the file name, and returns the data before the marker, +// the file name, and the data after the marker. +// If there is no next marker, findFileMarker returns before = fixNL(data), name = "", after = nil. +func findFileMarker(data []byte) (before []byte, name string, after []byte) { + var i int + for { + if name, after = isMarker(data[i:]); name != "" { + return data[:i], name, after + } + j := bytes.Index(data[i:], newlineMarker) + if j < 0 { + return fixNL(data), "", nil + } + i += j + 1 // positioned at start of new possible marker + } +} + +// isMarker checks whether data begins with a file marker line. +// If so, it returns the name from the line and the data after the line. +// Otherwise it returns name == "" with an unspecified after. +func isMarker(data []byte) (name string, after []byte) { + if !bytes.HasPrefix(data, marker) { + return "", nil + } + if i := bytes.IndexByte(data, '\n'); i >= 0 { + data, after = data[:i], data[i+1:] + } + if !bytes.HasSuffix(data, markerEnd) { + return "", nil + } + return strings.TrimSpace(string(data[len(marker) : len(data)-len(markerEnd)])), after +} + +// If data is empty or ends in \n, fixNL returns data. +// Otherwise fixNL returns a new slice consisting of data with a final \n added. +func fixNL(data []byte) []byte { + if len(data) == 0 || data[len(data)-1] == '\n' { + return data + } + d := make([]byte, len(data)+1) + copy(d, data) + d[len(data)] = '\n' + return d +} diff --git a/internal/txtar/archive_test.go b/internal/txtar/archive_test.go new file mode 100644 index 0000000000..3f734f6762 --- /dev/null +++ b/internal/txtar/archive_test.go @@ -0,0 +1,67 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package txtar + +import ( + "bytes" + "fmt" + "reflect" + "testing" +) + +var tests = []struct { + name string + text string + parsed *Archive +}{ + { + name: "basic", + text: `comment1 +comment2 +-- file1 -- +File 1 text. +-- foo --- +More file 1 text. +-- file 2 -- +File 2 text. +-- empty -- +-- noNL -- +hello world`, + parsed: &Archive{ + Comment: []byte("comment1\ncomment2\n"), + Files: []File{ + {"file1", []byte("File 1 text.\n-- foo ---\nMore file 1 text.\n")}, + {"file 2", []byte("File 2 text.\n")}, + {"empty", []byte{}}, + {"noNL", []byte("hello world\n")}, + }, + }, + }, +} + +func Test(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := Parse([]byte(tt.text)) + if !reflect.DeepEqual(a, tt.parsed) { + t.Fatalf("Parse: wrong output:\nhave:\n%s\nwant:\n%s", shortArchive(a), shortArchive(tt.parsed)) + } + text := Format(a) + a = Parse(text) + if !reflect.DeepEqual(a, tt.parsed) { + t.Fatalf("Parse after Format: wrong output:\nhave:\n%s\nwant:\n%s", shortArchive(a), shortArchive(tt.parsed)) + } + }) + } +} + +func shortArchive(a *Archive) string { + var buf bytes.Buffer + fmt.Fprintf(&buf, "comment: %q\n", a.Comment) + for _, f := range a.Files { + fmt.Fprintf(&buf, "file %q: %q\n", f.Name, f.Data) + } + return buf.String() +}