cmd/godoc: exclude /pkg from search indexing

On tip, search included redundant source results from /pkg/bootstrap
(with broken links as godoc doesn't support source files under /pkg).
This change excludes all directories under /pkg from indexing.

Fixes golang/go#10024.

Change-Id: I0c69d22ff08d131f9c37c91a7711db6a4ec53fd4
Reviewed-on: https://go-review.googlesource.com/7267
Reviewed-by: Andrew Gerrand <adg@golang.org>

cmd/godoc/appinit.go

@@ -50,6 +50,7 @@ func init() {
 	if err := corpus.Init(); err != nil {
 		log.Fatal(err)
 	}
+	corpus.IndexDirectory = indexDirectoryDefault
 	go corpus.RunIndexer()
 
 	pres = godoc.NewPresentation(corpus)

cmd/godoc/godoc_test.go

@@ -131,18 +131,30 @@ func serverAddress(t *testing.T) string {
 	return ln.Addr().String()
 }
 
+const (
+	startTimeout = 5 * time.Minute
+	pollInterval = 200 * time.Millisecond
+)
+
+var indexingMsg = []byte("Indexing in progress: result may be inaccurate")
+
 func waitForServer(t *testing.T, address string) {
-	// Poll every 50ms for a total of 5s.
-	for i := 0; i < 100; i++ {
-		time.Sleep(50 * time.Millisecond)
-		conn, err := net.Dial("tcp", address)
+	// "health check" duplicated from x/tools/cmd/tipgodoc/tip.go
+	deadline := time.Now().Add(startTimeout)
+	for time.Now().Before(deadline) {
+		time.Sleep(pollInterval)
+		res, err := http.Get(fmt.Sprintf("http://%v/search?q=FALLTHROUGH", address))
 		if err != nil {
 			continue
 		}
-		conn.Close()
-		return
+		rbody, err := ioutil.ReadAll(res.Body)
+		res.Body.Close()
+		if err == nil && res.StatusCode == http.StatusOK &&
+			!bytes.Contains(rbody, indexingMsg) {
+			return
+		}
 	}
-	t.Fatalf("Server %q failed to respond in 5 seconds", address)
+	t.Fatalf("Server %q failed to respond in %v", address, startTimeout)
 }
 
 func killAndWait(cmd *exec.Cmd) {
@@ -155,7 +167,7 @@ func TestWeb(t *testing.T) {
 	bin, cleanup := buildGodoc(t)
 	defer cleanup()
 	addr := serverAddress(t)
-	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr))
+	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr), "-index", "-index_interval=-1s")
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
 	cmd.Args[0] = "godoc"
@@ -207,6 +219,15 @@ func TestWeb(t *testing.T) {
 				"cmd/gc",
 			},
 		},
+		{
+			path: "/search?q=notwithstanding",
+			match: []string{
+				"/src",
+			},
+			dontmatch: []string{
+				"/pkg/bootstrap",
+			},
+		},
 	}
 	for _, test := range tests {
 		url := fmt.Sprintf("http://%s%s", addr, test.path)

cmd/godoc/index.go

@@ -0,0 +1,11 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "strings"
+
+func indexDirectoryDefault(dir string) bool {
+	return dir != "/pkg" && !strings.HasPrefix(dir, "/pkg/")
+}

cmd/godoc/main.go

@@ -219,6 +219,7 @@ func main() {
 		corpus.IndexFullText = false
 	}
 	corpus.IndexFiles = *indexFiles
+	corpus.IndexDirectory = indexDirectoryDefault
 	corpus.IndexThrottle = *indexThrottle
 	corpus.IndexInterval = *indexInterval
 	if *writeIndex {