[dev.fuzz] testing: add basic go command support for fuzzing

This change adds support for a -fuzz flag in the go command, and sets up
the groundwork for native fuzzing support. These functions are no-ops
for now, but will be built out and tested in future PRs.

Change-Id: I58e78eceada5799bcb73acc4ae5a20372badbf40
Reviewed-on: https://go-review.googlesource.com/c/go/+/251441
Run-TryBot: Katie Hockman <katie@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>

api/except.txt

@@ -350,6 +350,7 @@ pkg syscall (openbsd-amd64-cgo), type Timespec struct, Pad_cgo_0 [4]uint8
 pkg syscall (openbsd-amd64-cgo), type Timespec struct, Sec int32
 pkg testing, func RegisterCover(Cover)
 pkg testing, func MainStart(func(string, string) (bool, error), []InternalTest, []InternalBenchmark, []InternalExample) *M
+pkg testing, func MainStart(testDeps, []InternalTest, []InternalBenchmark, []InternalExample) *M
 pkg text/template/parse, type DotNode bool
 pkg text/template/parse, type Node interface { Copy, String, Type }
 pkg unicode, const Version = "6.2.0"

api/next.txt

@@ -17,3 +17,36 @@ pkg text/template/parse, type CommentNode struct, embedded NodeType
 pkg text/template/parse, type CommentNode struct, embedded Pos
 pkg text/template/parse, type Mode uint
 pkg text/template/parse, type Tree struct, Mode Mode
+pkg testing, func Fuzz(func(*F)) FuzzResult
+pkg testing, func MainStart(testDeps, []InternalTest, []InternalBenchmark, []InternalFuzzTarget, []InternalExample) *M
+pkg testing, func RunFuzzTargets(func(string, string) (bool, error), []InternalFuzzTarget) bool
+pkg testing, func RunFuzzing(func(string, string) (bool, error), []InternalFuzzTarget) bool
+pkg testing, method (*F) Add(...interface{})
+pkg testing, method (*F) Cleanup(func())
+pkg testing, method (*F) Error(...interface{})
+pkg testing, method (*F) Errorf(string, ...interface{})
+pkg testing, method (*F) Fail()
+pkg testing, method (*F) FailNow()
+pkg testing, method (*F) Failed() bool
+pkg testing, method (*F) Fatal(...interface{})
+pkg testing, method (*F) Fatalf(string, ...interface{})
+pkg testing, method (*F) Fuzz(interface{})
+pkg testing, method (*F) Helper()
+pkg testing, method (*F) Log(...interface{})
+pkg testing, method (*F) Logf(string, ...interface{})
+pkg testing, method (*F) Name() string
+pkg testing, method (*F) Skip(...interface{})
+pkg testing, method (*F) SkipNow()
+pkg testing, method (*F) Skipf(string, ...interface{})
+pkg testing, method (*F) Skipped() bool
+pkg testing, method (*F) TempDir() string
+pkg testing, method (FuzzResult) String() string
+pkg testing, type F struct
+pkg testing, type FuzzResult struct
+pkg testing, type FuzzResult struct, Crasher entry
+pkg testing, type FuzzResult struct, Error error
+pkg testing, type FuzzResult struct, N int
+pkg testing, type FuzzResult struct, T time.Duration
+pkg testing, type InternalFuzzTarget struct
+pkg testing, type InternalFuzzTarget struct, Fn func(*F)
+pkg testing, type InternalFuzzTarget struct, Name string

src/cmd/go/internal/load/test.go

@@ -496,6 +496,7 @@ func formatTestmain(t *testFuncs) ([]byte, error) {
 type testFuncs struct {
 	Tests       []testFunc
 	Benchmarks  []testFunc
+	FuzzTargets []testFunc
 	Examples    []testFunc
 	TestMain    *testFunc
 	Package     *Package
@@ -588,6 +589,12 @@ func (t *testFuncs) load(filename, pkg string, doImport, seen *bool) error {
 			}
 			t.Benchmarks = append(t.Benchmarks, testFunc{pkg, name, "", false})
 			*doImport, *seen = true, true
+		case isTest(name, "Fuzz"):
+			err := checkTestFunc(n, "F")
+			if err != nil {
+				return err
+			}
+			t.FuzzTargets = append(t.FuzzTargets, testFunc{pkg, name, "", false})
 		}
 	}
 	ex := doc.Examples(f)
@@ -651,6 +658,12 @@ var benchmarks = []testing.InternalBenchmark{
 {{end}}
 }
 
+var fuzzTargets = []testing.InternalFuzzTarget{
+{{range .FuzzTargets}}
+	{"{{.Name}}", {{.Package}}.{{.Name}}},
+{{end}}
+}
+
 var examples = []testing.InternalExample{
 {{range .Examples}}
 	{"{{.Name}}", {{.Package}}.{{.Name}}, {{.Output | printf "%q"}}, {{.Unordered}}},
@@ -709,7 +722,7 @@ func main() {
 		CoveredPackages: {{printf "%q" .Covered}},
 	})
 {{end}}
-	m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, examples)
+	m := testing.MainStart(testdeps.TestDeps{}, tests, benchmarks, fuzzTargets, examples)
 {{with .TestMain}}
 	{{.Package}}.{{.Name}}(m)
 	os.Exit(int(reflect.ValueOf(m).Elem().FieldByName("exitCode").Int()))

src/cmd/go/internal/test/flagdefs.go

@@ -19,6 +19,7 @@ var passFlagToTest = map[string]bool{
 	"cpu":                  true,
 	"cpuprofile":           true,
 	"failfast":             true,
+	"fuzz":                 true,
 	"list":                 true,
 	"memprofile":           true,
 	"memprofilerate":       true,

src/cmd/go/internal/test/test.go

@@ -467,6 +467,7 @@ See the documentation of the testing package for more information.
 `,
 }
 
+// TODO(katiehockman): complete the testing here
 var (
 	testBench        string                            // -bench flag
 	testC            bool                              // -c flag
@@ -475,6 +476,7 @@ var (
 	testCoverPaths   []string                          // -coverpkg flag
 	testCoverPkgs    []*load.Package                   // -coverpkg flag
 	testCoverProfile string                            // -coverprofile flag
+	testFuzz         string                            // -fuzz flag
 	testJSON         bool                              // -json flag
 	testList         string                            // -list flag
 	testO            string                            // -o flag

src/cmd/go/internal/test/testflag.go

@@ -56,6 +56,7 @@ func init() {
 	cf.String("cpu", "", "")
 	cf.StringVar(&testCPUProfile, "cpuprofile", "", "")
 	cf.Bool("failfast", false, "")
+	cf.String("fuzz", "", "")
 	cf.StringVar(&testList, "list", "", "")
 	cf.StringVar(&testMemProfile, "memprofile", "", "")
 	cf.String("memprofilerate", "", "")

src/go/doc/example.go

@@ -44,13 +44,13 @@ type Example struct {
 //     identifiers from other packages (or predeclared identifiers, such as
 //     "int") and the test file does not include a dot import.
 //   - The entire test file is the example: the file contains exactly one
-//     example function, zero test or benchmark functions, and at least one
+//     example function, zero test, fuzz target, or benchmark function, and at
-//     top-level function, type, variable, or constant declaration other
+//     least one top-level function, type, variable, or constant declaration
-//     than the example function.
+//     other than the example function.
 func Examples(testFiles ...*ast.File) []*Example {
 	var list []*Example
 	for _, file := range testFiles {
-		hasTests := false // file contains tests or benchmarks
+		hasTests := false // file contains tests, fuzz targets, or benchmarks
 		numDecl := 0      // number of non-import declarations in the file
 		var flist []*Example
 		for _, decl := range file.Decls {
@@ -64,7 +64,7 @@ func Examples(testFiles ...*ast.File) []*Example {
 			}
 			numDecl++
 			name := f.Name.Name
-			if isTest(name, "Test") || isTest(name, "Benchmark") {
+			if isTest(name, "Test") || isTest(name, "Benchmark") || isTest(name, "Fuzz") {
 				hasTests = true
 				continue
 			}
@@ -133,9 +133,9 @@ func exampleOutput(b *ast.BlockStmt, comments []*ast.CommentGroup) (output strin
 	return "", false, false // no suitable comment found
 }
 
-// isTest tells whether name looks like a test, example, or benchmark.
+// isTest tells whether name looks like a test, example, fuzz target, or
-// It is a Test (say) if there is a character after Test that is not a
+// benchmark. It is a Test (say) if there is a character after Test that is not
-// lower-case letter. (We don't want Testiness.)
+// a lower-case letter. (We don't want Testiness.)
 func isTest(name, prefix string) bool {
 	if !strings.HasPrefix(name, prefix) {
 		return false

src/go/doc/example_test.go

@@ -307,6 +307,9 @@ func (X) TestBlah() {
 func (X) BenchmarkFoo() {
 }
 
+func (X) FuzzFoo() {
+}
+
 func Example() {
 	fmt.Println("Hello, world!")
 	// Output: Hello, world!
@@ -326,6 +329,9 @@ func (X) TestBlah() {
 func (X) BenchmarkFoo() {
 }
 
+func (X) FuzzFoo() {
+}
+
 func main() {
 	fmt.Println("Hello, world!")
 }

src/testing/fuzz.go

@@ -0,0 +1,196 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package testing
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"time"
+)
+
+func initFuzzFlags() {
+	matchFuzz = flag.String("test.fuzz", "", "run the fuzz target matching `regexp`")
+}
+
+var matchFuzz *string
+
+// InternalFuzzTarget is an internal type but exported because it is cross-package;
+// it is part of the implementation of the "go test" command.
+type InternalFuzzTarget struct {
+	Name string
+	Fn   func(f *F)
+}
+
+// F is a type passed to fuzz targets for fuzz testing.
+type F struct {
+	common
+	context  *fuzzContext
+	corpus   []corpusEntry // corpus is the in-memory corpus
+	result   FuzzResult    // result is the result of running the fuzz target
+	fuzzFunc func(f *F)    // fuzzFunc is the function which makes up the fuzz target
+	fuzz     bool          // fuzz indicates whether or not the fuzzing engine should run
+}
+
+// corpus corpusEntry
+type corpusEntry struct {
+	b []byte
+}
+
+// Add will add the arguments to the seed corpus for the fuzz target. This
+// cannot be invoked after or within the Fuzz function. The args must match
+// those in the Fuzz function.
+func (f *F) Add(args ...interface{}) {
+	return
+}
+
+// Fuzz runs the fuzz function, ff, for fuzz testing. It runs ff in a separate
+// goroutine. Only one call to Fuzz is allowed per fuzz target, and any
+// subsequent calls will panic. If ff fails for a set of arguments, those
+// arguments will be added to the seed corpus.
+func (f *F) Fuzz(ff interface{}) {
+	return
+}
+
+// FuzzResult contains the results of a fuzz run.
+type FuzzResult struct {
+	N       int           // The number of iterations.
+	T       time.Duration // The total time taken.
+	Crasher corpusEntry   // Crasher is the corpus entry that caused the crash
+	Error   error         // Error is the error from the crash
+}
+
+func (r FuzzResult) String() string {
+	s := ""
+	if len(r.Error.Error()) != 0 {
+		s = fmt.Sprintf("error: %s\ncrasher: %b", r.Error.Error(), r.Crasher)
+	}
+	return s
+}
+
+// fuzzContext holds all fields that are common to all fuzz targets.
+type fuzzContext struct {
+	runMatch  *matcher
+	fuzzMatch *matcher
+}
+
+// RunFuzzTargets is an internal function but exported because it is cross-package;
+// it is part of the implementation of the "go test" command.
+func RunFuzzTargets(matchString func(pat, str string) (bool, error), fuzzTargets []InternalFuzzTarget) (ok bool) {
+	_, ok = runFuzzTargets(matchString, fuzzTargets)
+	return ok
+}
+
+// runFuzzTargets runs the fuzz targets matching the pattern for -run. This will
+// only run the f.Fuzz function for each seed corpus without using the fuzzing
+// engine to generate or mutate inputs. If -fuzz matches a given fuzz target,
+// then such test will be skipped and run later during fuzzing.
+func runFuzzTargets(matchString func(pat, str string) (bool, error), fuzzTargets []InternalFuzzTarget) (ran, ok bool) {
+	ran, ok = true, true
+	if len(fuzzTargets) == 0 {
+		return false, ok
+	}
+	for _, ft := range fuzzTargets {
+		ctx := &fuzzContext{runMatch: newMatcher(matchString, *match, "-test.run")}
+		f := &F{
+			common: common{
+				signal:  make(chan bool),
+				barrier: make(chan bool),
+				w:       os.Stdout,
+				name:    ft.Name,
+			},
+			context: ctx,
+		}
+		testName, matched, _ := ctx.runMatch.fullName(&f.common, f.name)
+		if !matched {
+			continue
+		}
+		if *matchFuzz != "" {
+			ctx.fuzzMatch = newMatcher(matchString, *matchFuzz, "-test.fuzz")
+			if _, doFuzz, partial := ctx.fuzzMatch.fullName(&f.common, f.name); doFuzz && !partial {
+				continue // this will be run later when fuzzed
+			}
+		}
+		if Verbose() {
+			f.chatty = newChattyPrinter(f.w)
+		}
+		if f.chatty != nil {
+			f.chatty.Updatef(f.name, "=== RUN  %s\n", testName)
+		}
+	}
+	return ran, ok
+}
+
+// RunFuzzing is an internal function but exported because it is cross-package;
+// it is part of the implementation of the "go test" command.
+func RunFuzzing(matchString func(pat, str string) (bool, error), fuzzTargets []InternalFuzzTarget) (ok bool) {
+	_, ok = runFuzzing(matchString, fuzzTargets)
+	return ok
+}
+
+// runFuzzing runs the fuzz target matching the pattern for -fuzz. Only one such
+// fuzz target must match. This will run the f.Fuzz function for each seed
+// corpus and will run the fuzzing engine to generate and mutate new inputs
+// against f.Fuzz.
+func runFuzzing(matchString func(pat, str string) (bool, error), fuzzTargets []InternalFuzzTarget) (ran, ok bool) {
+	ran, ok = true, true
+	if len(fuzzTargets) == 0 {
+		return false, ok
+	}
+	ctx := &fuzzContext{
+		fuzzMatch: newMatcher(matchString, *matchFuzz, "-test.fuzz"),
+	}
+	if *matchFuzz == "" {
+		return false, true
+	}
+	f := &F{
+		common: common{
+			signal:  make(chan bool),
+			barrier: make(chan bool),
+			w:       os.Stdout,
+		},
+		context: ctx,
+		fuzz:    true,
+	}
+	var (
+		ft    InternalFuzzTarget
+		found int
+	)
+	for _, ft = range fuzzTargets {
+		testName, matched, _ := ctx.fuzzMatch.fullName(&f.common, ft.Name)
+		if matched {
+			found++
+			if found > 1 {
+				fmt.Fprintf(f.w, "testing: warning: -fuzz matched more than one target, won't run\n")
+				return false, ok
+			}
+			f.name = testName
+		}
+	}
+	if Verbose() {
+		f.chatty = newChattyPrinter(f.w)
+	}
+	if f.chatty != nil {
+		f.chatty.Updatef(f.name, "--- FUZZ  %s\n", f.name)
+	}
+	return ran, ok
+}
+
+// Fuzz runs a single fuzz target. It is useful for creating
+// custom fuzz targets that do not use the "go test" command.
+//
+// If fn depends on testing flags, then Init must be used to register
+// those flags before calling Fuzz and before calling flag.Parse.
+func Fuzz(fn func(f *F)) FuzzResult {
+	f := &F{
+		common: common{
+			signal: make(chan bool),
+			w:      discard{},
+		},
+		fuzzFunc: fn,
+	}
+	// TODO(katiehockman): run the test
+	return f.result
+}

src/testing/testing.go

@@ -302,6 +302,7 @@ func Init() {
 	testlog = flag.String("test.testlogfile", "", "write test action log to `file` (for use only by cmd/go)")
 
 	initBenchmarkFlags()
+	initFuzzFlags()
 }
 
 var (
@@ -1294,7 +1295,7 @@ func (f matchStringOnly) SetPanicOnExit0(bool)                        {}
 // new functionality is added to the testing package.
 // Systems simulating "go test" should be updated to use MainStart.
 func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) {
-	os.Exit(MainStart(matchStringOnly(matchString), tests, benchmarks, examples).Run())
+	os.Exit(MainStart(matchStringOnly(matchString), tests, benchmarks, nil, examples).Run())
 }
 
 // M is a type passed to a TestMain function to run the actual tests.
@@ -1302,6 +1303,7 @@ type M struct {
 	deps        testDeps
 	tests       []InternalTest
 	benchmarks  []InternalBenchmark
+	fuzzTargets []InternalFuzzTarget
 	examples    []InternalExample
 
 	timer     *time.Timer
@@ -1332,12 +1334,13 @@ type testDeps interface {
 // MainStart is meant for use by tests generated by 'go test'.
 // It is not meant to be called directly and is not subject to the Go 1 compatibility document.
 // It may change signature from release to release.
-func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) *M {
+func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchmark, fuzzTargets []InternalFuzzTarget, examples []InternalExample) *M {
 	Init()
 	return &M{
 		deps:        deps,
 		tests:       tests,
 		benchmarks:  benchmarks,
+		fuzzTargets: fuzzTargets,
 		examples:    examples,
 	}
 }
@@ -1367,7 +1370,7 @@ func (m *M) Run() (code int) {
 	}
 
 	if len(*matchList) != 0 {
-		listTests(m.deps.MatchString, m.tests, m.benchmarks, m.examples)
+		listTests(m.deps.MatchString, m.tests, m.benchmarks, m.fuzzTargets, m.examples)
 		m.exitCode = 0
 		return
 	}
@@ -1379,12 +1382,23 @@ func (m *M) Run() (code int) {
 	deadline := m.startAlarm()
 	haveExamples = len(m.examples) > 0
 	testRan, testOk := runTests(m.deps.MatchString, m.tests, deadline)
+	fuzzTargetsRan, fuzzTargetsOk := runFuzzTargets(m.deps.MatchString, m.fuzzTargets)
 	exampleRan, exampleOk := runExamples(m.deps.MatchString, m.examples)
 	m.stopAlarm()
-	if !testRan && !exampleRan && *matchBenchmarks == "" {
+	if !testRan && !exampleRan && !fuzzTargetsRan && *matchBenchmarks == "" && *matchFuzz == "" {
 		fmt.Fprintln(os.Stderr, "testing: warning: no tests to run")
 	}
-	if !testOk || !exampleOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 {
+	if !testOk || !exampleOk || !fuzzTargetsOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 {
+		fmt.Println("FAIL")
+		m.exitCode = 1
+		return
+	}
+
+	fuzzingRan, fuzzingOk := runFuzzing(m.deps.MatchString, m.fuzzTargets)
+	if *matchFuzz != "" && !fuzzingRan {
+		fmt.Fprintln(os.Stderr, "testing: warning: no fuzz targets to run")
+	}
+	if !fuzzingOk {
 		fmt.Println("FAIL")
 		m.exitCode = 1
 		return
@@ -1412,7 +1426,7 @@ func (t *T) report() {
 	}
 }
 
-func listTests(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) {
+func listTests(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, fuzzTargets []InternalFuzzTarget, examples []InternalExample) {
 	if _, err := matchString(*matchList, "non-empty"); err != nil {
 		fmt.Fprintf(os.Stderr, "testing: invalid regexp in -test.list (%q): %s\n", *matchList, err)
 		os.Exit(1)
@@ -1428,6 +1442,11 @@ func listTests(matchString func(pat, str string) (bool, error), tests []Internal
 			fmt.Println(bench.Name)
 		}
 	}
+	for _, fuzzTarget := range fuzzTargets {
+		if ok, _ := matchString(*matchList, fuzzTarget.Name); ok {
+			fmt.Println(fuzzTarget.Name)
+		}
+	}
 	for _, example := range examples {
 		if ok, _ := matchString(*matchList, example.Name); ok {
 			fmt.Println(example.Name)