[dev.fuzz] cmd/go/internal/test,testing: add documentation about fuzzing

Change-Id: Id43f7f75d6033a2c35bacd1cc0b8e3fbcaf69316
Reviewed-on: https://go-review.googlesource.com/c/go/+/317973
Trust: Katie Hockman <katie@golang.org>
Run-TryBot: Katie Hockman <katie@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>

src/cmd/go/alldocs.go

@@ -53,6 +53,7 @@
 // 	private         configuration for downloading non-public code
 // 	testflag        testing flags
 // 	testfunc        testing functions
+// 	fuzz            fuzzing
 // 	vcs             controlling version control with GOVCS
 //
 // Use "go help <topic>" for more information about that topic.
@@ -1377,8 +1378,8 @@
 //
 // 'Go test' recompiles each package along with any files with names matching
 // the file pattern "*_test.go".
-// These additional files can contain test functions, benchmark functions, and
+// These additional files can contain test functions, benchmark functions, fuzz
-// example functions. See 'go help testfunc' for more.
+// targets and example functions. See 'go help testfunc' for more.
 // Each listed package causes the execution of a separate test binary.
 // Files whose names begin with "_" (including "_test.go") or "." are ignored.
 //
@@ -1447,6 +1448,8 @@
 // so a successful package test result will be cached and reused
 // regardless of -timeout setting.
 //
+// Run 'go help fuzz' for details around how the go command handles fuzz targets.
+//
 // In addition to the build flags, the flags handled by 'go test' itself are:
 //
 // 	-args
@@ -2609,7 +2612,8 @@
 // 	    (for example, -benchtime 100x).
 //
 // 	-count n
-// 	    Run each test and benchmark n times (default 1).
+// 	    Run each test, benchmark, and fuzz targets' seed corpora n times
+// 	    (default 1).
 // 	    If -cpu is set, run n times for each GOMAXPROCS value.
 // 	    Examples are always run once.
 //
@@ -2638,32 +2642,51 @@
 // 	    Sets -cover.
 //
 // 	-cpu 1,2,4
-// 	    Specify a list of GOMAXPROCS values for which the tests or
+// 	    Specify a list of GOMAXPROCS values for which the tests, benchmarks or
-// 	    benchmarks should be executed. The default is the current value
+// 	    fuzz targets should be executed. The default is the current value
 // 	    of GOMAXPROCS.
 //
 // 	-failfast
 // 	    Do not start new tests after the first test failure.
 //
+// 	-fuzz name
+// 	    Run the fuzz target with the given regexp. Must match exactly one fuzz
+// 	    target. This is an experimental feature.
+//
+// 	-fuzztime t
+// 	    Run enough iterations of the fuzz test to take t, specified as a
+// 	    time.Duration (for example, -fuzztime 1h30s). The default is to run
+// 	    forever.
+// 	    The special syntax Nx means to run the fuzz test N times
+// 	    (for example, -fuzztime 100x).
+//
+// 	-keepfuzzing
+// 	    Keep running the fuzz target if a crasher is found.
+//
 // 	-list regexp
-// 	    List tests, benchmarks, or examples matching the regular expression.
+// 	    List tests, benchmarks, fuzz targets, or examples matching the regular
-// 	    No tests, benchmarks or examples will be run. This will only
+// 	    expression. No tests, benchmarks, fuzz targets, or examples will be run.
-// 	    list top-level tests. No subtest or subbenchmarks will be shown.
+// 	    This will only list top-level tests. No subtest or subbenchmarks will be
+// 	    shown.
 //
 // 	-parallel n
-// 	    Allow parallel execution of test functions that call t.Parallel.
+// 	    Allow parallel execution of test functions that call t.Parallel, and
+// 	    f.Fuzz functions that call t.Parallel when running the seed corpus.
 // 	    The value of this flag is the maximum number of tests to run
-// 	    simultaneously; by default, it is set to the value of GOMAXPROCS.
+// 	    simultaneously. While fuzzing, the value of this flag is the
+// 	    maximum number of workers to run the fuzz function simultaneously,
+// 	    regardless of whether t.Parallel has been called; by default, it is set
+// 	    to the value of GOMAXPROCS.
 // 	    Note that -parallel only applies within a single test binary.
 // 	    The 'go test' command may run tests for different packages
 // 	    in parallel as well, according to the setting of the -p flag
 // 	    (see 'go help build').
 //
 // 	-run regexp
-// 	    Run only those tests and examples matching the regular expression.
+// 	    Run only those tests, examples, and fuzz targets matching the regular
-// 	    For tests, the regular expression is split by unbracketed slash (/)
+// 	    expression. For tests, the regular expression is split by unbracketed
-// 	    characters into a sequence of regular expressions, and each part
+// 	    slash (/) characters into a sequence of regular expressions, and each
-// 	    of a test's identifier must match the corresponding element in
+// 	    part of a test's identifier must match the corresponding element in
 // 	    the sequence, if any. Note that possible parents of matches are
 // 	    run too, so that -run=X/Y matches and runs and reports the result
 // 	    of all tests matching X, even those without sub-tests matching Y,
@@ -2830,6 +2853,10 @@
 //
 // 	func BenchmarkXxx(b *testing.B) { ... }
 //
+// A fuzz target is one named FuzzXxx and should have the signature,
+//
+// 	func FuzzXxx(f *testing.F) { ... }
+//
 // An example function is similar to a test function but, instead of using
 // *testing.T to report success or failure, prints output to os.Stdout.
 // If the last comment in the function starts with "Output:" then the output
@@ -2869,11 +2896,30 @@
 //
 // The entire test file is presented as the example when it contains a single
 // example function, at least one other function, type, variable, or constant
-// declaration, and no test or benchmark functions.
+// declaration, and no fuzz targets or test or benchmark functions.
 //
 // See the documentation of the testing package for more information.
 //
 //
+// Fuzzing
+//
+// By default, go test will build and run the fuzz targets using the target's seed
+// corpus only. Any generated corpora in $GOCACHE that were previously written by
+// the fuzzing engine will not be run by default.
+//
+// When -fuzz is set, the binary will be instrumented for coverage. After all
+// tests, examples, benchmark functions, and the seed corpora for all fuzz targets
+// have been run, go test will begin to fuzz the specified fuzz target.
+// Note that this feature is experimental.
+//
+// -run can be used for testing a single seed corpus entry for a fuzz target. The
+// regular expression value of -run can be in the form $target/$name, where $target
+// is the name of the fuzz target, and $name is the name of the file (ignoring file
+// extensions) to run. For example, -run=FuzzFoo/497b6f87.
+//
+// See https://golang.org/s/draft-fuzzing-design for more details.
+//
+//
 // Controlling version control with GOVCS
 //
 // The 'go get' command can run version control commands like git

src/cmd/go/internal/test/test.go

@@ -60,8 +60,8 @@ followed by detailed output for each failed package.
 
 'Go test' recompiles each package along with any files with names matching
 the file pattern "*_test.go".
-These additional files can contain test functions, benchmark functions, and
+These additional files can contain test functions, benchmark functions, fuzz
-example functions. See 'go help testfunc' for more.
+targets and example functions. See 'go help testfunc' for more.
 Each listed package causes the execution of a separate test binary.
 Files whose names begin with "_" (including "_test.go") or "." are ignored.
 
@@ -130,6 +130,8 @@ A cached test result is treated as executing in no time at all,
 so a successful package test result will be cached and reused
 regardless of -timeout setting.
 
+Run 'go help fuzz' for details around how the go command handles fuzz targets.
+
 In addition to the build flags, the flags handled by 'go test' itself are:
 
 	-args
@@ -206,7 +208,8 @@ control the execution of any test:
 	    (for example, -benchtime 100x).
 
 	-count n
-	    Run each test and benchmark n times (default 1).
+	    Run each test, benchmark, and fuzz targets' seed corpora n times
+	    (default 1).
 	    If -cpu is set, run n times for each GOMAXPROCS value.
 	    Examples are always run once.
 
@@ -235,32 +238,51 @@ control the execution of any test:
 	    Sets -cover.
 
 	-cpu 1,2,4
-	    Specify a list of GOMAXPROCS values for which the tests or
+	    Specify a list of GOMAXPROCS values for which the tests, benchmarks or
-	    benchmarks should be executed. The default is the current value
+	    fuzz targets should be executed. The default is the current value
 	    of GOMAXPROCS.
 
 	-failfast
 	    Do not start new tests after the first test failure.
 
+	-fuzz name
+	    Run the fuzz target with the given regexp. Must match exactly one fuzz
+	    target. This is an experimental feature.
+
+	-fuzztime t
+	    Run enough iterations of the fuzz test to take t, specified as a
+	    time.Duration (for example, -fuzztime 1h30s). The default is to run
+	    forever.
+	    The special syntax Nx means to run the fuzz test N times
+	    (for example, -fuzztime 100x).
+
+	-keepfuzzing
+	    Keep running the fuzz target if a crasher is found.
+
 	-list regexp
-	    List tests, benchmarks, or examples matching the regular expression.
+	    List tests, benchmarks, fuzz targets, or examples matching the regular
-	    No tests, benchmarks or examples will be run. This will only
+	    expression. No tests, benchmarks, fuzz targets, or examples will be run.
-	    list top-level tests. No subtest or subbenchmarks will be shown.
+	    This will only list top-level tests. No subtest or subbenchmarks will be
+	    shown.
 
 	-parallel n
-	    Allow parallel execution of test functions that call t.Parallel.
+	    Allow parallel execution of test functions that call t.Parallel, and
+	    f.Fuzz functions that call t.Parallel when running the seed corpus.
 	    The value of this flag is the maximum number of tests to run
-	    simultaneously; by default, it is set to the value of GOMAXPROCS.
+	    simultaneously. While fuzzing, the value of this flag is the
+	    maximum number of workers to run the fuzz function simultaneously,
+	    regardless of whether t.Parallel has been called; by default, it is set
+	    to the value of GOMAXPROCS.
 	    Note that -parallel only applies within a single test binary.
 	    The 'go test' command may run tests for different packages
 	    in parallel as well, according to the setting of the -p flag
 	    (see 'go help build').
 
 	-run regexp
-	    Run only those tests and examples matching the regular expression.
+	    Run only those tests, examples, and fuzz targets matching the regular
-	    For tests, the regular expression is split by unbracketed slash (/)
+	    expression. For tests, the regular expression is split by unbracketed
-	    characters into a sequence of regular expressions, and each part
+	    slash (/) characters into a sequence of regular expressions, and each
-	    of a test's identifier must match the corresponding element in
+	    part of a test's identifier must match the corresponding element in
 	    the sequence, if any. Note that possible parents of matches are
 	    run too, so that -run=X/Y matches and runs and reports the result
 	    of all tests matching X, even those without sub-tests matching Y,
@@ -430,6 +452,10 @@ A benchmark function is one named BenchmarkXxx and should have the signature,
 
 	func BenchmarkXxx(b *testing.B) { ... }
 
+A fuzz target is one named FuzzXxx and should have the signature,
+
+	func FuzzXxx(f *testing.F) { ... }
+
 An example function is similar to a test function but, instead of using
 *testing.T to report success or failure, prints output to os.Stdout.
 If the last comment in the function starts with "Output:" then the output
@@ -469,12 +495,34 @@ Here is another example where the ordering of the output is ignored:
 
 The entire test file is presented as the example when it contains a single
 example function, at least one other function, type, variable, or constant
-declaration, and no test or benchmark functions.
+declaration, and no fuzz targets or test or benchmark functions.
 
 See the documentation of the testing package for more information.
 `,
 }
 
+var HelpFuzz = &base.Command{
+	UsageLine: "fuzz",
+	Short:     "fuzzing",
+	Long: `
+By default, go test will build and run the fuzz targets using the target's seed
+corpus only. Any generated corpora in $GOCACHE that were previously written by
+the fuzzing engine will not be run by default.
+
+When -fuzz is set, the binary will be instrumented for coverage. After all
+tests, examples, benchmark functions, and the seed corpora for all fuzz targets
+have been run, go test will begin to fuzz the specified fuzz target.
+Note that this feature is experimental.
+
+-run can be used for testing a single seed corpus entry for a fuzz target. The
+regular expression value of -run can be in the form $target/$name, where $target
+is the name of the fuzz target, and $name is the name of the file (ignoring file
+extensions) to run. For example, -run=FuzzFoo/497b6f87.
+
+See https://golang.org/s/draft-fuzzing-design for more details.
+`,
+}
+
 var (
 	testBench        string                            // -bench flag
 	testC            bool                              // -c flag

src/cmd/go/main.go

@@ -80,6 +80,7 @@ func init() {
 		modfetch.HelpPrivate,
 		test.HelpTestflag,
 		test.HelpTestfunc,
+		test.HelpFuzz,
 		modget.HelpVCS,
 	}
 }

src/testing/testing.go

@@ -34,7 +34,7 @@
 // its -bench flag is provided. Benchmarks are run sequentially.
 //
 // For a description of the testing flags, see
-// https://golang.org/cmd/go/#hdr-Testing_flags
+// https://golang.org/cmd/go/#hdr-Testing_flags.
 //
 // A sample benchmark function looks like this:
 //     func BenchmarkRandInt(b *testing.B) {
@@ -132,6 +132,27 @@
 // example function, at least one other function, type, variable, or constant
 // declaration, and no test or benchmark functions.
 //
+// Fuzzing
+//
+// Functions of the form
+//     func FuzzXxx(*testing.F)
+// are considered fuzz targets, and are executed by the "go test" command. When
+// the -fuzz flag is provided, the functions will be fuzzed.
+//
+// For a description of the testing flags, see
+// https://golang.org/cmd/go/#hdr-Testing_flags.
+//
+// For a description of fuzzing, see golang.org/s/draft-fuzzing-design.
+//
+// A sample fuzz target looks like this:
+//     func FuzzBytesCmp(f *testing.F) {
+//         f.Fuzz(func(t *testing.T, a, b []byte) {
+//             if bytes.HasPrefix(a, b) && !bytes.Contains(a, b) {
+//                 t.Error("HasPrefix is true, but Contains is false")
+//             }
+//         })
+//     }
+//
 // Skipping
 //
 // Tests or benchmarks may be skipped at run time with a call to
@@ -144,6 +165,21 @@
 //         ...
 //     }
 //
+// The Skip method of *T can be used in a fuzz target if the input is invalid,
+// but should not be considered a crash. For example:
+//
+//     func FuzzJSONMarshalling(f *testing.F) {
+//         f.Fuzz(func(t *testing.T, b []byte) {
+//             var v interface{}
+//             if err := json.Unmarshal(b, &v); err != nil {
+//                 t.Skip()
+//             }
+//             if _, err := json.Marshal(v); err != nil {
+//                 t.Error("Marshal: %v", err)
+//             }
+//         })
+//     }
+//
 // Subtests and Sub-benchmarks
 //
 // The Run methods of T and B allow defining subtests and sub-benchmarks,
@@ -163,17 +199,25 @@
 // of the top-level test and the sequence of names passed to Run, separated by
 // slashes, with an optional trailing sequence number for disambiguation.
 //
-// The argument to the -run and -bench command-line flags is an unanchored regular
+// The argument to the -run, -bench, and -fuzz command-line flags is an unanchored regular
 // expression that matches the test's name. For tests with multiple slash-separated
 // elements, such as subtests, the argument is itself slash-separated, with
 // expressions matching each name element in turn. Because it is unanchored, an
 // empty expression matches any string.
 // For example, using "matching" to mean "whose name contains":
 //
-//     go test -run ''      # Run all tests.
+//     go test -run ''        # Run all tests.
-//     go test -run Foo     # Run top-level tests matching "Foo", such as "TestFooBar".
+//     go test -run Foo       # Run top-level tests matching "Foo", such as "TestFooBar".
-//     go test -run Foo/A=  # For top-level tests matching "Foo", run subtests matching "A=".
+//     go test -run Foo/A=    # For top-level tests matching "Foo", run subtests matching "A=".
-//     go test -run /A=1    # For all top-level tests, run subtests matching "A=1".
+//     go test -run /A=1      # For all top-level tests, run subtests matching "A=1".
+//     go test -fuzz FuzzFoo  # Fuzz the target matching "FuzzFoo"
+//
+// The -run argument can also be used to run a specific value in the seed
+// corpus, for debugging. For example:
+//     go test -run=FuzzFoo/9ddb952d9814
+//
+// The -fuzz and -run flags can both be set, in order to fuzz a target but
+// skip the execution of all other tests.
 //
 // Subtests can also be used to control parallelism. A parent test will only
 // complete once all of its subtests complete. In this example, all tests are