mirror of https://github.com/golang/go.git
[dev.fuzz] cmd/go: split test_fuzz_minimize from test_fuzz_mutator
This CL moves test cases for minimization into a new script. The tests are changed to be more deterministic. Any non-zero byte now triggers a failure. Change-Id: Id45a721071dbfc3aec46be369966b0a67a745c55 Reviewed-on: https://go-review.googlesource.com/c/go/+/323851 Trust: Jay Conrod <jayconrod@google.com> Trust: Katie Hockman <katie@golang.org> Run-TryBot: Jay Conrod <jayconrod@google.com> Reviewed-by: Katie Hockman <katie@golang.org> TryBot-Result: Go Bot <gobot@golang.org>
This commit is contained in:
Jay Conrod
parent
2fed6926a1
commit
a3b5084204
|
|
@ -0,0 +1,159 @@
|
|||
# TODO(jayconrod): support shared memory on more platforms.
|
||||
[!darwin] [!linux] [!windows] skip
|
||||
|
||||
[short] skip
|
||||
|
||||
# We clean the fuzz cache during this test. Don't clean the user's cache.
|
||||
env GOCACHE=$WORK/gocache
|
||||
|
||||
# TODO(b/181800488): remove -parallel=1, here and below. For now, when a
|
||||
# crash is found, all workers keep running, wasting resources and reducing
|
||||
# the number of executions available to the minimizer, increasing flakiness.
|
||||
|
||||
# Test that minimization is working for recoverable errors.
|
||||
! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go
|
||||
! stdout '^ok'
|
||||
stdout 'got the minimum size!'
|
||||
stdout 'contains a non-zero byte'
|
||||
stdout FAIL
|
||||
|
||||
# Check that the bytes written to testdata are of length 50 (the minimum size)
|
||||
go run check_testdata.go FuzzMinimizerRecoverable 50
|
||||
|
||||
# Test that re-running the minimized value causes a crash.
|
||||
! go test -run=FuzzMinimizerRecoverable minimizer_test.go
|
||||
rm testdata
|
||||
|
||||
# Test that minimization is working for non-recoverable errors.
|
||||
! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go
|
||||
! stdout '^ok'
|
||||
# TODO(jayconrod): fix in CL 321835.
|
||||
stdout 'found a crash, minimizing'
|
||||
# stdout 'fuzzing process terminated unexpectedly while minimizing: exit status 99'
|
||||
stdout FAIL
|
||||
|
||||
# Check that re-running the value causes a crash.
|
||||
! go test -run=FuzzMinimizerNonrecoverable minimizer_test.go
|
||||
rm testdata
|
||||
|
||||
# Clear the fuzzing cache. There may already be minimized inputs that would
|
||||
# interfere with the next stage of the test.
|
||||
go clean -fuzzcache
|
||||
|
||||
# Test that minimization can be cancelled by fuzzminimizetime and the latest
|
||||
# crash will still be logged and written to testdata.
|
||||
! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -parallel=1 -fuzztime=100x -fuzzminimizetime=1x minimizer_test.go
|
||||
! stdout '^ok'
|
||||
stdout 'testdata[/\\]corpus[/\\]FuzzMinimizerRecoverable[/\\]'
|
||||
# TODO(jayconrod): implement -fuzzminimizetime in 321835.
|
||||
# ! stdout 'got the minimum size!' # it shouldn't have had enough time to minimize it
|
||||
stdout FAIL
|
||||
|
||||
# Test that re-running the unminimized value causes a crash.
|
||||
! go test -run=FuzzMinimizerRecoverable minimizer_test.go
|
||||
|
||||
# TODO(jayconrod,katiehockman): add a test which verifies that the right bytes
|
||||
# are written to testdata in the case of an interrupt during minimization.
|
||||
|
||||
-- go.mod --
|
||||
module m
|
||||
|
||||
go 1.16
|
||||
-- minimizer_test.go --
|
||||
package fuzz_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func FuzzMinimizerRecoverable(f *testing.F) {
|
||||
f.Add(make([]byte, 100))
|
||||
f.Fuzz(func(t *testing.T, b []byte) {
|
||||
if len(b) < 50 {
|
||||
// Make sure that b is large enough that it can be minimized
|
||||
return
|
||||
}
|
||||
// Given the randomness of the mutations, this should allow the
|
||||
// minimizer to trim down the value a bit.
|
||||
for _, n := range b {
|
||||
if n != 0 {
|
||||
if len(b) == 50 {
|
||||
t.Log("got the minimum size!")
|
||||
}
|
||||
t.Fatal("contains a non-zero byte")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzMinimizerNonrecoverable(f *testing.F) {
|
||||
f.Add(make([]byte, 100))
|
||||
f.Fuzz(func(t *testing.T, b []byte) {
|
||||
if len(b) < 50 {
|
||||
// Make sure that b is large enough that it can be minimized
|
||||
return
|
||||
}
|
||||
// Given the randomness of the mutations, this should allow the
|
||||
// minimizer to trim down the value a bit.
|
||||
for _, n := range b {
|
||||
if n != 0 {
|
||||
t.Log("contains a non-zero byte")
|
||||
os.Exit(99)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
-- check_testdata.go --
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func main() {
|
||||
target := os.Args[1]
|
||||
numBytes, err := strconv.Atoi(os.Args[2])
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Open the file in testdata (there should only be one)
|
||||
dir := fmt.Sprintf("testdata/corpus/%s", target)
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if len(files) != 1 {
|
||||
fmt.Fprintf(os.Stderr, "expected one file, got %d", len(files))
|
||||
os.Exit(1)
|
||||
}
|
||||
got, err := ioutil.ReadFile(filepath.Join(dir, files[0].Name()))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Make sure that there were exactly 100 bytes written to the corpus entry
|
||||
prefix := []byte("[]byte(")
|
||||
i := bytes.Index(got, prefix)
|
||||
gotBytes := got[i+len(prefix) : len(got)-1]
|
||||
s, err := strconv.Unquote(string(gotBytes))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if want, got := numBytes, len(s); want != got {
|
||||
fmt.Fprintf(os.Stderr, "want %d bytes, got %d\n", want, got)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
|
@ -10,11 +10,12 @@
|
|||
|
||||
[short] skip
|
||||
|
||||
go test -fuzz=FuzzA -fuzztime=100x -parallel=1 -log=fuzz
|
||||
go run check_logs.go fuzz fuzz.worker
|
||||
|
||||
# TODO(b/181800488): remove -parallel=1, here and below. For now, when a
|
||||
# crash is found, all workers keep running, wasting resources and reducing
|
||||
# the number of executions available to the minimizer, increasing flakiness.
|
||||
go test -fuzz=FuzzA -fuzztime=100x -parallel=1 -log=fuzz
|
||||
go run check_logs.go fuzz fuzz.worker
|
||||
|
||||
# Test that the mutator is good enough to find several unique mutations.
|
||||
! go test -fuzz=FuzzMutator -parallel=1 -fuzztime=100x mutator_test.go
|
||||
|
|
@ -22,41 +23,6 @@ go run check_logs.go fuzz fuzz.worker
|
|||
stdout FAIL
|
||||
stdout 'mutator found enough unique mutations'
|
||||
|
||||
# Test that minimization is working for recoverable errors.
|
||||
! go test -fuzz=FuzzMinimizerRecoverable -run=FuzzMinimizerRecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go
|
||||
! stdout '^ok'
|
||||
stdout 'got the minimum size!'
|
||||
stdout 'contains a letter'
|
||||
stdout FAIL
|
||||
|
||||
# Check that the bytes written to testdata are of length 50 (the minimum size)
|
||||
go run check_testdata.go FuzzMinimizerRecoverable 50
|
||||
|
||||
# Test that re-running the minimized value causes a crash.
|
||||
! go test -run=FuzzMinimizerRecoverable minimizer_test.go
|
||||
|
||||
# Test that minimization is working for non-recoverable errors.
|
||||
! go test -fuzz=FuzzMinimizerNonrecoverable -run=FuzzMinimizerNonrecoverable -fuzztime=100x -fuzzminimizetime=10000x -parallel=1 minimizer_test.go
|
||||
! stdout '^ok'
|
||||
stdout 'got the minimum size!'
|
||||
stdout 'contains a letter'
|
||||
stdout FAIL
|
||||
|
||||
# Check that the bytes written to testdata are of length 50 (the minimum size)
|
||||
go run check_testdata.go FuzzMinimizerNonrecoverable 50
|
||||
|
||||
# Test that minimization can be cancelled by fuzzminimizetime and the latest
|
||||
# crash will still be logged and written to testdata.
|
||||
! go test -fuzz=FuzzNonMinimizable -run=FuzzNonMinimizable -parallel=1 -fuzztime=100x -fuzzminimizetime=1x minimizer_test.go
|
||||
! stdout '^ok'
|
||||
stdout 'testdata[/\\]corpus[/\\]FuzzNonMinimizable[/\\]'
|
||||
! stdout 'got the minimum size!' # it shouldn't have had enough time to minimize it
|
||||
stdout 'at least 20 bytes'
|
||||
stdout FAIL
|
||||
|
||||
# TODO(jayconrod,katiehockman): add a test which verifies that the right bytes
|
||||
# are written to testdata in the case of an interrupt during minimization.
|
||||
|
||||
-- go.mod --
|
||||
module m
|
||||
|
||||
|
|
@ -105,63 +71,6 @@ func FuzzB(f *testing.F) {
|
|||
})
|
||||
}
|
||||
|
||||
-- minimizer_test.go --
|
||||
package fuzz_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func FuzzMinimizerRecoverable(f *testing.F) {
|
||||
f.Fuzz(func(t *testing.T, b []byte) {
|
||||
if len(b) < 50 {
|
||||
// Make sure that b is large enough that it can be minimized
|
||||
return
|
||||
}
|
||||
// Given the randomness of the mutations, this should allow the
|
||||
// minimizer to trim down the value a bit.
|
||||
if bytes.ContainsAny(b, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") {
|
||||
if len(b) == 50 {
|
||||
fmt.Fprint(os.Stderr, "got the minimum size!\n")
|
||||
}
|
||||
t.Error("contains a letter")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzMinimizerNonrecoverable(f *testing.F) {
|
||||
f.Fuzz(func(t *testing.T, b []byte) {
|
||||
if len(b) < 50 {
|
||||
// Make sure that b is large enough that it can be minimized
|
||||
return
|
||||
}
|
||||
// Given the randomness of the mutations, this should allow the
|
||||
// minimizer to trim down the value quite a bit.
|
||||
if bytes.ContainsAny(b, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") {
|
||||
if len(b) == 50 {
|
||||
fmt.Fprint(os.Stderr, "got the minimum size!\n")
|
||||
}
|
||||
panic("contains a letter")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func FuzzNonMinimizable(f *testing.F) {
|
||||
f.Fuzz(func(t *testing.T, b []byte) {
|
||||
if len(b) < 20 {
|
||||
// Make sure that b is large enough that minimization will try to run.
|
||||
return
|
||||
}
|
||||
panic("at least 20 bytes")
|
||||
if len(b) == 20 {
|
||||
fmt.Fprint(os.Stderr, "got the minimum size!\n")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
-- check_logs.go --
|
||||
// +build ignore
|
||||
|
||||
|
|
@ -255,59 +164,6 @@ func checkWorkerLog(r io.Reader) error {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
-- check_testdata.go --
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func main() {
|
||||
target := os.Args[1]
|
||||
numBytes, err := strconv.Atoi(os.Args[2])
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Open the file in testdata (there should only be one)
|
||||
dir := fmt.Sprintf("testdata/corpus/%s", target)
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if len(files) != 1 {
|
||||
fmt.Fprintf(os.Stderr, "expected one file, got %d", len(files))
|
||||
os.Exit(1)
|
||||
}
|
||||
got, err := ioutil.ReadFile(filepath.Join(dir, files[0].Name()))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Make sure that there were exactly 100 bytes written to the corpus entry
|
||||
prefix := []byte("[]byte(")
|
||||
i := bytes.Index(got, prefix)
|
||||
gotBytes := got[i+len(prefix) : len(got)-1]
|
||||
s, err := strconv.Unquote(string(gotBytes))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if want, got := numBytes, len(s); want != got {
|
||||
fmt.Fprintf(os.Stderr, "want %d bytes, got %d\n", want, got)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
-- mutator_test.go --
|
||||
package fuzz_test
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue