net: deflake lookup tests

The build dashboard is dotted with net test failures.
We cannot declare all builders to have flaky networks,
although all fundamentally do.

Instead, add a simple retry/backoff loop to the ones that
show up most commonly on the dashboard at this moment.

If this approach works well in practice, we can
incrementally apply it to other flaky net tests.

Change-Id: I69c1ca6ce5b347ad549c7eb18d0438373f6e2489
Reviewed-on: https://go-review.googlesource.com/102397
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
This commit is contained in:
Josh Bleecher Snyder 2018-03-24 22:09:51 -07:00
parent 6f08b9faf2
commit 5ce92d0384

View File

@ -60,6 +60,8 @@ var lookupGoogleSRVTests = []struct {
},
}
var backoffDuration = [...]time.Duration{time.Second, 5 * time.Second, 30 * time.Second}
func TestLookupGoogleSRV(t *testing.T) {
if testenv.Builder() == "" {
testenv.MustHaveExternalNetwork(t)
@ -69,10 +71,20 @@ func TestLookupGoogleSRV(t *testing.T) {
t.Skip("IPv4 is required")
}
for _, tt := range lookupGoogleSRVTests {
attempts := 0
for i := 0; i < len(lookupGoogleSRVTests); i++ {
tt := lookupGoogleSRVTests[i]
cname, srvs, err := LookupSRV(tt.service, tt.proto, tt.name)
if err != nil {
testenv.SkipFlakyNet(t)
if attempts < len(backoffDuration) {
dur := backoffDuration[attempts]
t.Logf("backoff %v after failure %v\n", dur, err)
time.Sleep(dur)
attempts++
i--
continue
}
t.Fatal(err)
}
if len(srvs) == 0 {
@ -107,9 +119,20 @@ func TestLookupGmailMX(t *testing.T) {
defer dnsWaitGroup.Wait()
for _, tt := range lookupGmailMXTests {
attempts := 0
for i := 0; i < len(lookupGmailMXTests); i++ {
tt := lookupGmailMXTests[i]
mxs, err := LookupMX(tt.name)
if err != nil {
testenv.SkipFlakyNet(t)
if attempts < len(backoffDuration) {
dur := backoffDuration[attempts]
t.Logf("backoff %v after failure %v\n", dur, err)
time.Sleep(dur)
attempts++
i--
continue
}
t.Fatal(err)
}
if len(mxs) == 0 {
@ -176,9 +199,20 @@ func TestLookupGmailTXT(t *testing.T) {
defer dnsWaitGroup.Wait()
for _, tt := range lookupGmailTXTTests {
attempts := 0
for i := 0; i < len(lookupGmailTXTTests); i++ {
tt := lookupGmailTXTTests[i]
txts, err := LookupTXT(tt.name)
if err != nil {
testenv.SkipFlakyNet(t)
if attempts < len(backoffDuration) {
dur := backoffDuration[attempts]
t.Logf("backoff %v after failure %v\n", dur, err)
time.Sleep(dur)
attempts++
i--
continue
}
t.Fatal(err)
}
if len(txts) == 0 {