Merge pull request #4263 from carl-mastrangelo/percent

Make http2 interop tests always pass, and instead give a report

tools/http2_interop/http2interop_test.go

@@ -3,6 +3,7 @@ package http2interop
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -67,7 +68,8 @@ func (ctx *HTTP2InteropCtx) Close() error {
 	return nil
 }
 
-func TestClientShortSettings(t *testing.T) {
+func TestSoonClientShortSettings(t *testing.T) {
+	defer Report(t)
 	if *testCase != "framing" {
 		t.SkipNow()
 	}
@@ -78,7 +80,8 @@ func TestClientShortSettings(t *testing.T) {
 	}
 }
 
-func TestShortPreface(t *testing.T) {
+func TestSoonShortPreface(t *testing.T) {
+	defer Report(t)
 	if *testCase != "framing" {
 		t.SkipNow()
 	}
@@ -89,7 +92,8 @@ func TestShortPreface(t *testing.T) {
 	}
 }
 
-func TestUnknownFrameType(t *testing.T) {
+func TestSoonUnknownFrameType(t *testing.T) {
+	defer Report(t)
 	if *testCase != "framing" {
 		t.SkipNow()
 	}
@@ -99,7 +103,8 @@ func TestUnknownFrameType(t *testing.T) {
 	}
 }
 
-func TestClientPrefaceWithStreamId(t *testing.T) {
+func TestSoonClientPrefaceWithStreamId(t *testing.T) {
+	defer Report(t)
 	if *testCase != "framing" {
 		t.SkipNow()
 	}
@@ -108,7 +113,8 @@ func TestClientPrefaceWithStreamId(t *testing.T) {
 	matchError(t, err, "EOF")
 }
 
-func TestTLSApplicationProtocol(t *testing.T) {
+func TestSoonTLSApplicationProtocol(t *testing.T) {
+	defer Report(t)
 	if *testCase != "tls" {
 		t.SkipNow()
 	}
@@ -117,7 +123,8 @@ func TestTLSApplicationProtocol(t *testing.T) {
 	matchError(t, err, "EOF", "broken pipe")
 }
 
-func TestTLSMaxVersion(t *testing.T) {
+func TestSoonTLSMaxVersion(t *testing.T) {
+	defer Report(t)
 	if *testCase != "tls" {
 		t.SkipNow()
 	}
@@ -128,7 +135,8 @@ func TestTLSMaxVersion(t *testing.T) {
 	matchError(t, err, "EOF", "server selected unsupported protocol")
 }
 
-func TestTLSBadCipherSuites(t *testing.T) {
+func TestSoonTLSBadCipherSuites(t *testing.T) {
+	defer Report(t)
 	if *testCase != "tls" {
 		t.SkipNow()
 	}
@@ -151,5 +159,25 @@ func matchError(t *testing.T, err error, matches ...string) {
 
 func TestMain(m *testing.M) {
 	flag.Parse()
-	os.Exit(m.Run())
+	m.Run()
+	var fatal bool
+	var any bool
+	for _, ci := range allCaseInfos.Cases {
+		if ci.Skipped {
+			continue
+		}
+		any = true
+		if !ci.Passed && ci.Fatal {
+			fatal = true
+		}
+	}
+
+	if err := json.NewEncoder(os.Stderr).Encode(&allCaseInfos); err != nil {
+		fmt.Println("Failed to encode", err)
+	}
+	var code int
+	if !any || fatal {
+		code = 1
+	}
+	os.Exit(code)
 }

tools/http2_interop/s6.5.go

@@ -11,7 +11,6 @@ func testSmallMaxFrameSize(ctx *HTTP2InteropCtx) error {
 		return err
 	}
 	defer conn.Close()
-	conn.Log = ctx.T.Log
 	conn.SetDeadline(time.Now().Add(defaultTimeout))
 
 	sf := &SettingsFrame{

tools/http2_interop/s6.5_test.go

@@ -4,8 +4,9 @@ import (
 	"testing"
 )
 
-func TestSmallMaxFrameSize(t *testing.T) {
-	if *testCase != "experimental" {
+func TestSoonSmallMaxFrameSize(t *testing.T) {
+	defer Report(t)
+	if *testCase != "framing" {
 		t.SkipNow()
 	}
 	ctx := InteropCtx(t)

tools/http2_interop/testsuite.go

@@ -0,0 +1,56 @@
+package http2interop
+
+import (
+	"path"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+)
+
+// When a test is skipped or fails, runtime.Goexit() is called which destroys the callstack.
+// This means the name of the test case is lost, so we need to grab a copy of pc before.
+func Report(t testing.TB) {
+	// If the goroutine panics, Fatal()s, or Skip()s, the function name is at the 3rd callstack
+	// layer.  On success, its at 1st.  Since it's hard to check which happened, just try both.
+	pcs := make([]uintptr, 10)
+	total := runtime.Callers(1, pcs)
+	var name string
+	for _, pc := range pcs[:total] {
+		fn := runtime.FuncForPC(pc)
+		fullName := fn.Name()
+		if strings.HasPrefix(path.Ext(fullName), ".Test") {
+			// Skip the leading .
+			name = string([]byte(path.Ext(fullName))[1:])
+			break
+		}
+	}
+	if name == "" {
+		return
+	}
+
+	allCaseInfos.lock.Lock()
+	defer allCaseInfos.lock.Unlock()
+	allCaseInfos.Cases = append(allCaseInfos.Cases, &caseInfo{
+		Name:    name,
+		Passed:  !t.Failed() && !t.Skipped(),
+		Skipped: t.Skipped(),
+		Fatal:   t.Failed() && !strings.HasPrefix(name, "TestSoon"),
+	})
+}
+
+type caseInfo struct {
+	Name    string `json:"name"`
+	Passed  bool   `json:"passed"`
+	Skipped bool   `json:"skipped,omitempty"`
+	Fatal   bool   `json:"fatal,omitempty"`
+}
+
+type caseInfos struct {
+	lock  sync.Mutex
+	Cases []*caseInfo `json:"cases"`
+}
+
+var (
+	allCaseInfos = caseInfos{}
+)

tools/run_tests/run_interop_tests.py

@@ -661,9 +661,6 @@ try:
           
     if args.http2_interop:
       for test_case in _HTTP2_TEST_CASES:
-        if server_name == "go":
-          # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434
-          continue 
         test_job = cloud_to_cloud_jobspec(http2Interop,
                                           test_case,
                                           server_name,