diff --git a/templates/test/core/end2end/end2end_defs.include b/templates/test/core/end2end/end2end_defs.include index d40ba2065be..91fa1d9502a 100644 --- a/templates/test/core/end2end/end2end_defs.include +++ b/templates/test/core/end2end/end2end_defs.include @@ -27,7 +27,6 @@ #include -#include "test/core/util/debugger_macros.h" static bool g_pre_init_called = false; @@ -39,7 +38,6 @@ extern void ${test}_pre_init(void); void grpc_end2end_tests_pre_init(void) { GPR_ASSERT(!g_pre_init_called); g_pre_init_called = true; - grpc_summon_debugger_macros(); % for test in tests: ${test}_pre_init(); % endfor diff --git a/test/core/end2end/end2end_nosec_tests.cc b/test/core/end2end/end2end_nosec_tests.cc index 3ab55527da6..e01e6ad0104 100644 --- a/test/core/end2end/end2end_nosec_tests.cc +++ b/test/core/end2end/end2end_nosec_tests.cc @@ -26,7 +26,6 @@ #include -#include "test/core/util/debugger_macros.h" static bool g_pre_init_called = false; @@ -188,7 +187,6 @@ extern void write_buffering_at_end_pre_init(void); void grpc_end2end_tests_pre_init(void) { GPR_ASSERT(!g_pre_init_called); g_pre_init_called = true; - grpc_summon_debugger_macros(); authority_not_supported_pre_init(); bad_hostname_pre_init(); bad_ping_pre_init(); diff --git a/test/core/end2end/end2end_tests.cc b/test/core/end2end/end2end_tests.cc index b680da4433f..76fb046b367 100644 --- a/test/core/end2end/end2end_tests.cc +++ b/test/core/end2end/end2end_tests.cc @@ -26,7 +26,6 @@ #include -#include "test/core/util/debugger_macros.h" static bool g_pre_init_called = false; @@ -190,7 +189,6 @@ extern void write_buffering_at_end_pre_init(void); void grpc_end2end_tests_pre_init(void) { GPR_ASSERT(!g_pre_init_called); g_pre_init_called = true; - grpc_summon_debugger_macros(); authority_not_supported_pre_init(); bad_hostname_pre_init(); bad_ping_pre_init(); diff --git a/test/core/util/debugger_macros.cc b/test/core/util/debugger_macros.cc index fed6ad97285..fde68f32178 100644 --- a/test/core/util/debugger_macros.cc +++ b/test/core/util/debugger_macros.cc @@ -21,7 +21,6 @@ * Not intended to be robust for main-line code, often cuts across abstraction * boundaries. */ - #include #include "src/core/ext/filters/client_channel/client_channel.h" @@ -29,8 +28,6 @@ #include "src/core/lib/channel/connected_channel.h" #include "src/core/lib/surface/call.h" -void grpc_summon_debugger_macros() {} - grpc_stream* grpc_transport_stream_from_call(grpc_call* call) { grpc_call_stack* cs = grpc_call_get_call_stack(call); for (;;) { diff --git a/test/core/util/debugger_macros.h b/test/core/util/debugger_macros.h index c6b3720c5ad..71228c6e875 100644 --- a/test/core/util/debugger_macros.h +++ b/test/core/util/debugger_macros.h @@ -19,6 +19,9 @@ #ifndef GRPC_TEST_CORE_UTIL_DEBUGGER_MACROS_H #define GRPC_TEST_CORE_UTIL_DEBUGGER_MACROS_H -void grpc_summon_debugger_macros(); +#include "src/core/ext/transport/chttp2/transport/internal.h" +#include "src/core/lib/surface/call.h" + +grpc_chttp2_stream* grpc_chttp2_stream_from_call(grpc_call* call); #endif /* GRPC_TEST_CORE_UTIL_DEBUGGER_MACROS_H */ diff --git a/test/cpp/end2end/flaky_network_test.cc b/test/cpp/end2end/flaky_network_test.cc index 63a6897f931..626b5a56226 100644 --- a/test/cpp/end2end/flaky_network_test.cc +++ b/test/cpp/end2end/flaky_network_test.cc @@ -16,12 +16,6 @@ * */ -#include -#include -#include -#include -#include - #include #include #include @@ -35,16 +29,22 @@ #include #include #include +#include + +#include +#include +#include +#include +#include #include "src/core/lib/backoff/backoff.h" #include "src/core/lib/gpr/env.h" - #include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "test/core/util/debugger_macros.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" #include "test/cpp/end2end/test_service_impl.h" - -#include +#include "test/cpp/util/test_credentials_provider.h" #ifdef GPR_LINUX using grpc::testing::EchoRequest; @@ -54,14 +54,20 @@ namespace grpc { namespace testing { namespace { -class FlakyNetworkTest : public ::testing::Test { +struct TestScenario { + TestScenario(const grpc::string& creds_type, const grpc::string& content) + : credentials_type(creds_type), message_content(content) {} + const grpc::string credentials_type; + const grpc::string message_content; +}; + +class FlakyNetworkTest : public ::testing::TestWithParam { protected: FlakyNetworkTest() : server_host_("grpctest"), interface_("lo:1"), ipv4_address_("10.0.0.1"), - netmask_("/32"), - kRequestMessage_("🖖") {} + netmask_("/32") {} void InterfaceUp() { std::ostringstream cmd; @@ -129,10 +135,11 @@ class FlakyNetworkTest : public ::testing::Test { void FlakeNetwork() { std::ostringstream cmd; // Emulate a flaky network connection over interface_. Add a delay of 100ms - // +/- 590ms, 3% packet loss, 1% duplicates and 0.1% corrupt packets. + // +/- 20ms, 0.1% packet loss, 1% duplicates and 0.01% corrupt packets. cmd << "tc qdisc replace dev " << interface_ - << " root netem delay 100ms 50ms distribution normal loss 3% duplicate " - "1% corrupt 0.1% "; + << " root netem delay 100ms 20ms distribution normal loss 0.1% " + "duplicate " + "0.1% corrupt 0.01% "; std::system(cmd.str().c_str()); } @@ -172,7 +179,7 @@ class FlakyNetworkTest : public ::testing::Test { // ip6-looopback, but ipv6 support is not enabled by default in docker. port_ = SERVER_PORT; - server_.reset(new ServerData(port_)); + server_.reset(new ServerData(port_, GetParam().credentials_type)); server_->Start(server_host_); } void StopServer() { server_->Shutdown(); } @@ -188,10 +195,11 @@ class FlakyNetworkTest : public ::testing::Test { if (lb_policy_name.size() > 0) { args.SetLoadBalancingPolicyName(lb_policy_name); } // else, default to pick first + auto channel_creds = GetCredentialsProvider()->GetChannelCredentials( + GetParam().credentials_type, &args); std::ostringstream server_address; server_address << server_host_ << ":" << port_; - return CreateCustomChannel(server_address.str(), - InsecureChannelCredentials(), args); + return CreateCustomChannel(server_address.str(), channel_creds, args); } bool SendRpc( @@ -199,7 +207,8 @@ class FlakyNetworkTest : public ::testing::Test { int timeout_ms = 0, bool wait_for_ready = false) { auto response = std::unique_ptr(new EchoResponse()); EchoRequest request; - request.set_message(kRequestMessage_); + auto& msg = GetParam().message_content; + request.set_message(msg); ClientContext context; if (timeout_ms > 0) { context.set_deadline(grpc_timeout_milliseconds_to_deadline(timeout_ms)); @@ -211,22 +220,33 @@ class FlakyNetworkTest : public ::testing::Test { } Status status = stub->Echo(&context, request, response.get()); auto ok = status.ok(); + int stream_id = 0; + grpc_call* call = context.c_call(); + if (call) { + grpc_chttp2_stream* stream = grpc_chttp2_stream_from_call(call); + if (stream) { + stream_id = stream->id; + } + } if (ok) { - gpr_log(GPR_DEBUG, "RPC returned %s\n", response->message().c_str()); + gpr_log(GPR_DEBUG, "RPC with stream_id %d succeeded", stream_id); } else { - gpr_log(GPR_DEBUG, "RPC failed: %s", status.error_message().c_str()); + gpr_log(GPR_DEBUG, "RPC with stream_id %d failed: %s", stream_id, + status.error_message().c_str()); } return ok; } struct ServerData { int port_; + const grpc::string creds_; std::unique_ptr server_; TestServiceImpl service_; std::unique_ptr thread_; bool server_ready_ = false; - explicit ServerData(int port) { port_ = port; } + ServerData(int port, const grpc::string& creds) + : port_(port), creds_(creds) {} void Start(const grpc::string& server_host) { gpr_log(GPR_INFO, "starting server on port %d", port_); @@ -245,8 +265,9 @@ class FlakyNetworkTest : public ::testing::Test { std::ostringstream server_address; server_address << server_host << ":" << port_; ServerBuilder builder; - builder.AddListeningPort(server_address.str(), - InsecureServerCredentials()); + auto server_creds = + GetCredentialsProvider()->GetServerCredentials(creds_); + builder.AddListeningPort(server_address.str(), server_creds); builder.RegisterService(&service_); server_ = builder.BuildAndStart(); std::lock_guard lock(*mu); @@ -291,11 +312,43 @@ class FlakyNetworkTest : public ::testing::Test { std::unique_ptr server_; const int SERVER_PORT = 32750; int port_; - const grpc::string kRequestMessage_; }; +std::vector CreateTestScenarios() { + std::vector scenarios; + std::vector credentials_types; + std::vector messages; + + credentials_types.push_back(kInsecureCredentialsType); + auto sec_list = GetCredentialsProvider()->GetSecureCredentialsTypeList(); + for (auto sec = sec_list.begin(); sec != sec_list.end(); sec++) { + credentials_types.push_back(*sec); + } + + messages.push_back("🖖"); + for (size_t k = 1; k < GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH / 1024; k *= 32) { + grpc::string big_msg; + for (size_t i = 0; i < k * 1024; ++i) { + char c = 'a' + (i % 26); + big_msg += c; + } + messages.push_back(big_msg); + } + for (auto cred = credentials_types.begin(); cred != credentials_types.end(); + ++cred) { + for (auto msg = messages.begin(); msg != messages.end(); msg++) { + scenarios.emplace_back(*cred, *msg); + } + } + + return scenarios; +} + +INSTANTIATE_TEST_CASE_P(FlakyNetworkTest, FlakyNetworkTest, + ::testing::ValuesIn(CreateTestScenarios())); + // Network interface connected to server flaps -TEST_F(FlakyNetworkTest, NetworkTransition) { +TEST_P(FlakyNetworkTest, NetworkTransition) { const int kKeepAliveTimeMs = 1000; const int kKeepAliveTimeoutMs = 1000; ChannelArguments args; @@ -336,7 +389,7 @@ TEST_F(FlakyNetworkTest, NetworkTransition) { } // Traffic to server server is blackholed temporarily with keepalives enabled -TEST_F(FlakyNetworkTest, ServerUnreachableWithKeepalive) { +TEST_P(FlakyNetworkTest, ServerUnreachableWithKeepalive) { const int kKeepAliveTimeMs = 1000; const int kKeepAliveTimeoutMs = 1000; const int kReconnectBackoffMs = 1000; @@ -385,7 +438,7 @@ TEST_F(FlakyNetworkTest, ServerUnreachableWithKeepalive) { // // Traffic to server server is blackholed temporarily with keepalives disabled -TEST_F(FlakyNetworkTest, ServerUnreachableNoKeepalive) { +TEST_P(FlakyNetworkTest, ServerUnreachableNoKeepalive) { auto channel = BuildChannel("pick_first", ChannelArguments()); auto stub = BuildStub(channel); // Channel should be in READY state after we send an RPC @@ -411,7 +464,7 @@ TEST_F(FlakyNetworkTest, ServerUnreachableNoKeepalive) { } // Send RPCs over a flaky network connection -TEST_F(FlakyNetworkTest, FlakyNetwork) { +TEST_P(FlakyNetworkTest, FlakyNetwork) { const int kKeepAliveTimeMs = 1000; const int kKeepAliveTimeoutMs = 1000; const int kMessageCount = 100; @@ -438,7 +491,7 @@ TEST_F(FlakyNetworkTest, FlakyNetwork) { } // Server is shutdown gracefully and restarted. Client keepalives are enabled -TEST_F(FlakyNetworkTest, ServerRestartKeepaliveEnabled) { +TEST_P(FlakyNetworkTest, ServerRestartKeepaliveEnabled) { const int kKeepAliveTimeMs = 1000; const int kKeepAliveTimeoutMs = 1000; ChannelArguments args; @@ -468,7 +521,7 @@ TEST_F(FlakyNetworkTest, ServerRestartKeepaliveEnabled) { } // Server is shutdown gracefully and restarted. Client keepalives are enabled -TEST_F(FlakyNetworkTest, ServerRestartKeepaliveDisabled) { +TEST_P(FlakyNetworkTest, ServerRestartKeepaliveDisabled) { auto channel = BuildChannel("pick_first", ChannelArguments()); auto stub = BuildStub(channel); // Channel should be in READY state after we send an RPC diff --git a/tools/internal_ci/linux/grpc_flaky_network_in_docker.sh b/tools/internal_ci/linux/grpc_flaky_network_in_docker.sh index 7fc8f146727..d574638d3ef 100755 --- a/tools/internal_ci/linux/grpc_flaky_network_in_docker.sh +++ b/tools/internal_ci/linux/grpc_flaky_network_in_docker.sh @@ -28,4 +28,4 @@ cd /var/local/git/grpc/test/cpp/end2end # iptables is used to drop traffic between client and server apt-get install -y iptables -bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=all :flaky_network_test --test_env=GRPC_VERBOSITY=debug --test_env=GRPC_TRACE=channel,client_channel,call_error,connectivity_state,tcp +bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=all --test_timeout=1200 :flaky_network_test --test_env=GRPC_TRACE=http --test_env=GRPC_VERBOSITY=DEBUG