From 55b0405c86230c0d699c1c00aea2d83b5b5c59cd Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 14 Apr 2022 08:32:33 -0700 Subject: [PATCH] xds_end2end_test: Move fault injection tests into their own file (#29283) * move some code around * remove num_backends parameter from XdsEnd2endTest * remove use_xds_enabled_server param from XdsEnd2endTest * remove xds_resource_does_not_exist_timeout_ms param from XdsEnd2endTest * remove client_load_reporting_interval_seconds param from XdsEnd2endTest * start moving CreateAndStartBackends() into individual tests * finish moving CreateAndStartBackends() into individual tests * remove unused variable * remove SetEdsResourceWithDelay * fix test flake * clang-tidy * clang-format * move test framework to its own library * fix build * clang-format * fix windows build * move fault injection tests to their own file * rename TestType to XdsTestType * move BackendServiceImpl inside of BackendServerThread * clang-format * generate_projects * appease clang-tidy * move AdminServerThread to CSDS test suite * remove unnecessary deps * generate_projects * don't mark test as flaky --- CMakeLists.txt | 176 ++++++ build_autogenerated.yaml | 55 ++ test/cpp/end2end/xds/BUILD | 25 + test/cpp/end2end/xds/xds_end2end_test.cc | 507 ---------------- .../xds/xds_fault_injection_end2end_test.cc | 565 ++++++++++++++++++ tools/run_tests/generated/tests.json | 22 + 6 files changed, 843 insertions(+), 507 deletions(-) create mode 100644 test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index a028a7aa91e..a28b69ccbfd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1186,6 +1186,9 @@ if(gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx xds_end2end_test) endif() + if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + add_dependencies(buildtests_cxx xds_fault_injection_end2end_test) + endif() add_dependencies(buildtests_cxx xds_interop_client) add_dependencies(buildtests_cxx xds_interop_server) @@ -17558,6 +17561,179 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) ) +endif() +endif() +if(gRPC_BUILD_TESTS) +if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) + + add_executable(xds_fault_injection_end2end_test + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/duplicate/echo_duplicate.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/duplicate/echo_duplicate.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/simple_messages.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/simple_messages.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/simple_messages.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/simple_messages.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/ads_for_test.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/ads_for_test.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/ads_for_test.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/ads_for_test.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/eds_for_test.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/eds_for_test.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/eds_for_test.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/eds_for_test.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/lrs_for_test.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/lrs_for_test.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/lrs_for_test.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/lrs_for_test.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/address.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/address.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/address.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/address.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/ads.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/ads.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/ads.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/ads.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/base.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/base.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/base.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/base.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/cluster.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/cluster.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/cluster.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/cluster.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/config_source.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/config_source.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/config_source.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/config_source.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/discovery.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/discovery.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/discovery.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/discovery.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/endpoint.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/endpoint.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/endpoint.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/endpoint.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/expr.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/expr.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/expr.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/expr.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/extension.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/extension.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/extension.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/extension.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/fault.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/fault.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/fault.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/fault.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/fault_common.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/fault_common.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/fault_common.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/fault_common.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/http_connection_manager.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/http_connection_manager.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/http_connection_manager.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/http_connection_manager.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/http_filter_rbac.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/http_filter_rbac.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/http_filter_rbac.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/http_filter_rbac.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/listener.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/listener.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/listener.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/listener.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/load_report.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/load_report.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/load_report.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/load_report.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/lrs.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/lrs.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/lrs.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/lrs.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/metadata.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/metadata.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/metadata.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/metadata.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/path.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/path.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/path.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/path.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/percent.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/percent.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/percent.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/percent.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/protocol.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/protocol.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/protocol.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/protocol.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/range.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/range.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/range.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/range.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/rbac.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/rbac.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/rbac.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/rbac.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/regex.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/regex.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/regex.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/regex.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/route.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/route.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/route.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/route.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/router.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/router.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/router.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/router.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/string.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/string.grpc.pb.cc + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/string.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/xds/v3/string.grpc.pb.h + test/cpp/end2end/test_service_impl.cc + test/cpp/end2end/xds/xds_end2end_test_lib.cc + test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc + test/cpp/end2end/xds/xds_server.cc + test/cpp/util/tls_test_utils.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc + ) + + target_include_directories(xds_fault_injection_end2end_test + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} + third_party/googletest/googletest/include + third_party/googletest/googletest + third_party/googletest/googlemock/include + third_party/googletest/googlemock + ${_gRPC_PROTO_GENS_DIR} + ) + + target_link_libraries(xds_fault_injection_end2end_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc++_test_util + ) + + endif() endif() if(gRPC_BUILD_TESTS) diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 77a8dcfe152..e5d11fa98e3 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -8523,6 +8523,61 @@ targets: - linux - posix - mac +- name: xds_fault_injection_end2end_test + gtest: true + build: test + language: c++ + headers: + - test/cpp/end2end/counted_service.h + - test/cpp/end2end/test_service_impl.h + - test/cpp/end2end/xds/xds_end2end_test_lib.h + - test/cpp/end2end/xds/xds_server.h + - test/cpp/util/tls_test_utils.h + src: + - src/proto/grpc/testing/duplicate/echo_duplicate.proto + - src/proto/grpc/testing/echo.proto + - src/proto/grpc/testing/echo_messages.proto + - src/proto/grpc/testing/simple_messages.proto + - src/proto/grpc/testing/xds/ads_for_test.proto + - src/proto/grpc/testing/xds/eds_for_test.proto + - src/proto/grpc/testing/xds/lrs_for_test.proto + - src/proto/grpc/testing/xds/v3/address.proto + - src/proto/grpc/testing/xds/v3/ads.proto + - src/proto/grpc/testing/xds/v3/base.proto + - src/proto/grpc/testing/xds/v3/cluster.proto + - src/proto/grpc/testing/xds/v3/config_source.proto + - src/proto/grpc/testing/xds/v3/discovery.proto + - src/proto/grpc/testing/xds/v3/endpoint.proto + - src/proto/grpc/testing/xds/v3/expr.proto + - src/proto/grpc/testing/xds/v3/extension.proto + - src/proto/grpc/testing/xds/v3/fault.proto + - src/proto/grpc/testing/xds/v3/fault_common.proto + - src/proto/grpc/testing/xds/v3/http_connection_manager.proto + - src/proto/grpc/testing/xds/v3/http_filter_rbac.proto + - src/proto/grpc/testing/xds/v3/listener.proto + - src/proto/grpc/testing/xds/v3/load_report.proto + - src/proto/grpc/testing/xds/v3/lrs.proto + - src/proto/grpc/testing/xds/v3/metadata.proto + - src/proto/grpc/testing/xds/v3/path.proto + - src/proto/grpc/testing/xds/v3/percent.proto + - src/proto/grpc/testing/xds/v3/protocol.proto + - src/proto/grpc/testing/xds/v3/range.proto + - src/proto/grpc/testing/xds/v3/rbac.proto + - src/proto/grpc/testing/xds/v3/regex.proto + - src/proto/grpc/testing/xds/v3/route.proto + - src/proto/grpc/testing/xds/v3/router.proto + - src/proto/grpc/testing/xds/v3/string.proto + - test/cpp/end2end/test_service_impl.cc + - test/cpp/end2end/xds/xds_end2end_test_lib.cc + - test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc + - test/cpp/end2end/xds/xds_server.cc + - test/cpp/util/tls_test_utils.cc + deps: + - grpc++_test_util + platforms: + - linux + - posix + - mac - name: xds_interop_client build: test run: false diff --git a/test/cpp/end2end/xds/BUILD b/test/cpp/end2end/xds/BUILD index 0e82183d65f..c788365bde9 100644 --- a/test/cpp/end2end/xds/BUILD +++ b/test/cpp/end2end/xds/BUILD @@ -165,6 +165,31 @@ grpc_cc_test( ], ) +grpc_cc_test( + name = "xds_fault_injection_end2end_test", + size = "large", + srcs = ["xds_fault_injection_end2end_test.cc"], + external_deps = [ + "gtest", + ], + linkstatic = True, # Fixes dyld error on MacOS + tags = [ + "no_test_ios", + "no_windows", + ], # TODO(jtattermusch): fix test on windows + deps = [ + ":xds_end2end_test_lib", + "//:gpr", + "//:grpc", + "//:grpc++", + "//src/proto/grpc/testing/xds/v3:cluster_proto", + "//src/proto/grpc/testing/xds/v3:fault_proto", + "//src/proto/grpc/testing/xds/v3:http_connection_manager_proto", + "//src/proto/grpc/testing/xds/v3:router_proto", + "//test/core/util:grpc_test_util", + ], +) + grpc_cc_test( name = "xds_credentials_end2end_test", srcs = ["xds_credentials_end2end_test.cc"], diff --git a/test/cpp/end2end/xds/xds_end2end_test.cc b/test/cpp/end2end/xds/xds_end2end_test.cc index 207f3db8a60..505d5836096 100644 --- a/test/cpp/end2end/xds/xds_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_end2end_test.cc @@ -131,8 +131,6 @@ using ::envoy::extensions::clusters::aggregate::v3::ClusterConfig; using ::envoy::extensions::filters::http::fault::v3::HTTPFault; using ::envoy::extensions::filters::http::rbac::v3::RBAC; using ::envoy::extensions::filters::http::rbac::v3::RBACPerRoute; -using ::envoy::extensions::filters::network::http_connection_manager::v3:: - HttpFilter; using ::envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext; using ::envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext; using ::envoy::type::matcher::v3::StringMatcher; @@ -11202,501 +11200,6 @@ TEST_P(ClientLoadReportingTest, DropStats) { ::testing::DoubleNear(kDropRateForThrottle, kErrorTolerance)); } -class FaultInjectionTest : public XdsEnd2endTest { - public: - // Builds a Listener with Fault Injection filter config. If the http_fault - // is nullptr, then assign an empty filter config. This filter config is - // required to enable the fault injection features. - static Listener BuildListenerWithFaultInjection( - const HTTPFault& http_fault = HTTPFault()) { - HttpConnectionManager http_connection_manager; - Listener listener; - listener.set_name(kServerName); - HttpFilter* fault_filter = http_connection_manager.add_http_filters(); - fault_filter->set_name("envoy.fault"); - fault_filter->mutable_typed_config()->PackFrom(http_fault); - HttpFilter* router_filter = http_connection_manager.add_http_filters(); - router_filter->set_name("router"); - router_filter->mutable_typed_config()->PackFrom( - envoy::extensions::filters::http::router::v3::Router()); - listener.mutable_api_listener()->mutable_api_listener()->PackFrom( - http_connection_manager); - return listener; - } - - RouteConfiguration BuildRouteConfigurationWithFaultInjection( - const HTTPFault& http_fault) { - // Package as Any - google::protobuf::Any filter_config; - filter_config.PackFrom(http_fault); - // Plug into the RouteConfiguration - RouteConfiguration new_route_config = default_route_config_; - auto* config_map = new_route_config.mutable_virtual_hosts(0) - ->mutable_routes(0) - ->mutable_typed_per_filter_config(); - (*config_map)["envoy.fault"] = std::move(filter_config); - return new_route_config; - } - - void SetFilterConfig(HTTPFault& http_fault) { - switch (GetParam().filter_config_setup()) { - case XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute: { - Listener listener = BuildListenerWithFaultInjection(); - RouteConfiguration route = - BuildRouteConfigurationWithFaultInjection(http_fault); - SetListenerAndRouteConfiguration(balancer_.get(), listener, route); - break; - } - case XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInListener: { - Listener listener = BuildListenerWithFaultInjection(http_fault); - SetListenerAndRouteConfiguration(balancer_.get(), listener, - default_route_config_); - } - }; - } -}; - -// Test to ensure the most basic fault injection config works. -TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysAbort) { - const uint32_t kAbortPercentagePerHundred = 100; - // Construct the fault injection filter config - HTTPFault http_fault; - auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); - abort_percentage->set_numerator(kAbortPercentagePerHundred); - abort_percentage->set_denominator(FractionalPercent::HUNDRED); - http_fault.mutable_abort()->set_grpc_status( - static_cast(StatusCode::ABORTED)); - // Config fault injection via different setup - SetFilterConfig(http_fault); - // Fire several RPCs, and expect all of them to be aborted. - CheckRpcSendFailure( - CheckRpcSendFailureOptions() - .set_times(5) - .set_rpc_options(RpcOptions().set_wait_for_ready(true)) - .set_expected_error_code(StatusCode::ABORTED)); -} - -// Without the listener config, the fault injection won't be enabled. -TEST_P(FaultInjectionTest, XdsFaultInjectionWithoutListenerFilter) { - CreateAndStartBackends(1); - const uint32_t kAbortPercentagePerHundred = 100; - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Construct the fault injection filter config - HTTPFault http_fault; - auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); - abort_percentage->set_numerator(kAbortPercentagePerHundred); - abort_percentage->set_denominator(FractionalPercent::HUNDRED); - http_fault.mutable_abort()->set_grpc_status( - static_cast(StatusCode::ABORTED)); - // Turn on fault injection - RouteConfiguration route = - BuildRouteConfigurationWithFaultInjection(http_fault); - SetListenerAndRouteConfiguration(balancer_.get(), default_listener_, route); - // Fire several RPCs, and expect all of them to be pass. - CheckRpcSendOk(5, RpcOptions().set_wait_for_ready(true)); -} - -TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbort) { - CreateAndStartBackends(1); - const uint32_t kAbortPercentagePerHundred = 50; - const double kAbortRate = kAbortPercentagePerHundred / 100.0; - const double kErrorTolerance = 0.05; - const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance); - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Construct the fault injection filter config - HTTPFault http_fault; - auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); - abort_percentage->set_numerator(kAbortPercentagePerHundred); - abort_percentage->set_denominator(FractionalPercent::HUNDRED); - http_fault.mutable_abort()->set_grpc_status( - static_cast(StatusCode::ABORTED)); - // Config fault injection via different setup - SetFilterConfig(http_fault); - // Send kNumRpcs RPCs and count the aborts. - size_t num_aborted = - SendRpcsAndCountFailuresWithMessage(kNumRpcs, "Fault injected"); - // The abort rate should be roughly equal to the expectation. - const double seen_abort_rate = static_cast(num_aborted) / kNumRpcs; - EXPECT_THAT(seen_abort_rate, - ::testing::DoubleNear(kAbortRate, kErrorTolerance)); -} - -TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbortViaHeaders) { - CreateAndStartBackends(1); - const uint32_t kAbortPercentageCap = 100; - const uint32_t kAbortPercentage = 50; - const double kAbortRate = kAbortPercentage / 100.0; - const double kErrorTolerance = 0.05; - const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance); - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Construct the fault injection filter config - HTTPFault http_fault; - http_fault.mutable_abort()->mutable_header_abort(); - http_fault.mutable_abort()->mutable_percentage()->set_numerator( - kAbortPercentageCap); - // Config fault injection via different setup - SetFilterConfig(http_fault); - // Send kNumRpcs RPCs and count the aborts. - std::vector> metadata = { - {"x-envoy-fault-abort-grpc-request", "10"}, - {"x-envoy-fault-abort-percentage", std::to_string(kAbortPercentage)}, - }; - size_t num_aborted = SendRpcsAndCountFailuresWithMessage( - kNumRpcs, "Fault injected", RpcOptions().set_metadata(metadata)); - // The abort rate should be roughly equal to the expectation. - const double seen_abort_rate = static_cast(num_aborted) / kNumRpcs; - EXPECT_THAT(seen_abort_rate, - ::testing::DoubleNear(kAbortRate, kErrorTolerance)); -} - -TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) { - CreateAndStartBackends(1); - const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 3000; - const uint32_t kFixedDelaySeconds = 100; - const uint32_t kDelayPercentagePerHundred = 50; - const double kDelayRate = kDelayPercentagePerHundred / 100.0; - const double kErrorTolerance = 0.05; - const size_t kNumRpcs = ComputeIdealNumRpcs(kDelayRate, kErrorTolerance); - const size_t kMaxConcurrentRequests = kNumRpcs; - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Loosen the max concurrent request limit - Cluster cluster = default_cluster_; - auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds(); - threshold->set_priority(RoutingPriority::DEFAULT); - threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests); - balancer_->ads_service()->SetCdsResource(cluster); - // Construct the fault injection filter config - HTTPFault http_fault; - auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); - delay_percentage->set_numerator(kDelayPercentagePerHundred); - delay_percentage->set_denominator(FractionalPercent::HUNDRED); - auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); - fixed_delay->set_seconds(kFixedDelaySeconds); - // Config fault injection via different setup - SetFilterConfig(http_fault); - // Send kNumRpcs RPCs and count the delays. - RpcOptions rpc_options = RpcOptions() - .set_timeout_ms(kRpcTimeoutMilliseconds) - .set_skip_cancelled_check(true); - std::vector rpcs = - SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options); - size_t num_delayed = 0; - for (auto& rpc : rpcs) { - if (rpc.status.error_code() == StatusCode::OK) continue; - EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code()); - ++num_delayed; - } - // The delay rate should be roughly equal to the expectation. - const double seen_delay_rate = static_cast(num_delayed) / kNumRpcs; - EXPECT_THAT(seen_delay_rate, - ::testing::DoubleNear(kDelayRate, kErrorTolerance)); -} - -TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) { - CreateAndStartBackends(1); - const uint32_t kFixedDelayMilliseconds = 100000; - const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 3000; - const uint32_t kDelayPercentageCap = 100; - const uint32_t kDelayPercentage = 50; - const double kDelayRate = kDelayPercentage / 100.0; - const double kErrorTolerance = 0.05; - const size_t kNumRpcs = ComputeIdealNumRpcs(kDelayRate, kErrorTolerance); - const size_t kMaxConcurrentRequests = kNumRpcs; - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Loosen the max concurrent request limit - Cluster cluster = default_cluster_; - auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds(); - threshold->set_priority(RoutingPriority::DEFAULT); - threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests); - balancer_->ads_service()->SetCdsResource(cluster); - // Construct the fault injection filter config - HTTPFault http_fault; - http_fault.mutable_delay()->mutable_header_delay(); - http_fault.mutable_delay()->mutable_percentage()->set_numerator( - kDelayPercentageCap); - // Config fault injection via different setup - SetFilterConfig(http_fault); - // Send kNumRpcs RPCs and count the delays. - std::vector> metadata = { - {"x-envoy-fault-delay-request", std::to_string(kFixedDelayMilliseconds)}, - {"x-envoy-fault-delay-request-percentage", - std::to_string(kDelayPercentage)}, - }; - RpcOptions rpc_options = RpcOptions() - .set_metadata(metadata) - .set_timeout_ms(kRpcTimeoutMilliseconds) - .set_skip_cancelled_check(true); - std::vector rpcs = - SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options); - size_t num_delayed = 0; - for (auto& rpc : rpcs) { - if (rpc.status.error_code() == StatusCode::OK) continue; - EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code()); - ++num_delayed; - } - // The delay rate should be roughly equal to the expectation. - const double seen_delay_rate = static_cast(num_delayed) / kNumRpcs; - EXPECT_THAT(seen_delay_rate, - ::testing::DoubleNear(kDelayRate, kErrorTolerance)); -} - -TEST_P(FaultInjectionTest, XdsFaultInjectionAbortAfterDelayForStreamCall) { - CreateAndStartBackends(1); - const uint32_t kFixedDelaySeconds = 1; - const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Construct the fault injection filter config - HTTPFault http_fault; - auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); - abort_percentage->set_numerator(100); // Always inject ABORT! - abort_percentage->set_denominator(FractionalPercent::HUNDRED); - http_fault.mutable_abort()->set_grpc_status( - static_cast(StatusCode::ABORTED)); - auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); - delay_percentage->set_numerator(100); // Always inject DELAY! - delay_percentage->set_denominator(FractionalPercent::HUNDRED); - auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); - fixed_delay->set_seconds(kFixedDelaySeconds); - // Config fault injection via different setup - SetFilterConfig(http_fault); - // Send a stream RPC and check its status code - ClientContext context; - context.set_deadline( - grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds)); - auto stream = stub_->BidiStream(&context); - stream->WritesDone(); - auto status = stream->Finish(); - EXPECT_EQ(StatusCode::ABORTED, status.error_code()) - << status.error_message() << ", " << status.error_details() << ", " - << context.debug_error_string(); -} - -TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) { - CreateAndStartBackends(1); - const uint32_t kAbortPercentagePerHundred = 50; - const double kAbortRate = kAbortPercentagePerHundred / 100.0; - const uint32_t kFixedDelaySeconds = 1; - const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach - const uint32_t kConnectionTimeoutMilliseconds = - 10 * 1000; // 10s should not reach - const double kErrorTolerance = 0.05; - const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance); - const size_t kMaxConcurrentRequests = kNumRpcs; - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Loosen the max concurrent request limit - Cluster cluster = default_cluster_; - auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds(); - threshold->set_priority(RoutingPriority::DEFAULT); - threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests); - balancer_->ads_service()->SetCdsResource(cluster); - // Construct the fault injection filter config - HTTPFault http_fault; - auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); - abort_percentage->set_numerator(kAbortPercentagePerHundred); - abort_percentage->set_denominator(FractionalPercent::HUNDRED); - http_fault.mutable_abort()->set_grpc_status( - static_cast(StatusCode::ABORTED)); - auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); - delay_percentage->set_numerator(1000000); // Always inject DELAY! - delay_percentage->set_denominator(FractionalPercent::MILLION); - auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); - fixed_delay->set_seconds(kFixedDelaySeconds); - // Config fault injection via different setup - SetFilterConfig(http_fault); - // Allow the channel to connect to one backends, so the herd of queued RPCs - // won't be executed on the same ExecCtx object and using the cached Now() - // value, which causes millisecond level delay error. - channel_->WaitForConnected( - grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)); - // Send kNumRpcs RPCs and count the aborts. - int num_aborted = 0; - RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds); - std::vector rpcs = - SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options); - for (auto& rpc : rpcs) { - EXPECT_GE(rpc.elapsed_time, - grpc_core::Duration::Seconds(kFixedDelaySeconds)); - if (rpc.status.error_code() == StatusCode::OK) continue; - EXPECT_EQ("Fault injected", rpc.status.error_message()); - ++num_aborted; - } - // The abort rate should be roughly equal to the expectation. - const double seen_abort_rate = static_cast(num_aborted) / kNumRpcs; - EXPECT_THAT(seen_abort_rate, - ::testing::DoubleNear(kAbortRate, kErrorTolerance)); -} - -// This test and the above test apply different denominators to delay and -// abort. This ensures that we are using the right denominator for each -// injected fault in our code. -TEST_P(FaultInjectionTest, - XdsFaultInjectionAlwaysDelayPercentageAbortSwitchDenominator) { - CreateAndStartBackends(1); - const uint32_t kAbortPercentagePerMillion = 500000; - const double kAbortRate = kAbortPercentagePerMillion / 1000000.0; - const uint32_t kFixedDelaySeconds = 1; // 1s - const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach - const uint32_t kConnectionTimeoutMilliseconds = - 10 * 1000; // 10s should not reach - const double kErrorTolerance = 0.05; - const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance); - const size_t kMaxConcurrentRequests = kNumRpcs; - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Loosen the max concurrent request limit - Cluster cluster = default_cluster_; - auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds(); - threshold->set_priority(RoutingPriority::DEFAULT); - threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests); - balancer_->ads_service()->SetCdsResource(cluster); - // Construct the fault injection filter config - HTTPFault http_fault; - auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); - abort_percentage->set_numerator(kAbortPercentagePerMillion); - abort_percentage->set_denominator(FractionalPercent::MILLION); - http_fault.mutable_abort()->set_grpc_status( - static_cast(StatusCode::ABORTED)); - auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); - delay_percentage->set_numerator(100); // Always inject DELAY! - delay_percentage->set_denominator(FractionalPercent::HUNDRED); - auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); - fixed_delay->set_seconds(kFixedDelaySeconds); - // Config fault injection via different setup - SetFilterConfig(http_fault); - // Allow the channel to connect to one backends, so the herd of queued RPCs - // won't be executed on the same ExecCtx object and using the cached Now() - // value, which causes millisecond level delay error. - channel_->WaitForConnected( - grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)); - // Send kNumRpcs RPCs and count the aborts. - int num_aborted = 0; - RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds); - std::vector rpcs = - SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options); - for (auto& rpc : rpcs) { - EXPECT_GE(rpc.elapsed_time, - grpc_core::Duration::Seconds(kFixedDelaySeconds)); - if (rpc.status.error_code() == StatusCode::OK) continue; - EXPECT_EQ("Fault injected", rpc.status.error_message()); - ++num_aborted; - } - // The abort rate should be roughly equal to the expectation. - const double seen_abort_rate = static_cast(num_aborted) / kNumRpcs; - EXPECT_THAT(seen_abort_rate, - ::testing::DoubleNear(kAbortRate, kErrorTolerance)); -} - -TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) { - CreateAndStartBackends(1); - const uint32_t kMaxFault = 10; - const uint32_t kNumRpcs = 30; // kNumRpcs should be bigger than kMaxFault - const uint32_t kRpcTimeoutMs = 4000; // 4 seconds - const uint32_t kLongDelaySeconds = 100; // 100 seconds - const uint32_t kAlwaysDelayPercentage = 100; - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Construct the fault injection filter config - HTTPFault http_fault; - auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); - delay_percentage->set_numerator( - kAlwaysDelayPercentage); // Always inject DELAY! - delay_percentage->set_denominator(FractionalPercent::HUNDRED); - auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); - fixed_delay->set_seconds(kLongDelaySeconds); - http_fault.mutable_max_active_faults()->set_value(kMaxFault); - // Config fault injection via different setup - SetFilterConfig(http_fault); - // Sends a batch of long running RPCs with long timeout to consume all - // active faults quota. - int num_delayed = 0; - RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMs); - std::vector rpcs = - SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options); - for (auto& rpc : rpcs) { - if (rpc.status.error_code() == StatusCode::OK) continue; - EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code()); - ++num_delayed; - } - // Only kMaxFault number of RPC should be fault injected.. - EXPECT_EQ(kMaxFault, num_delayed); -} - -TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) { - CreateAndStartBackends(1); - // kRpcTimeoutMilliseconds is 10s should never be reached. - const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 10000; - const uint32_t kFixedDelaySeconds = 1; - const uint32_t kDelayPercentagePerHundred = 100; - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Construct the fault injection filter config - HTTPFault http_fault; - auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); - delay_percentage->set_numerator(kDelayPercentagePerHundred); - delay_percentage->set_denominator(FractionalPercent::HUNDRED); - auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); - fixed_delay->set_seconds(kFixedDelaySeconds); - // Config fault injection via different setup - SetFilterConfig(http_fault); - ClientContext context; - context.set_deadline( - grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds)); - auto stream = stub_->BidiStream(&context); - stream->WritesDone(); - auto status = stream->Finish(); - EXPECT_TRUE(status.ok()) << status.error_message() << ", " - << status.error_details() << ", " - << context.debug_error_string(); -} - -// This case catches a bug in the retry code that was triggered by a bad -// interaction with the FI code. See https://github.com/grpc/grpc/pull/27217 -// for description. -TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayError) { - CreateAndStartBackends(1); - const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 500; - const uint32_t kFixedDelaySeconds = 100; - const uint32_t kDelayPercentagePerHundred = 100; - // Create an EDS resource - EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); - balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); - // Construct the fault injection filter config - HTTPFault http_fault; - auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); - delay_percentage->set_numerator(kDelayPercentagePerHundred); - delay_percentage->set_denominator(FractionalPercent::HUNDRED); - auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); - fixed_delay->set_seconds(kFixedDelaySeconds); - // Config fault injection via different setup - SetFilterConfig(http_fault); - ClientContext context; - context.set_deadline( - grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds)); - auto stream = stub_->BidiStream(&context); - stream->WritesDone(); - auto status = stream->Finish(); - EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, status.error_code()) - << status.error_message() << ", " << status.error_details() << ", " - << context.debug_error_string(); -} - using BootstrapSourceTest = XdsEnd2endTest; TEST_P(BootstrapSourceTest, Vanilla) { @@ -12001,16 +11504,6 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Values(XdsTestType().set_enable_load_reporting()), &XdsTestType::Name); -INSTANTIATE_TEST_SUITE_P( - XdsTest, FaultInjectionTest, - ::testing::Values( - XdsTestType(), XdsTestType().set_enable_rds_testing(), - XdsTestType().set_filter_config_setup( - XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute), - XdsTestType().set_enable_rds_testing().set_filter_config_setup( - XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute)), - &XdsTestType::Name); - INSTANTIATE_TEST_SUITE_P( XdsTest, BootstrapSourceTest, ::testing::Values( diff --git a/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc b/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc new file mode 100644 index 00000000000..ca53412a98e --- /dev/null +++ b/test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc @@ -0,0 +1,565 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include +#include + +#include +#include + +#include "src/core/ext/filters/client_channel/backup_poller.h" +#include "src/proto/grpc/testing/xds/v3/cluster.grpc.pb.h" +#include "src/proto/grpc/testing/xds/v3/fault.grpc.pb.h" +#include "src/proto/grpc/testing/xds/v3/http_connection_manager.grpc.pb.h" +#include "src/proto/grpc/testing/xds/v3/router.grpc.pb.h" +#include "test/core/util/test_config.h" +#include "test/cpp/end2end/xds/xds_end2end_test_lib.h" + +namespace grpc { +namespace testing { +namespace { + +using ::envoy::config::cluster::v3::RoutingPriority; +using ::envoy::extensions::filters::http::fault::v3::HTTPFault; +using ::envoy::extensions::filters::network::http_connection_manager::v3:: + HttpFilter; +using ::envoy::type::v3::FractionalPercent; + +class FaultInjectionTest : public XdsEnd2endTest { + public: + // Builds a Listener with Fault Injection filter config. If the http_fault + // is nullptr, then assign an empty filter config. This filter config is + // required to enable the fault injection features. + static Listener BuildListenerWithFaultInjection( + const HTTPFault& http_fault = HTTPFault()) { + HttpConnectionManager http_connection_manager; + Listener listener; + listener.set_name(kServerName); + HttpFilter* fault_filter = http_connection_manager.add_http_filters(); + fault_filter->set_name("envoy.fault"); + fault_filter->mutable_typed_config()->PackFrom(http_fault); + HttpFilter* router_filter = http_connection_manager.add_http_filters(); + router_filter->set_name("router"); + router_filter->mutable_typed_config()->PackFrom( + envoy::extensions::filters::http::router::v3::Router()); + listener.mutable_api_listener()->mutable_api_listener()->PackFrom( + http_connection_manager); + return listener; + } + + RouteConfiguration BuildRouteConfigurationWithFaultInjection( + const HTTPFault& http_fault) { + // Package as Any + google::protobuf::Any filter_config; + filter_config.PackFrom(http_fault); + // Plug into the RouteConfiguration + RouteConfiguration new_route_config = default_route_config_; + auto* config_map = new_route_config.mutable_virtual_hosts(0) + ->mutable_routes(0) + ->mutable_typed_per_filter_config(); + (*config_map)["envoy.fault"] = std::move(filter_config); + return new_route_config; + } + + void SetFilterConfig(HTTPFault& http_fault) { + switch (GetParam().filter_config_setup()) { + case XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute: { + Listener listener = BuildListenerWithFaultInjection(); + RouteConfiguration route = + BuildRouteConfigurationWithFaultInjection(http_fault); + SetListenerAndRouteConfiguration(balancer_.get(), listener, route); + break; + } + case XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInListener: { + Listener listener = BuildListenerWithFaultInjection(http_fault); + SetListenerAndRouteConfiguration(balancer_.get(), listener, + default_route_config_); + } + }; + } +}; + +// Run with all combinations of RDS disabled/enabled and the HTTP filter +// config in the Listener vs. in the Route. +INSTANTIATE_TEST_SUITE_P( + XdsTest, FaultInjectionTest, + ::testing::Values( + XdsTestType(), XdsTestType().set_enable_rds_testing(), + XdsTestType().set_filter_config_setup( + XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute), + XdsTestType().set_enable_rds_testing().set_filter_config_setup( + XdsTestType::HttpFilterConfigLocation::kHttpFilterConfigInRoute)), + &XdsTestType::Name); + +// Test to ensure the most basic fault injection config works. +TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysAbort) { + const uint32_t kAbortPercentagePerHundred = 100; + // Construct the fault injection filter config + HTTPFault http_fault; + auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); + abort_percentage->set_numerator(kAbortPercentagePerHundred); + abort_percentage->set_denominator(FractionalPercent::HUNDRED); + http_fault.mutable_abort()->set_grpc_status( + static_cast(StatusCode::ABORTED)); + // Config fault injection via different setup + SetFilterConfig(http_fault); + // Fire several RPCs, and expect all of them to be aborted. + CheckRpcSendFailure( + CheckRpcSendFailureOptions() + .set_times(5) + .set_rpc_options(RpcOptions().set_wait_for_ready(true)) + .set_expected_error_code(StatusCode::ABORTED)); +} + +// Without the listener config, the fault injection won't be enabled. +TEST_P(FaultInjectionTest, XdsFaultInjectionWithoutListenerFilter) { + CreateAndStartBackends(1); + const uint32_t kAbortPercentagePerHundred = 100; + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Construct the fault injection filter config + HTTPFault http_fault; + auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); + abort_percentage->set_numerator(kAbortPercentagePerHundred); + abort_percentage->set_denominator(FractionalPercent::HUNDRED); + http_fault.mutable_abort()->set_grpc_status( + static_cast(StatusCode::ABORTED)); + // Turn on fault injection + RouteConfiguration route = + BuildRouteConfigurationWithFaultInjection(http_fault); + SetListenerAndRouteConfiguration(balancer_.get(), default_listener_, route); + // Fire several RPCs, and expect all of them to be pass. + CheckRpcSendOk(5, RpcOptions().set_wait_for_ready(true)); +} + +TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbort) { + CreateAndStartBackends(1); + const uint32_t kAbortPercentagePerHundred = 50; + const double kAbortRate = kAbortPercentagePerHundred / 100.0; + const double kErrorTolerance = 0.05; + const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance); + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Construct the fault injection filter config + HTTPFault http_fault; + auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); + abort_percentage->set_numerator(kAbortPercentagePerHundred); + abort_percentage->set_denominator(FractionalPercent::HUNDRED); + http_fault.mutable_abort()->set_grpc_status( + static_cast(StatusCode::ABORTED)); + // Config fault injection via different setup + SetFilterConfig(http_fault); + // Send kNumRpcs RPCs and count the aborts. + size_t num_aborted = + SendRpcsAndCountFailuresWithMessage(kNumRpcs, "Fault injected"); + // The abort rate should be roughly equal to the expectation. + const double seen_abort_rate = static_cast(num_aborted) / kNumRpcs; + EXPECT_THAT(seen_abort_rate, + ::testing::DoubleNear(kAbortRate, kErrorTolerance)); +} + +TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbortViaHeaders) { + CreateAndStartBackends(1); + const uint32_t kAbortPercentageCap = 100; + const uint32_t kAbortPercentage = 50; + const double kAbortRate = kAbortPercentage / 100.0; + const double kErrorTolerance = 0.05; + const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance); + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Construct the fault injection filter config + HTTPFault http_fault; + http_fault.mutable_abort()->mutable_header_abort(); + http_fault.mutable_abort()->mutable_percentage()->set_numerator( + kAbortPercentageCap); + // Config fault injection via different setup + SetFilterConfig(http_fault); + // Send kNumRpcs RPCs and count the aborts. + std::vector> metadata = { + {"x-envoy-fault-abort-grpc-request", "10"}, + {"x-envoy-fault-abort-percentage", std::to_string(kAbortPercentage)}, + }; + size_t num_aborted = SendRpcsAndCountFailuresWithMessage( + kNumRpcs, "Fault injected", RpcOptions().set_metadata(metadata)); + // The abort rate should be roughly equal to the expectation. + const double seen_abort_rate = static_cast(num_aborted) / kNumRpcs; + EXPECT_THAT(seen_abort_rate, + ::testing::DoubleNear(kAbortRate, kErrorTolerance)); +} + +TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) { + CreateAndStartBackends(1); + const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 3000; + const uint32_t kFixedDelaySeconds = 100; + const uint32_t kDelayPercentagePerHundred = 50; + const double kDelayRate = kDelayPercentagePerHundred / 100.0; + const double kErrorTolerance = 0.05; + const size_t kNumRpcs = ComputeIdealNumRpcs(kDelayRate, kErrorTolerance); + const size_t kMaxConcurrentRequests = kNumRpcs; + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Loosen the max concurrent request limit + Cluster cluster = default_cluster_; + auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds(); + threshold->set_priority(RoutingPriority::DEFAULT); + threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests); + balancer_->ads_service()->SetCdsResource(cluster); + // Construct the fault injection filter config + HTTPFault http_fault; + auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); + delay_percentage->set_numerator(kDelayPercentagePerHundred); + delay_percentage->set_denominator(FractionalPercent::HUNDRED); + auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); + fixed_delay->set_seconds(kFixedDelaySeconds); + // Config fault injection via different setup + SetFilterConfig(http_fault); + // Send kNumRpcs RPCs and count the delays. + RpcOptions rpc_options = RpcOptions() + .set_timeout_ms(kRpcTimeoutMilliseconds) + .set_skip_cancelled_check(true); + std::vector rpcs = + SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options); + size_t num_delayed = 0; + for (auto& rpc : rpcs) { + if (rpc.status.error_code() == StatusCode::OK) continue; + EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code()); + ++num_delayed; + } + // The delay rate should be roughly equal to the expectation. + const double seen_delay_rate = static_cast(num_delayed) / kNumRpcs; + EXPECT_THAT(seen_delay_rate, + ::testing::DoubleNear(kDelayRate, kErrorTolerance)); +} + +TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) { + CreateAndStartBackends(1); + const uint32_t kFixedDelayMilliseconds = 100000; + const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 3000; + const uint32_t kDelayPercentageCap = 100; + const uint32_t kDelayPercentage = 50; + const double kDelayRate = kDelayPercentage / 100.0; + const double kErrorTolerance = 0.05; + const size_t kNumRpcs = ComputeIdealNumRpcs(kDelayRate, kErrorTolerance); + const size_t kMaxConcurrentRequests = kNumRpcs; + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Loosen the max concurrent request limit + Cluster cluster = default_cluster_; + auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds(); + threshold->set_priority(RoutingPriority::DEFAULT); + threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests); + balancer_->ads_service()->SetCdsResource(cluster); + // Construct the fault injection filter config + HTTPFault http_fault; + http_fault.mutable_delay()->mutable_header_delay(); + http_fault.mutable_delay()->mutable_percentage()->set_numerator( + kDelayPercentageCap); + // Config fault injection via different setup + SetFilterConfig(http_fault); + // Send kNumRpcs RPCs and count the delays. + std::vector> metadata = { + {"x-envoy-fault-delay-request", std::to_string(kFixedDelayMilliseconds)}, + {"x-envoy-fault-delay-request-percentage", + std::to_string(kDelayPercentage)}, + }; + RpcOptions rpc_options = RpcOptions() + .set_metadata(metadata) + .set_timeout_ms(kRpcTimeoutMilliseconds) + .set_skip_cancelled_check(true); + std::vector rpcs = + SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options); + size_t num_delayed = 0; + for (auto& rpc : rpcs) { + if (rpc.status.error_code() == StatusCode::OK) continue; + EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code()); + ++num_delayed; + } + // The delay rate should be roughly equal to the expectation. + const double seen_delay_rate = static_cast(num_delayed) / kNumRpcs; + EXPECT_THAT(seen_delay_rate, + ::testing::DoubleNear(kDelayRate, kErrorTolerance)); +} + +TEST_P(FaultInjectionTest, XdsFaultInjectionAbortAfterDelayForStreamCall) { + CreateAndStartBackends(1); + const uint32_t kFixedDelaySeconds = 1; + const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Construct the fault injection filter config + HTTPFault http_fault; + auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); + abort_percentage->set_numerator(100); // Always inject ABORT! + abort_percentage->set_denominator(FractionalPercent::HUNDRED); + http_fault.mutable_abort()->set_grpc_status( + static_cast(StatusCode::ABORTED)); + auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); + delay_percentage->set_numerator(100); // Always inject DELAY! + delay_percentage->set_denominator(FractionalPercent::HUNDRED); + auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); + fixed_delay->set_seconds(kFixedDelaySeconds); + // Config fault injection via different setup + SetFilterConfig(http_fault); + // Send a stream RPC and check its status code + ClientContext context; + context.set_deadline( + grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds)); + auto stream = stub_->BidiStream(&context); + stream->WritesDone(); + auto status = stream->Finish(); + EXPECT_EQ(StatusCode::ABORTED, status.error_code()) + << status.error_message() << ", " << status.error_details() << ", " + << context.debug_error_string(); +} + +TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) { + CreateAndStartBackends(1); + const uint32_t kAbortPercentagePerHundred = 50; + const double kAbortRate = kAbortPercentagePerHundred / 100.0; + const uint32_t kFixedDelaySeconds = 1; + const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach + const uint32_t kConnectionTimeoutMilliseconds = + 10 * 1000; // 10s should not reach + const double kErrorTolerance = 0.05; + const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance); + const size_t kMaxConcurrentRequests = kNumRpcs; + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Loosen the max concurrent request limit + Cluster cluster = default_cluster_; + auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds(); + threshold->set_priority(RoutingPriority::DEFAULT); + threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests); + balancer_->ads_service()->SetCdsResource(cluster); + // Construct the fault injection filter config + HTTPFault http_fault; + auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); + abort_percentage->set_numerator(kAbortPercentagePerHundred); + abort_percentage->set_denominator(FractionalPercent::HUNDRED); + http_fault.mutable_abort()->set_grpc_status( + static_cast(StatusCode::ABORTED)); + auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); + delay_percentage->set_numerator(1000000); // Always inject DELAY! + delay_percentage->set_denominator(FractionalPercent::MILLION); + auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); + fixed_delay->set_seconds(kFixedDelaySeconds); + // Config fault injection via different setup + SetFilterConfig(http_fault); + // Allow the channel to connect to one backends, so the herd of queued RPCs + // won't be executed on the same ExecCtx object and using the cached Now() + // value, which causes millisecond level delay error. + channel_->WaitForConnected( + grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)); + // Send kNumRpcs RPCs and count the aborts. + int num_aborted = 0; + RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds); + std::vector rpcs = + SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options); + for (auto& rpc : rpcs) { + EXPECT_GE(rpc.elapsed_time, + grpc_core::Duration::Seconds(kFixedDelaySeconds)); + if (rpc.status.error_code() == StatusCode::OK) continue; + EXPECT_EQ("Fault injected", rpc.status.error_message()); + ++num_aborted; + } + // The abort rate should be roughly equal to the expectation. + const double seen_abort_rate = static_cast(num_aborted) / kNumRpcs; + EXPECT_THAT(seen_abort_rate, + ::testing::DoubleNear(kAbortRate, kErrorTolerance)); +} + +// This test and the above test apply different denominators to delay and +// abort. This ensures that we are using the right denominator for each +// injected fault in our code. +TEST_P(FaultInjectionTest, + XdsFaultInjectionAlwaysDelayPercentageAbortSwitchDenominator) { + CreateAndStartBackends(1); + const uint32_t kAbortPercentagePerMillion = 500000; + const double kAbortRate = kAbortPercentagePerMillion / 1000000.0; + const uint32_t kFixedDelaySeconds = 1; // 1s + const uint32_t kRpcTimeoutMilliseconds = 100 * 1000; // 100s should not reach + const uint32_t kConnectionTimeoutMilliseconds = + 10 * 1000; // 10s should not reach + const double kErrorTolerance = 0.05; + const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance); + const size_t kMaxConcurrentRequests = kNumRpcs; + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Loosen the max concurrent request limit + Cluster cluster = default_cluster_; + auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds(); + threshold->set_priority(RoutingPriority::DEFAULT); + threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests); + balancer_->ads_service()->SetCdsResource(cluster); + // Construct the fault injection filter config + HTTPFault http_fault; + auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage(); + abort_percentage->set_numerator(kAbortPercentagePerMillion); + abort_percentage->set_denominator(FractionalPercent::MILLION); + http_fault.mutable_abort()->set_grpc_status( + static_cast(StatusCode::ABORTED)); + auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); + delay_percentage->set_numerator(100); // Always inject DELAY! + delay_percentage->set_denominator(FractionalPercent::HUNDRED); + auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); + fixed_delay->set_seconds(kFixedDelaySeconds); + // Config fault injection via different setup + SetFilterConfig(http_fault); + // Allow the channel to connect to one backends, so the herd of queued RPCs + // won't be executed on the same ExecCtx object and using the cached Now() + // value, which causes millisecond level delay error. + channel_->WaitForConnected( + grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)); + // Send kNumRpcs RPCs and count the aborts. + int num_aborted = 0; + RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds); + std::vector rpcs = + SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options); + for (auto& rpc : rpcs) { + EXPECT_GE(rpc.elapsed_time, + grpc_core::Duration::Seconds(kFixedDelaySeconds)); + if (rpc.status.error_code() == StatusCode::OK) continue; + EXPECT_EQ("Fault injected", rpc.status.error_message()); + ++num_aborted; + } + // The abort rate should be roughly equal to the expectation. + const double seen_abort_rate = static_cast(num_aborted) / kNumRpcs; + EXPECT_THAT(seen_abort_rate, + ::testing::DoubleNear(kAbortRate, kErrorTolerance)); +} + +TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) { + CreateAndStartBackends(1); + const uint32_t kMaxFault = 10; + const uint32_t kNumRpcs = 30; // kNumRpcs should be bigger than kMaxFault + const uint32_t kRpcTimeoutMs = 4000; // 4 seconds + const uint32_t kLongDelaySeconds = 100; // 100 seconds + const uint32_t kAlwaysDelayPercentage = 100; + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Construct the fault injection filter config + HTTPFault http_fault; + auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); + delay_percentage->set_numerator( + kAlwaysDelayPercentage); // Always inject DELAY! + delay_percentage->set_denominator(FractionalPercent::HUNDRED); + auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); + fixed_delay->set_seconds(kLongDelaySeconds); + http_fault.mutable_max_active_faults()->set_value(kMaxFault); + // Config fault injection via different setup + SetFilterConfig(http_fault); + // Sends a batch of long running RPCs with long timeout to consume all + // active faults quota. + int num_delayed = 0; + RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMs); + std::vector rpcs = + SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options); + for (auto& rpc : rpcs) { + if (rpc.status.error_code() == StatusCode::OK) continue; + EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code()); + ++num_delayed; + } + // Only kMaxFault number of RPC should be fault injected.. + EXPECT_EQ(kMaxFault, num_delayed); +} + +TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) { + CreateAndStartBackends(1); + // kRpcTimeoutMilliseconds is 10s should never be reached. + const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 10000; + const uint32_t kFixedDelaySeconds = 1; + const uint32_t kDelayPercentagePerHundred = 100; + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Construct the fault injection filter config + HTTPFault http_fault; + auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); + delay_percentage->set_numerator(kDelayPercentagePerHundred); + delay_percentage->set_denominator(FractionalPercent::HUNDRED); + auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); + fixed_delay->set_seconds(kFixedDelaySeconds); + // Config fault injection via different setup + SetFilterConfig(http_fault); + ClientContext context; + context.set_deadline( + grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds)); + auto stream = stub_->BidiStream(&context); + stream->WritesDone(); + auto status = stream->Finish(); + EXPECT_TRUE(status.ok()) << status.error_message() << ", " + << status.error_details() << ", " + << context.debug_error_string(); +} + +// This case catches a bug in the retry code that was triggered by a bad +// interaction with the FI code. See https://github.com/grpc/grpc/pull/27217 +// for description. +TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayError) { + CreateAndStartBackends(1); + const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 500; + const uint32_t kFixedDelaySeconds = 100; + const uint32_t kDelayPercentagePerHundred = 100; + // Create an EDS resource + EdsResourceArgs args({{"locality0", CreateEndpointsForBackends()}}); + balancer_->ads_service()->SetEdsResource(BuildEdsResource(args)); + // Construct the fault injection filter config + HTTPFault http_fault; + auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage(); + delay_percentage->set_numerator(kDelayPercentagePerHundred); + delay_percentage->set_denominator(FractionalPercent::HUNDRED); + auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay(); + fixed_delay->set_seconds(kFixedDelaySeconds); + // Config fault injection via different setup + SetFilterConfig(http_fault); + ClientContext context; + context.set_deadline( + grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds)); + auto stream = stub_->BidiStream(&context); + stream->WritesDone(); + auto status = stream->Finish(); + EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, status.error_code()) + << status.error_message() << ", " << status.error_details() << ", " + << context.debug_error_string(); +} + +} // namespace +} // namespace testing +} // namespace grpc + +int main(int argc, char** argv) { + grpc::testing::TestEnvironment env(&argc, argv); + ::testing::InitGoogleTest(&argc, argv); + // Make the backup poller poll very frequently in order to pick up + // updates from all the subchannels's FDs. + GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1); +#if TARGET_OS_IPHONE + // Workaround Apple CFStream bug + gpr_setenv("grpc_cfstream", "0"); +#endif + grpc_init(); + const auto result = RUN_ALL_TESTS(); + grpc_shutdown(); + return result; +} diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 77ace05ce01..5e53409a6cd 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -7685,6 +7685,28 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "xds_fault_injection_end2end_test", + "platforms": [ + "linux", + "mac", + "posix" + ], + "uses_polling": true + }, { "args": [], "boringssl": true,