diff --git a/src/core/lib/security/authorization/grpc_authorization_policy_provider.cc b/src/core/lib/security/authorization/grpc_authorization_policy_provider.cc index 8ede541cce4..6712d5a0128 100644 --- a/src/core/lib/security/authorization/grpc_authorization_policy_provider.cc +++ b/src/core/lib/security/authorization/grpc_authorization_policy_provider.cc @@ -130,25 +130,43 @@ FileWatcherAuthorizationPolicyProvider::FileWatcherAuthorizationPolicyProvider( refresh_thread_->Start(); } +void FileWatcherAuthorizationPolicyProvider::SetCallbackForTesting( + std::function cb) { + MutexLock lock(&mu_); + cb_ = std::move(cb); +} + absl::Status FileWatcherAuthorizationPolicyProvider::ForceUpdate() { + bool contents_changed = false; + auto done_early = [&](absl::Status status) { + MutexLock lock(&mu_); + if (cb_ != nullptr) { + cb_(contents_changed, status); + } + return status; + }; absl::StatusOr file_contents = ReadPolicyFromFile(authz_policy_path_); if (!file_contents.ok()) { - return file_contents.status(); + return done_early(file_contents.status()); } if (file_contents_ == *file_contents) { - return absl::OkStatus(); + return done_early(absl::OkStatus()); } file_contents_ = std::move(*file_contents); + contents_changed = true; auto rbac_policies_or = GenerateRbacPolicies(file_contents_); if (!rbac_policies_or.ok()) { - return rbac_policies_or.status(); + return done_early(rbac_policies_or.status()); } MutexLock lock(&mu_); allow_engine_ = MakeRefCounted( std::move(rbac_policies_or->allow_policy)); deny_engine_ = MakeRefCounted( std::move(rbac_policies_or->deny_policy)); + if (cb_ != nullptr) { + cb_(contents_changed, absl::OkStatus()); + } if (GRPC_TRACE_FLAG_ENABLED(grpc_authz_trace)) { gpr_log(GPR_INFO, "authorization policy reload status: successfully loaded new " diff --git a/src/core/lib/security/authorization/grpc_authorization_policy_provider.h b/src/core/lib/security/authorization/grpc_authorization_policy_provider.h index 3c5fa0fa6e6..6dfe40d8db1 100644 --- a/src/core/lib/security/authorization/grpc_authorization_policy_provider.h +++ b/src/core/lib/security/authorization/grpc_authorization_policy_provider.h @@ -17,6 +17,7 @@ #include +#include #include #include @@ -83,6 +84,9 @@ class FileWatcherAuthorizationPolicyProvider unsigned int refresh_interval_sec, absl::Status* status); + void SetCallbackForTesting( + std::function cb); + void Orphan() override; AuthorizationEngines engines() override { @@ -102,6 +106,9 @@ class FileWatcherAuthorizationPolicyProvider gpr_event shutdown_event_; Mutex mu_; + // Callback is executed on every reload. This is useful for testing purpose. + std::function cb_ + ABSL_GUARDED_BY(mu_) = nullptr; // Engines created using authz_policy_. RefCountedPtr allow_engine_ ABSL_GUARDED_BY(mu_); RefCountedPtr deny_engine_ ABSL_GUARDED_BY(mu_); diff --git a/test/core/end2end/tests/grpc_authz.cc b/test/core/end2end/tests/grpc_authz.cc index ff08b8e2dee..d130e9e72c5 100644 --- a/test/core/end2end/tests/grpc_authz.cc +++ b/test/core/end2end/tests/grpc_authz.cc @@ -51,13 +51,6 @@ static gpr_timespec five_seconds_from_now(void) { return n_seconds_from_now(5); } -static void wait_for_policy_reload(void) { - // Wait for the provider's refresh thread to read the updated files. - // TODO(jtattermusch): Refactor the tests to use a more reliable mechanism of - // detecting that the policy has been reloaded. See b/204329811 - gpr_sleep_until(grpc_timeout_seconds_to_deadline(5)); -} - static void drain_cq(grpc_completion_queue* cq) { grpc_event ev; do { @@ -70,8 +63,7 @@ static void shutdown_server(grpc_end2end_test_fixture* f) { grpc_server_shutdown_and_notify(f->server, f->cq, tag(1000)); grpc_event ev; do { - ev = grpc_completion_queue_next(f->cq, grpc_timeout_seconds_to_deadline(5), - nullptr); + ev = grpc_completion_queue_next(f->cq, five_seconds_from_now(), nullptr); } while (ev.type != GRPC_OP_COMPLETE || ev.tag != tag(1000)); grpc_server_destroy(f->server); f->server = nullptr; @@ -557,6 +549,17 @@ static void test_file_watcher_valid_policy_reload( config, "test_file_watcher_valid_policy_reload", nullptr, &server_args); grpc_authorization_policy_provider_release(provider); test_allow_authorized_request(f); + gpr_event on_reload_done; + gpr_event_init(&on_reload_done); + std::function callback = + [&on_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + GPR_ASSERT(status.ok()); + gpr_event_set(&on_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast(provider) + ->SetCallbackForTesting(std::move(callback)); // Replace existing policy in file with a different authorization policy. authz_policy = "{" @@ -583,8 +586,12 @@ static void test_file_watcher_valid_policy_reload( " ]" "}"; tmp_policy.RewriteFile(authz_policy); - wait_for_policy_reload(); + GPR_ASSERT( + reinterpret_cast(1) == + gpr_event_wait(&on_reload_done, gpr_inf_future(GPR_CLOCK_MONOTONIC))); test_deny_unauthorized_request(f); + dynamic_cast(provider) + ->SetCallbackForTesting(nullptr); end_test(&f); config.tear_down_data(&f); @@ -626,17 +633,32 @@ static void test_file_watcher_invalid_policy_skip_reload( nullptr, &server_args); grpc_authorization_policy_provider_release(provider); test_allow_authorized_request(f); + gpr_event on_reload_done; + gpr_event_init(&on_reload_done); + std::function callback = + [&on_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + GPR_ASSERT(absl::StatusCode::kInvalidArgument == status.code()); + GPR_ASSERT("\"name\" field is not present." == status.message()); + gpr_event_set(&on_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast(provider) + ->SetCallbackForTesting(std::move(callback)); // Replace exisiting policy in file with an invalid policy. authz_policy = "{}"; tmp_policy.RewriteFile(authz_policy); - wait_for_policy_reload(); + GPR_ASSERT( + reinterpret_cast(1) == + gpr_event_wait(&on_reload_done, gpr_inf_future(GPR_CLOCK_MONOTONIC))); test_allow_authorized_request(f); + dynamic_cast(provider) + ->SetCallbackForTesting(nullptr); end_test(&f); config.tear_down_data(&f); } -#ifndef GPR_APPLE static void test_file_watcher_recovers_from_failure( grpc_end2end_test_config config) { const char* authz_policy = @@ -672,11 +694,36 @@ static void test_file_watcher_recovers_from_failure( config, "test_file_watcher_recovers_from_failure", nullptr, &server_args); grpc_authorization_policy_provider_release(provider); test_allow_authorized_request(f); + gpr_event on_first_reload_done; + gpr_event_init(&on_first_reload_done); + std::function callback1 = + [&on_first_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + GPR_ASSERT(absl::StatusCode::kInvalidArgument == status.code()); + GPR_ASSERT("\"name\" field is not present." == status.message()); + gpr_event_set(&on_first_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast(provider) + ->SetCallbackForTesting(std::move(callback1)); // Replace exisiting policy in file with an invalid policy. authz_policy = "{}"; tmp_policy.RewriteFile(authz_policy); - wait_for_policy_reload(); + GPR_ASSERT(reinterpret_cast(1) == + gpr_event_wait(&on_first_reload_done, + gpr_inf_future(GPR_CLOCK_MONOTONIC))); test_allow_authorized_request(f); + gpr_event on_second_reload_done; + gpr_event_init(&on_second_reload_done); + std::function callback2 = + [&on_second_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + GPR_ASSERT(status.ok()); + gpr_event_set(&on_second_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast(provider) + ->SetCallbackForTesting(std::move(callback2)); // Recover from reload errors, by replacing invalid policy in file with a // valid policy. authz_policy = @@ -704,13 +751,16 @@ static void test_file_watcher_recovers_from_failure( " ]" "}"; tmp_policy.RewriteFile(authz_policy); - wait_for_policy_reload(); + GPR_ASSERT(reinterpret_cast(1) == + gpr_event_wait(&on_second_reload_done, + gpr_inf_future(GPR_CLOCK_MONOTONIC))); test_deny_unauthorized_request(f); + dynamic_cast(provider) + ->SetCallbackForTesting(nullptr); end_test(&f); config.tear_down_data(&f); } -#endif void grpc_authz(grpc_end2end_test_config config) { test_static_init_allow_authorized_request(config); @@ -721,10 +771,7 @@ void grpc_authz(grpc_end2end_test_config config) { test_file_watcher_init_deny_request_no_match_in_policy(config); test_file_watcher_valid_policy_reload(config); test_file_watcher_invalid_policy_skip_reload(config); -#ifndef GPR_APPLE // test case highly flaky on Mac - // TODO(jtattermusch): reenable the test once b/204329811 is fixed. test_file_watcher_recovers_from_failure(config); -#endif } void grpc_authz_pre_init(void) {} diff --git a/test/core/security/grpc_authorization_policy_provider_test.cc b/test/core/security/grpc_authorization_policy_provider_test.cc index 0b672e48cd3..88d40c8026e 100644 --- a/test/core/security/grpc_authorization_policy_provider_test.cc +++ b/test/core/security/grpc_authorization_policy_provider_test.cc @@ -106,10 +106,23 @@ TEST(AuthorizationPolicyProviderTest, FileWatcherSuccessValidPolicyRefresh) { ASSERT_NE(deny_engine, nullptr); EXPECT_EQ(deny_engine->action(), Rbac::Action::kDeny); EXPECT_EQ(deny_engine->num_policies(), 1); + gpr_event on_reload_done; + gpr_event_init(&on_reload_done); + std::function callback = + [&on_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + EXPECT_TRUE(status.ok()); + gpr_event_set(&on_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast(provider->get()) + ->SetCallbackForTesting(std::move(callback)); // Rewrite the file with a different valid authorization policy. tmp_authz_policy->RewriteFile(testing::GetFileContents(VALID_POLICY_PATH_2)); - // Wait 2 seconds for the provider's refresh thread to read the updated files. - gpr_sleep_until(grpc_timeout_seconds_to_deadline(2)); + // Wait for the provider's refresh thread to read the updated files. + ASSERT_EQ( + gpr_event_wait(&on_reload_done, gpr_inf_future(GPR_CLOCK_MONOTONIC)), + reinterpret_cast(1)); engines = (*provider)->engines(); allow_engine = dynamic_cast(engines.allow_engine.get()); @@ -121,6 +134,8 @@ TEST(AuthorizationPolicyProviderTest, FileWatcherSuccessValidPolicyRefresh) { ASSERT_NE(deny_engine, nullptr); EXPECT_EQ(deny_engine->action(), Rbac::Action::kDeny); EXPECT_EQ(deny_engine->num_policies(), 0); + dynamic_cast(provider->get()) + ->SetCallbackForTesting(nullptr); } TEST(AuthorizationPolicyProviderTest, @@ -141,10 +156,24 @@ TEST(AuthorizationPolicyProviderTest, ASSERT_NE(deny_engine, nullptr); EXPECT_EQ(deny_engine->action(), Rbac::Action::kDeny); EXPECT_EQ(deny_engine->num_policies(), 1); + gpr_event on_reload_done; + gpr_event_init(&on_reload_done); + std::function callback = + [&on_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument); + EXPECT_EQ(status.message(), "\"name\" field is not present."); + gpr_event_set(&on_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast(provider->get()) + ->SetCallbackForTesting(std::move(callback)); // Skips the following policy update, and continues to use the valid policy. tmp_authz_policy->RewriteFile(testing::GetFileContents(INVALID_POLICY_PATH)); - // Wait 2 seconds for the provider's refresh thread to read the updated files. - gpr_sleep_until(grpc_timeout_seconds_to_deadline(2)); + // Wait for the provider's refresh thread to read the updated files. + ASSERT_EQ( + gpr_event_wait(&on_reload_done, gpr_inf_future(GPR_CLOCK_MONOTONIC)), + reinterpret_cast(1)); engines = (*provider)->engines(); allow_engine = dynamic_cast(engines.allow_engine.get()); @@ -156,6 +185,8 @@ TEST(AuthorizationPolicyProviderTest, ASSERT_NE(deny_engine, nullptr); EXPECT_EQ(deny_engine->action(), Rbac::Action::kDeny); EXPECT_EQ(deny_engine->num_policies(), 1); + dynamic_cast(provider->get()) + ->SetCallbackForTesting(nullptr); } TEST(AuthorizationPolicyProviderTest, FileWatcherRecoversFromFailure) { @@ -175,10 +206,24 @@ TEST(AuthorizationPolicyProviderTest, FileWatcherRecoversFromFailure) { ASSERT_NE(deny_engine, nullptr); EXPECT_EQ(deny_engine->action(), Rbac::Action::kDeny); EXPECT_EQ(deny_engine->num_policies(), 1); + gpr_event on_first_reload_done; + gpr_event_init(&on_first_reload_done); + std::function callback1 = + [&on_first_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument); + EXPECT_EQ(status.message(), "\"name\" field is not present."); + gpr_event_set(&on_first_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast(provider->get()) + ->SetCallbackForTesting(std::move(callback1)); // Skips the following policy update, and continues to use the valid policy. tmp_authz_policy->RewriteFile(testing::GetFileContents(INVALID_POLICY_PATH)); - // Wait 2 seconds for the provider's refresh thread to read the updated files. - gpr_sleep_until(grpc_timeout_seconds_to_deadline(2)); + // Wait for the provider's refresh thread to read the updated files. + ASSERT_EQ(gpr_event_wait(&on_first_reload_done, + gpr_inf_future(GPR_CLOCK_MONOTONIC)), + reinterpret_cast(1)); engines = (*provider)->engines(); allow_engine = dynamic_cast(engines.allow_engine.get()); @@ -190,10 +235,23 @@ TEST(AuthorizationPolicyProviderTest, FileWatcherRecoversFromFailure) { ASSERT_NE(deny_engine, nullptr); EXPECT_EQ(deny_engine->action(), Rbac::Action::kDeny); EXPECT_EQ(deny_engine->num_policies(), 1); + gpr_event on_second_reload_done; + gpr_event_init(&on_second_reload_done); + std::function callback2 = + [&on_second_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + EXPECT_TRUE(status.ok()); + gpr_event_set(&on_second_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast(provider->get()) + ->SetCallbackForTesting(std::move(callback2)); // Rewrite the file with a valid authorization policy. tmp_authz_policy->RewriteFile(testing::GetFileContents(VALID_POLICY_PATH_2)); - // Wait 2 seconds for the provider's refresh thread to read the updated files. - gpr_sleep_until(grpc_timeout_seconds_to_deadline(2)); + // Wait for the provider's refresh thread to read the updated files. + ASSERT_EQ(gpr_event_wait(&on_second_reload_done, + gpr_inf_future(GPR_CLOCK_MONOTONIC)), + reinterpret_cast(1)); engines = (*provider)->engines(); allow_engine = dynamic_cast(engines.allow_engine.get()); @@ -205,6 +263,8 @@ TEST(AuthorizationPolicyProviderTest, FileWatcherRecoversFromFailure) { ASSERT_NE(deny_engine, nullptr); EXPECT_EQ(deny_engine->action(), Rbac::Action::kDeny); EXPECT_EQ(deny_engine->num_policies(), 0); + dynamic_cast(provider->get()) + ->SetCallbackForTesting(nullptr); } } // namespace grpc_core diff --git a/test/cpp/end2end/grpc_authz_end2end_test.cc b/test/cpp/end2end/grpc_authz_end2end_test.cc index cbf933be8a2..01d64bbe021 100644 --- a/test/cpp/end2end/grpc_authz_end2end_test.cc +++ b/test/cpp/end2end/grpc_authz_end2end_test.cc @@ -23,6 +23,7 @@ #include #include "src/core/lib/iomgr/load_file.h" +#include "src/core/lib/security/authorization/grpc_authorization_policy_provider.h" #include "src/core/lib/security/credentials/fake/fake_credentials.h" #include "src/cpp/client/secure_credentials.h" #include "src/cpp/server/secure_server_credentials.h" @@ -658,13 +659,26 @@ TEST_F(GrpcAuthzEnd2EndTest, FileWatcherValidPolicyRefresh) { " ]" "}"; grpc_core::testing::TmpFile tmp_policy(policy); - InitServer(CreateFileWatcherAuthzPolicyProvider(tmp_policy.name(), 1)); + auto provider = CreateFileWatcherAuthzPolicyProvider(tmp_policy.name(), 1); + InitServer(provider); auto channel = BuildChannel(); ClientContext context1; grpc::testing::EchoResponse resp1; grpc::Status status = SendRpc(channel, &context1, &resp1); EXPECT_TRUE(status.ok()); EXPECT_EQ(resp1.message(), kMessage); + gpr_event on_reload_done; + gpr_event_init(&on_reload_done); + std::function callback = + [&on_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + EXPECT_TRUE(status.ok()); + gpr_event_set(&on_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast( + provider->c_provider()) + ->SetCallbackForTesting(std::move(callback)); // Replace the existing policy with a new authorization policy. policy = "{" @@ -691,14 +705,19 @@ TEST_F(GrpcAuthzEnd2EndTest, FileWatcherValidPolicyRefresh) { " ]" "}"; tmp_policy.RewriteFile(policy); - // Wait 2 seconds for the provider's refresh thread to read the updated files. - gpr_sleep_until(grpc_timeout_seconds_to_deadline(2)); + // Wait for the provider's refresh thread to read the updated files. + ASSERT_EQ( + gpr_event_wait(&on_reload_done, gpr_inf_future(GPR_CLOCK_MONOTONIC)), + reinterpret_cast(1)); ClientContext context2; grpc::testing::EchoResponse resp2; status = SendRpc(channel, &context2, &resp2); EXPECT_EQ(status.error_code(), grpc::StatusCode::PERMISSION_DENIED); EXPECT_EQ(status.error_message(), "Unauthorized RPC request rejected."); EXPECT_TRUE(resp2.message().empty()); + dynamic_cast( + provider->c_provider()) + ->SetCallbackForTesting(nullptr); } TEST_F(GrpcAuthzEnd2EndTest, FileWatcherInvalidPolicyRefreshSkipsReload) { @@ -717,23 +736,42 @@ TEST_F(GrpcAuthzEnd2EndTest, FileWatcherInvalidPolicyRefreshSkipsReload) { " ]" "}"; grpc_core::testing::TmpFile tmp_policy(policy); - InitServer(CreateFileWatcherAuthzPolicyProvider(tmp_policy.name(), 1)); + auto provider = CreateFileWatcherAuthzPolicyProvider(tmp_policy.name(), 1); + InitServer(provider); auto channel = BuildChannel(); ClientContext context1; grpc::testing::EchoResponse resp1; grpc::Status status = SendRpc(channel, &context1, &resp1); EXPECT_TRUE(status.ok()); EXPECT_EQ(resp1.message(), kMessage); + gpr_event on_reload_done; + gpr_event_init(&on_reload_done); + std::function callback = + [&on_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument); + EXPECT_EQ(status.message(), "\"name\" field is not present."); + gpr_event_set(&on_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast( + provider->c_provider()) + ->SetCallbackForTesting(std::move(callback)); // Replaces existing policy with an invalid authorization policy. policy = "{}"; tmp_policy.RewriteFile(policy); - // Wait 2 seconds for the provider's refresh thread to read the updated files. - gpr_sleep_until(grpc_timeout_seconds_to_deadline(2)); + // Wait for the provider's refresh thread to read the updated files. + ASSERT_EQ( + gpr_event_wait(&on_reload_done, gpr_inf_future(GPR_CLOCK_MONOTONIC)), + reinterpret_cast(1)); ClientContext context2; grpc::testing::EchoResponse resp2; status = SendRpc(channel, &context2, &resp2); EXPECT_TRUE(status.ok()); EXPECT_EQ(resp2.message(), kMessage); + dynamic_cast( + provider->c_provider()) + ->SetCallbackForTesting(nullptr); } TEST_F(GrpcAuthzEnd2EndTest, FileWatcherRecoversFromFailure) { @@ -752,23 +790,51 @@ TEST_F(GrpcAuthzEnd2EndTest, FileWatcherRecoversFromFailure) { " ]" "}"; grpc_core::testing::TmpFile tmp_policy(policy); - InitServer(CreateFileWatcherAuthzPolicyProvider(tmp_policy.name(), 1)); + auto provider = CreateFileWatcherAuthzPolicyProvider(tmp_policy.name(), 1); + InitServer(provider); auto channel = BuildChannel(); ClientContext context1; grpc::testing::EchoResponse resp1; grpc::Status status = SendRpc(channel, &context1, &resp1); EXPECT_TRUE(status.ok()); EXPECT_EQ(resp1.message(), kMessage); + gpr_event on_first_reload_done; + gpr_event_init(&on_first_reload_done); + std::function callback1 = + [&on_first_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument); + EXPECT_EQ(status.message(), "\"name\" field is not present."); + gpr_event_set(&on_first_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast( + provider->c_provider()) + ->SetCallbackForTesting(std::move(callback1)); // Replaces existing policy with an invalid authorization policy. policy = "{}"; tmp_policy.RewriteFile(policy); - // Wait 2 seconds for the provider's refresh thread to read the updated files. - gpr_sleep_until(grpc_timeout_seconds_to_deadline(2)); + // Wait for the provider's refresh thread to read the updated files. + ASSERT_EQ(gpr_event_wait(&on_first_reload_done, + gpr_inf_future(GPR_CLOCK_MONOTONIC)), + reinterpret_cast(1)); ClientContext context2; grpc::testing::EchoResponse resp2; status = SendRpc(channel, &context2, &resp2); EXPECT_TRUE(status.ok()); EXPECT_EQ(resp2.message(), kMessage); + gpr_event on_second_reload_done; + gpr_event_init(&on_second_reload_done); + std::function callback2 = + [&on_second_reload_done](bool contents_changed, absl::Status status) { + if (contents_changed) { + EXPECT_TRUE(status.ok()); + gpr_event_set(&on_second_reload_done, reinterpret_cast(1)); + } + }; + dynamic_cast( + provider->c_provider()) + ->SetCallbackForTesting(std::move(callback2)); // Replace the existing invalid policy with a valid authorization policy. policy = "{" @@ -795,14 +861,19 @@ TEST_F(GrpcAuthzEnd2EndTest, FileWatcherRecoversFromFailure) { " ]" "}"; tmp_policy.RewriteFile(policy); - // Wait 2 seconds for the provider's refresh thread to read the updated files. - gpr_sleep_until(grpc_timeout_seconds_to_deadline(2)); + // Wait for the provider's refresh thread to read the updated files. + ASSERT_EQ(gpr_event_wait(&on_second_reload_done, + gpr_inf_future(GPR_CLOCK_MONOTONIC)), + reinterpret_cast(1)); ClientContext context3; grpc::testing::EchoResponse resp3; status = SendRpc(channel, &context3, &resp3); EXPECT_EQ(status.error_code(), grpc::StatusCode::PERMISSION_DENIED); EXPECT_EQ(status.error_message(), "Unauthorized RPC request rejected."); EXPECT_TRUE(resp3.message().empty()); + dynamic_cast( + provider->c_provider()) + ->SetCallbackForTesting(nullptr); } } // namespace