From e34b0a73c6d32c079bd4518092c59e22ef4b9a58 Mon Sep 17 00:00:00 2001
From: Lidi Zheng <lidiz@google.com>
Date: Thu, 11 Mar 2021 14:47:39 -0800
Subject: [PATCH] Fix a typo in fault injection (#25693)

---
 .../fault_injection/fault_injection_filter.cc |  2 +-
 test/cpp/end2end/xds_end2end_test.cc          | 52 +++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/src/core/ext/filters/fault_injection/fault_injection_filter.cc b/src/core/ext/filters/fault_injection/fault_injection_filter.cc
index 3748f1057cb..6cf21c56718 100644
--- a/src/core/ext/filters/fault_injection/fault_injection_filter.cc
+++ b/src/core/ext/filters/fault_injection/fault_injection_filter.cc
@@ -384,7 +384,7 @@ void CallData::DecideWhetherToInjectFaults(
   // Roll the dice
   delay_request_ = fi_policy_->delay != 0 &&
                    UnderFraction(fi_policy_->delay_percentage_numerator,
-                                 fi_policy_->abort_percentage_denominator);
+                                 fi_policy_->delay_percentage_denominator);
   abort_request_ = fi_policy_->abort_code != GRPC_STATUS_OK &&
                    UnderFraction(fi_policy_->abort_percentage_numerator,
                                  fi_policy_->abort_percentage_denominator);
diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc
index d099f4c6240..0f86c544122 100644
--- a/test/cpp/end2end/xds_end2end_test.cc
+++ b/test/cpp/end2end/xds_end2end_test.cc
@@ -9796,6 +9796,58 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) {
   http_fault.mutable_abort()->set_grpc_status(
       static_cast<uint32_t>(StatusCode::ABORTED));
   auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
+  delay_percentage->set_numerator(1000000);  // Always inject DELAY!
+  delay_percentage->set_denominator(FractionalPercent::MILLION);
+  auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
+  fixed_delay->set_nanos(kFixedDelayNanos);
+  // Config fault injection via different setup
+  SetFilterConfig(http_fault);
+  // Send kNumRpcs RPCs and count the aborts.
+  int num_total = 0, num_ok = 0, num_failure = 0, num_aborted = 0;
+  for (size_t i = 0; i < kNumRpcs; ++i) {
+    grpc_millis t0 = NowFromCycleCounter();
+    SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_aborted,
+                    RpcOptions(), "Fault injected");
+    grpc_millis t1 = NowFromCycleCounter();
+    EXPECT_GE(t1, t0 + kFixedDelayNanos / 1000 / 1000);
+  }
+  EXPECT_EQ(kNumRpcs, num_total);
+  EXPECT_EQ(0, num_failure);
+  // The abort rate should be roughly equal to the expectation.
+  const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
+  EXPECT_THAT(seen_abort_rate,
+              ::testing::AllOf(::testing::Ge(kAbortRate - kErrorTolerance),
+                               ::testing::Le(kAbortRate + kErrorTolerance)));
+  gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION");
+}
+
+// This test and the above test apply different denominators to delay and abort.
+// This ensures that we are using the right denominator for each injected fault
+// in our code.
+TEST_P(FaultInjectionTest,
+       XdsFaultInjectionAlwaysDelayPercentageAbortSwitchDenominator) {
+  gpr_setenv("GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION", "true");
+  const size_t kNumRpcs = 100;
+  const uint32_t kAbortPercentagePerMillion = 500000;
+  const double kAbortRate = kAbortPercentagePerMillion / 1000000.0;
+  const uint32_t kFixedDelayNanos = 10 * 1000 * 1000;  // 10 ms
+  const double kErrorTolerance = 0.2;
+  SetNextResolution({});
+  SetNextResolutionForLbChannelAllBalancers();
+  // Create an EDS resource
+  AdsServiceImpl::EdsResourceArgs args({
+      {"locality0", GetBackendPorts()},
+  });
+  balancers_[0]->ads_service()->SetEdsResource(
+      BuildEdsResource(args, DefaultEdsServiceName()));
+  // Construct the fault injection filter config
+  HTTPFault http_fault;
+  auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
+  abort_percentage->set_numerator(kAbortPercentagePerMillion);
+  abort_percentage->set_denominator(FractionalPercent::MILLION);
+  http_fault.mutable_abort()->set_grpc_status(
+      static_cast<uint32_t>(StatusCode::ABORTED));
+  auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
   delay_percentage->set_numerator(100);  // Always inject DELAY!
   delay_percentage->set_denominator(FractionalPercent::HUNDRED);
   auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();