[xds] tests to wait for asynchronous metrics reports (#37005)

Reports to the metrics reporter may happen asynchronously with work serializer dispatch.

<!--

If you know who should review your pull request, please assign it to that
person, otherwise the pull request would get assigned randomly.

If your pull request is for a specific language, please add the appropriate
lang label.

-->

Closes #37005

COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/37005 from yousukseung:xds-client-tests-fix d65d00a410
PiperOrigin-RevId: 647066953

test/core/xds/xds_client_test.cc

@@ -637,6 +637,37 @@ class XdsClientTest : public ::testing::Test {
     }
     const ServerFailureMap& server_failures() const { return server_failures_; }
 
+    // Returns true if matchers return true before the timeout.
+    // Runs matchers once as soon as it is called and then again
+    // every time the metrics reporter sees an update.
+    bool WaitForMetricsReporterData(
+        ::testing::Matcher<ResourceUpdateMap> resource_updates_valid_matcher,
+        ::testing::Matcher<ResourceUpdateMap> resource_updates_invalid_matcher,
+        ::testing::Matcher<ServerFailureMap> server_failures_matcher,
+        absl::Duration timeout = absl::Seconds(3),
+        SourceLocation location = SourceLocation()) {
+      const absl::Time deadline =
+          absl::Now() + (timeout * grpc_test_slowdown_factor());
+      MutexLock lock(&mu_);
+      while (true) {
+        if (::testing::Matches(resource_updates_valid_matcher)(
+                resource_updates_valid_) &&
+            ::testing::Matches(resource_updates_invalid_matcher)(
+                resource_updates_invalid_) &&
+            ::testing::Matches(server_failures_matcher)(server_failures_)) {
+          return true;
+        }
+        if (cond_.WaitWithDeadline(&mu_, deadline)) break;
+      }
+      EXPECT_THAT(resource_updates_valid_, resource_updates_valid_matcher)
+          << location.file() << ":" << location.line();
+      EXPECT_THAT(resource_updates_invalid_, resource_updates_invalid_matcher)
+          << location.file() << ":" << location.line();
+      EXPECT_THAT(server_failures_, server_failures_matcher)
+          << location.file() << ":" << location.line();
+      return false;
+    }
+
    private:
     void ReportResourceUpdates(absl::string_view xds_server,
                                absl::string_view resource_type,
@@ -651,17 +682,20 @@ class XdsClientTest : public ::testing::Test {
       if (num_resources_invalid > 0) {
         resource_updates_invalid_[key] += num_resources_invalid;
       }
+      cond_.SignalAll();
     }
 
     void ReportServerFailure(absl::string_view xds_server) override {
       MutexLock lock(&mu_);
       ++server_failures_[std::string(xds_server)];
+      cond_.SignalAll();
     }
 
     mutable Mutex mu_;
     ResourceUpdateMap resource_updates_valid_ ABSL_GUARDED_BY(mu_);
     ResourceUpdateMap resource_updates_invalid_ ABSL_GUARDED_BY(mu_);
     ServerFailureMap server_failures_ ABSL_GUARDED_BY(mu_);
+    CondVar cond_;
   };
 
   using ResourceCounts =
@@ -930,13 +964,12 @@ TEST_F(XdsClientTest, BasicWatch) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -956,13 +989,12 @@ TEST_F(XdsClientTest, BasicWatch) {
   CancelFooWatch(watcher.get(), "foo1");
   EXPECT_TRUE(stream->Orphaned());
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(GetResourceCounts(), ::testing::ElementsAre());
   EXPECT_THAT(GetServerConnections(), ::testing::ElementsAre());
 }
@@ -997,13 +1029,12 @@ TEST_F(XdsClientTest, UpdateFromServer) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1030,13 +1061,12 @@ TEST_F(XdsClientTest, UpdateFromServer) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 9);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  2)));
+          2)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1085,13 +1115,12 @@ TEST_F(XdsClientTest, MultipleWatchersForSameResource) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1132,13 +1161,12 @@ TEST_F(XdsClientTest, MultipleWatchersForSameResource) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 9);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  2)));
+          2)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1202,13 +1230,12 @@ TEST_F(XdsClientTest, SubscribeToMultipleResources) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1257,13 +1284,12 @@ TEST_F(XdsClientTest, SubscribeToMultipleResources) {
   EXPECT_EQ(resource->name, "foo2");
   EXPECT_EQ(resource->value, 7);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  2)));
+          2)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1355,13 +1381,12 @@ TEST_F(XdsClientTest, UpdateContainsOnlyChangedResource) {
   EXPECT_EQ(resource->name, "foo2");
   EXPECT_EQ(resource->value, 7);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  2)));
+          2)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1388,13 +1413,12 @@ TEST_F(XdsClientTest, UpdateContainsOnlyChangedResource) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 9);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  3)));
+          3)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1466,13 +1490,13 @@ TEST_F(XdsClientTest, ResourceValidationFailure) {
             "[field:value error:is not a number] (node ID:xds_client_test)")
       << *error;
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre());
+      ::testing::ElementsAre(),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
+      ::testing::_));
   EXPECT_THAT(GetResourceCounts(),
               ::testing::ElementsAre(::testing::Pair(
                   ResourceCountLabelsEq(XdsClient::kOldStyleAuthority,
@@ -1519,16 +1543,16 @@ TEST_F(XdsClientTest, ResourceValidationFailure) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 9);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
+      ::testing::_));
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1579,10 +1603,8 @@ TEST_F(XdsClientTest, ResourceValidationFailureMultipleResources) {
   // Before the server responds, add a watch for another resource.
   auto watcher2 = StartFooWatch("foo2");
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre());
+      ::testing::ElementsAre(), ::testing::ElementsAre(), ::testing::_));
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
-              ::testing::ElementsAre());
   EXPECT_THAT(GetResourceCounts(),
               ::testing::ElementsAre(::testing::Pair(
                   ResourceCountLabelsEq(XdsClient::kOldStyleAuthority,
@@ -1599,10 +1621,8 @@ TEST_F(XdsClientTest, ResourceValidationFailureMultipleResources) {
   // Add a watch for a third resource.
   auto watcher3 = StartFooWatch("foo3");
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre());
+      ::testing::ElementsAre(), ::testing::ElementsAre(), ::testing::_));
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
-              ::testing::ElementsAre());
   EXPECT_THAT(GetResourceCounts(),
               ::testing::ElementsAre(::testing::Pair(
                   ResourceCountLabelsEq(XdsClient::kOldStyleAuthority,
@@ -1619,10 +1639,8 @@ TEST_F(XdsClientTest, ResourceValidationFailureMultipleResources) {
   // Add a watch for a fourth resource.
   auto watcher4 = StartFooWatch("foo4");
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre());
+      ::testing::ElementsAre(), ::testing::ElementsAre(), ::testing::_));
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
-              ::testing::ElementsAre());
   EXPECT_THAT(GetResourceCounts(),
               ::testing::ElementsAre(::testing::Pair(
                   ResourceCountLabelsEq(XdsClient::kOldStyleAuthority,
@@ -1688,16 +1706,16 @@ TEST_F(XdsClientTest, ResourceValidationFailureMultipleResources) {
   EXPECT_EQ(resource->name, "foo4");
   EXPECT_EQ(resource->value, 5);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  5)));
+          5)),
+      ::testing::_));
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(
@@ -1783,13 +1801,12 @@ TEST_F(XdsClientTest, ResourceValidationFailureForCachedResource) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1820,16 +1837,16 @@ TEST_F(XdsClientTest, ResourceValidationFailureForCachedResource) {
             "[field:value error:is not a number] (node ID:xds_client_test)")
       << *error;
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
+      ::testing::_));
   EXPECT_THAT(GetResourceCounts(),
               ::testing::ElementsAre(::testing::Pair(
                   ResourceCountLabelsEq(XdsClient::kOldStyleAuthority,
@@ -1901,18 +1918,16 @@ TEST_F(XdsClientTest, WildcardCapableResponseWithEmptyResource) {
   EXPECT_EQ(resource->name, "wc1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-      metrics_reporter_->resource_updates_valid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsWildcardCapableResourceType::Get()->type_url()),
-          1)));
+          1)),
-  EXPECT_THAT(
-      metrics_reporter_->resource_updates_invalid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsWildcardCapableResourceType::Get()->type_url()),
-          1)));
+          1)),
+      ::testing::_));
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -1971,14 +1986,12 @@ TEST_F(XdsClientTest, ResourceDeletion) {
   EXPECT_EQ(resource->name, "wc1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-      metrics_reporter_->resource_updates_valid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsWildcardCapableResourceType::Get()->type_url()),
-          1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -2003,14 +2016,12 @@ TEST_F(XdsClientTest, ResourceDeletion) {
   // Watcher should see the does-not-exist event.
   EXPECT_TRUE(watcher->WaitForDoesNotExist(absl::Seconds(1)));
   // Check metric data.
-  EXPECT_THAT(
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-      metrics_reporter_->resource_updates_valid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsWildcardCapableResourceType::Get()->type_url()),
-          1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(GetResourceCounts(),
               ::testing::ElementsAre(::testing::Pair(
                   ResourceCountLabelsEq(
@@ -2046,14 +2057,12 @@ TEST_F(XdsClientTest, ResourceDeletion) {
   EXPECT_EQ(resource->name, "wc1");
   EXPECT_EQ(resource->value, 7);
   // Check metric data.
-  EXPECT_THAT(
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-      metrics_reporter_->resource_updates_valid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsWildcardCapableResourceType::Get()->type_url()),
-          2)));
+          2)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -2107,14 +2116,12 @@ TEST_F(XdsClientTest, ResourceDeletionIgnoredWhenConfigured) {
   EXPECT_EQ(resource->name, "wc1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-      metrics_reporter_->resource_updates_valid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsWildcardCapableResourceType::Get()->type_url()),
-          1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -2140,14 +2147,12 @@ TEST_F(XdsClientTest, ResourceDeletionIgnoredWhenConfigured) {
   // deletion.
   EXPECT_TRUE(watcher->ExpectNoEvent(absl::Seconds(1)));
   // Check metric data.
-  EXPECT_THAT(
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-      metrics_reporter_->resource_updates_valid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsWildcardCapableResourceType::Get()->type_url()),
-          1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -2186,14 +2191,12 @@ TEST_F(XdsClientTest, ResourceDeletionIgnoredWhenConfigured) {
   EXPECT_EQ(resource->name, "wc1");
   EXPECT_EQ(resource->value, 7);
   // Check metric data.
-  EXPECT_THAT(
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-      metrics_reporter_->resource_updates_valid(),
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsWildcardCapableResourceType::Get()->type_url()),
-          2)));
+          2)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -2335,8 +2338,9 @@ TEST_F(XdsClientTest, StreamClosedByServerWithoutSeeingResponse) {
   // Check metric data.
   EXPECT_THAT(GetServerConnections(), ::testing::ElementsAre(::testing::Pair(
                                           kDefaultXdsServerUrl, false)));
-  EXPECT_THAT(metrics_reporter_->server_failures(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1)));
+      ::testing::_, ::testing::_,
+      ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1))));
   // XdsClient should report an error to the watcher.
   auto error = watcher->WaitForNextError();
   ASSERT_TRUE(error.has_value());
@@ -2360,8 +2364,9 @@ TEST_F(XdsClientTest, StreamClosedByServerWithoutSeeingResponse) {
   // Connection still reported as unhappy until we get a response.
   EXPECT_THAT(GetServerConnections(), ::testing::ElementsAre(::testing::Pair(
                                           kDefaultXdsServerUrl, false)));
-  EXPECT_THAT(metrics_reporter_->server_failures(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1)));
+      ::testing::_, ::testing::_,
+      ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1))));
   // Server now sends the requested resource.
   stream->SendMessageToClient(
       ResponseBuilder(XdsFooResourceType::Get()->type_url())
@@ -2431,8 +2436,9 @@ TEST_F(XdsClientTest, ConnectionFails) {
   // Connection reported as unhappy.
   EXPECT_THAT(GetServerConnections(), ::testing::ElementsAre(::testing::Pair(
                                           kDefaultXdsServerUrl, false)));
-  EXPECT_THAT(metrics_reporter_->server_failures(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1)));
+      ::testing::_, ::testing::_,
+      ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1))));
   // We should not see a resource-does-not-exist event, because the
   // timer should not be running while the channel is disconnected.
   EXPECT_TRUE(watcher->ExpectNoEvent(absl::Seconds(4)));
@@ -2516,10 +2522,8 @@ TEST_F(XdsClientTest, ResourceDoesNotExistUponTimeout) {
   // not existing.
   EXPECT_TRUE(watcher->WaitForDoesNotExist(absl::Seconds(5)));
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre());
+      ::testing::ElementsAre(), ::testing::ElementsAre(), ::testing::_));
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
-              ::testing::ElementsAre());
   EXPECT_THAT(GetResourceCounts(),
               ::testing::ElementsAre(::testing::Pair(
                   ResourceCountLabelsEq(XdsClient::kOldStyleAuthority,
@@ -2547,13 +2551,12 @@ TEST_F(XdsClientTest, ResourceDoesNotExistUponTimeout) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -2620,10 +2623,8 @@ TEST_F(XdsClientTest, ResourceDoesNotExistAfterStreamRestart) {
             "(node ID:xds_client_test)")
       << *error;
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre());
+      ::testing::ElementsAre(), ::testing::ElementsAre(), ::testing::_));
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
-              ::testing::ElementsAre());
   EXPECT_THAT(GetResourceCounts(),
               ::testing::ElementsAre(::testing::Pair(
                   ResourceCountLabelsEq(XdsClient::kOldStyleAuthority,
@@ -2645,10 +2646,8 @@ TEST_F(XdsClientTest, ResourceDoesNotExistAfterStreamRestart) {
   // Client should receive a resource does-not-exist.
   ASSERT_TRUE(watcher->WaitForDoesNotExist(absl::Seconds(4)));
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre());
+      ::testing::ElementsAre(), ::testing::ElementsAre(), ::testing::_));
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
-              ::testing::ElementsAre());
   EXPECT_THAT(GetResourceCounts(),
               ::testing::ElementsAre(::testing::Pair(
                   ResourceCountLabelsEq(XdsClient::kOldStyleAuthority,
@@ -2668,13 +2667,12 @@ TEST_F(XdsClientTest, ResourceDoesNotExistAfterStreamRestart) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -2812,13 +2810,12 @@ TEST_F(XdsClientTest,
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -2895,13 +2892,12 @@ TEST_F(XdsClientTest,
   EXPECT_EQ(resource->name, "foo2");
   EXPECT_EQ(resource->value, 7);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  3)));
+          3)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -2961,13 +2957,12 @@ TEST_F(XdsClientTest, DoNotSendDoesNotExistForCachedResource) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -3002,13 +2997,12 @@ TEST_F(XdsClientTest, DoNotSendDoesNotExistForCachedResource) {
   // resending it.
   EXPECT_TRUE(watcher->ExpectNoEvent(absl::Seconds(4)));
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -3025,13 +3019,12 @@ TEST_F(XdsClientTest, DoNotSendDoesNotExistForCachedResource) {
   // Watcher will not see any update, since the resource is unchanged.
   EXPECT_TRUE(watcher->ExpectNoEvent(absl::Seconds(1)));
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  2)));
+          2)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -3083,13 +3076,12 @@ TEST_F(XdsClientTest, ResourceWrappedInResourceMessage) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -3138,13 +3130,12 @@ TEST_F(XdsClientTest, MultipleResourceTypes) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -3182,7 +3173,7 @@ TEST_F(XdsClientTest, MultipleResourceTypes) {
   EXPECT_EQ(resource2->name, "bar1");
   EXPECT_EQ(resource2->value, "whee");
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(
           ::testing::Pair(
               ::testing::Pair(kDefaultXdsServerUrl,
@@ -3191,9 +3182,8 @@ TEST_F(XdsClientTest, MultipleResourceTypes) {
           ::testing::Pair(
               ::testing::Pair(kDefaultXdsServerUrl,
                               XdsFooResourceType::Get()->type_url()),
-                      1)));
+              1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::UnorderedElementsAre(
@@ -3269,13 +3259,12 @@ TEST_F(XdsClientTest, Federation) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -3337,7 +3326,7 @@ TEST_F(XdsClientTest, Federation) {
   EXPECT_EQ(resource->name, kXdstpResourceName);
   EXPECT_EQ(resource->value, 3);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(
           ::testing::Pair(
               ::testing::Pair(kDefaultXdsServerUrl,
@@ -3346,9 +3335,8 @@ TEST_F(XdsClientTest, Federation) {
           ::testing::Pair(
               ::testing::Pair(authority_server.server_uri(),
                               XdsFooResourceType::Get()->type_url()),
-                      1)));
+              1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(
@@ -3415,13 +3403,12 @@ TEST_F(XdsClientTest, FederationAuthorityDefaultsToTopLevelXdsServer) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -3463,13 +3450,12 @@ TEST_F(XdsClientTest, FederationAuthorityDefaultsToTopLevelXdsServer) {
   EXPECT_EQ(resource->name, kXdstpResourceName);
   EXPECT_EQ(resource->value, 3);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  2)));
+          2)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(
@@ -3621,13 +3607,12 @@ TEST_F(XdsClientTest, FederationChannelFailureReportedToWatchers) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -3678,7 +3663,7 @@ TEST_F(XdsClientTest, FederationChannelFailureReportedToWatchers) {
   EXPECT_EQ(resource->name, kXdstpResourceName);
   EXPECT_EQ(resource->value, 3);
   // Check metric data.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(
           ::testing::Pair(
               ::testing::Pair(kDefaultXdsServerUrl,
@@ -3687,9 +3672,8 @@ TEST_F(XdsClientTest, FederationChannelFailureReportedToWatchers) {
           ::testing::Pair(
               ::testing::Pair(authority_server.server_uri(),
                               XdsFooResourceType::Get()->type_url()),
-                      1)));
+              1)),
-  EXPECT_THAT(metrics_reporter_->resource_updates_invalid(),
+      ::testing::ElementsAre(), ::testing::_));
-              ::testing::ElementsAre());
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(
@@ -3730,9 +3714,10 @@ TEST_F(XdsClientTest, FederationChannelFailureReportedToWatchers) {
               ::testing::ElementsAre(
                   ::testing::Pair(kDefaultXdsServerUrl, true),
                   ::testing::Pair(authority_server.server_uri(), false)));
-  EXPECT_THAT(metrics_reporter_->server_failures(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
+      ::testing::_, ::testing::_,
       ::testing::ElementsAre(
-                  ::testing::Pair(authority_server.server_uri(), 1)));
+          ::testing::Pair(authority_server.server_uri(), 1))));
   // Cancel watch for "foo1".
   CancelFooWatch(watcher.get(), "foo1");
   EXPECT_TRUE(stream->Orphaned());
@@ -3843,9 +3828,8 @@ TEST_F(XdsClientTest, FallbackAndRecover) {
                                         XdsFooResourceType::Get()->type_url(),
                                         "requested"),
                   1)));
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::IsEmpty());
+      ::testing::IsEmpty(), ::testing::_, ::testing::ElementsAre()));
-  EXPECT_THAT(metrics_reporter_->server_failures(), ::testing::ElementsAre());
   // XdsClient should have created an ADS stream.
   auto stream = WaitForAdsStream();
   ASSERT_TRUE(stream != nullptr);
@@ -3869,11 +3853,12 @@ TEST_F(XdsClientTest, FallbackAndRecover) {
   EXPECT_EQ(resource->name, "foo1");
   EXPECT_EQ(resource->value, 6);
   // Result (local): Metrics show 1 resource update and 1 cached resource.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(::testing::Pair(
           ::testing::Pair(kDefaultXdsServerUrl,
                           XdsFooResourceType::Get()->type_url()),
-                  1)));
+          1)),
+      ::testing::_, ::testing::_));
   EXPECT_THAT(
       GetResourceCounts(),
       ::testing::ElementsAre(::testing::Pair(
@@ -3900,15 +3885,17 @@ TEST_F(XdsClientTest, FallbackAndRecover) {
   // Result (local): The metrics show the channel as being unhealthy.
   EXPECT_THAT(GetServerConnections(), ::testing::ElementsAre(::testing::Pair(
                                           kDefaultXdsServerUrl, false)));
-  EXPECT_THAT(metrics_reporter_->server_failures(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1)));
+      ::testing::_, ::testing::_,
+      ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1))));
   // Input: Trigger stream failure.
   stream->MaybeSendStatusToClient(absl::UnavailableError("Stream failure"));
   // Result (local): The metrics still show the channel as being unhealthy.
   EXPECT_THAT(GetServerConnections(), ::testing::ElementsAre(::testing::Pair(
                                           kDefaultXdsServerUrl, false)));
-  EXPECT_THAT(metrics_reporter_->server_failures(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1)));
+      ::testing::_, ::testing::_,
+      ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1))));
   // Result (remote): The client starts a new stream and sends a subscription
   //   message. Note that the server does not respond, so the channel will still
   //   have non-OK status.
@@ -3978,7 +3965,7 @@ TEST_F(XdsClientTest, FallbackAndRecover) {
   EXPECT_EQ(resource->name, "foo2");
   EXPECT_EQ(resource->value, 30);
   // Result (local): Metrics show an update from fallback server.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(
           ::testing::Pair(
               ::testing::Pair(kDefaultXdsServerUrl,
@@ -3987,7 +3974,8 @@ TEST_F(XdsClientTest, FallbackAndRecover) {
           ::testing::Pair(
               ::testing::Pair(fallback_server.server_uri(),
                               XdsFooResourceType::Get()->type_url()),
-                      2)));
+              2)),
+      ::testing::_, ::testing::_));
   EXPECT_THAT(GetServerConnections(),
               ::testing::ElementsAre(
                   ::testing::Pair(kDefaultXdsServerUrl, false),
@@ -4015,7 +4003,7 @@ TEST_F(XdsClientTest, FallbackAndRecover) {
           .Serialize());
   // Result (local): Metrics show that we've closed the channel to the fallback
   //   server and received resource updates from the primary server.
-  EXPECT_THAT(metrics_reporter_->resource_updates_valid(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
       ::testing::ElementsAre(
           ::testing::Pair(
               ::testing::Pair(kDefaultXdsServerUrl,
@@ -4024,11 +4012,11 @@ TEST_F(XdsClientTest, FallbackAndRecover) {
           ::testing::Pair(
               ::testing::Pair(fallback_server.server_uri(),
                               XdsFooResourceType::Get()->type_url()),
-                      2)));
+              2)),
+      ::testing::_,
+      ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1))));
   EXPECT_THAT(GetServerConnections(), ::testing::ElementsAre(::testing::Pair(
                                           kDefaultXdsServerUrl, true)));
-  EXPECT_THAT(metrics_reporter_->server_failures(),
-              ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1)));
   // Result (remote): The stream to the fallback server has been orphaned.
   EXPECT_TRUE(stream2->Orphaned());
   // Result (local): Resources are delivered to watchers.
@@ -4084,8 +4072,9 @@ TEST_F(XdsClientTest, FallbackReportsError) {
               ::testing::ElementsAre(
                   ::testing::Pair(kDefaultXdsServerUrl, false),
                   ::testing::Pair(fallback_server.server_uri(), true)));
-  EXPECT_THAT(metrics_reporter_->server_failures(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1)));
+      ::testing::_, ::testing::_,
+      ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1))));
   // Fallback happens now
   stream = WaitForAdsStream(fallback_server);
   ASSERT_NE(stream, nullptr);
@@ -4101,10 +4090,11 @@ TEST_F(XdsClientTest, FallbackReportsError) {
               ::testing::ElementsAre(
                   ::testing::Pair(kDefaultXdsServerUrl, false),
                   ::testing::Pair(fallback_server.server_uri(), false)));
-  EXPECT_THAT(
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-      metrics_reporter_->server_failures(),
+      ::testing::_, ::testing::_,
-      ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1),
+      ::testing::ElementsAre(
-                             ::testing::Pair(fallback_server.server_uri(), 1)));
+          ::testing::Pair(kDefaultXdsServerUrl, 1),
+          ::testing::Pair(fallback_server.server_uri(), 1))));
   auto error = watcher->WaitForNextError();
   ASSERT_TRUE(error.has_value());
   EXPECT_THAT(error->code(), absl::StatusCode::kUnavailable);
@@ -4154,8 +4144,9 @@ TEST_F(XdsClientTest, FallbackOnStartup) {
               ::testing::ElementsAre(
                   ::testing::Pair(kDefaultXdsServerUrl, false),
                   ::testing::Pair(fallback_server.server_uri(), true)));
-  EXPECT_THAT(metrics_reporter_->server_failures(),
+  EXPECT_TRUE(metrics_reporter_->WaitForMetricsReporterData(
-              ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1)));
+      ::testing::_, ::testing::_,
+      ::testing::ElementsAre(::testing::Pair(kDefaultXdsServerUrl, 1))));
   // XdsClient should have delivered the response to the watcher.
   auto resource = watcher->WaitForNextResource();
   ASSERT_NE(resource, nullptr);