[CallTracer] Allow registering multiple call tracers on a call context (#33928)

This PR adds in delegating call tracers that work like so -
If this is the first call tracer that is being added onto the call
context, just add it as earlier.
If this is the second call tracer that is being added onto the call
context, create a delegating call tracer that contains a list of call
tracers (including the first call tracer).
Any more call tracers added, just get added to the list of tracers in
the delegating call tracer.

(This is not yet used other than through tests.)

<!--

If you know who should review your pull request, please assign it to
that
person, otherwise the pull request would get assigned randomly.

If your pull request is for a specific language, please add the
appropriate
lang label.

-->

CMakeLists.txt

@@ -862,6 +862,7 @@ if(gRPC_BUILD_TESTS)
   add_dependencies(buildtests_cxx call_creds_test)
   add_dependencies(buildtests_cxx call_finalization_test)
   add_dependencies(buildtests_cxx call_host_override_test)
+  add_dependencies(buildtests_cxx call_tracer_test)
   add_dependencies(buildtests_cxx cancel_after_accept_test)
   add_dependencies(buildtests_cxx cancel_after_client_done_test)
   add_dependencies(buildtests_cxx cancel_after_invoke_test)
@@ -7463,6 +7464,43 @@ target_link_libraries(call_host_override_test
 )
 
 
+endif()
+if(gRPC_BUILD_TESTS)
+
+add_executable(call_tracer_test
+  test/core/channel/call_tracer_test.cc
+  third_party/googletest/googletest/src/gtest-all.cc
+  third_party/googletest/googlemock/src/gmock-all.cc
+)
+target_compile_features(call_tracer_test PUBLIC cxx_std_14)
+target_include_directories(call_tracer_test
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
+    ${_gRPC_RE2_INCLUDE_DIR}
+    ${_gRPC_SSL_INCLUDE_DIR}
+    ${_gRPC_UPB_GENERATED_DIR}
+    ${_gRPC_UPB_GRPC_GENERATED_DIR}
+    ${_gRPC_UPB_INCLUDE_DIR}
+    ${_gRPC_XXHASH_INCLUDE_DIR}
+    ${_gRPC_ZLIB_INCLUDE_DIR}
+    third_party/googletest/googletest/include
+    third_party/googletest/googletest
+    third_party/googletest/googlemock/include
+    third_party/googletest/googlemock
+    ${_gRPC_PROTO_GENS_DIR}
+)
+
+target_link_libraries(call_tracer_test
+  ${_gRPC_BASELIB_LIBRARIES}
+  ${_gRPC_PROTOBUF_LIBRARIES}
+  ${_gRPC_ZLIB_LIBRARIES}
+  ${_gRPC_ALLTARGETS_LIBRARIES}
+  grpc_test_util
+)
+
+
 endif()
 if(gRPC_BUILD_TESTS)
 

build_autogenerated.yaml

@@ -5348,6 +5348,16 @@ targets:
   - grpc_authorization_provider
   - grpc_unsecure
   - grpc_test_util
+- name: call_tracer_test
+  gtest: true
+  build: test
+  language: c++
+  headers: []
+  src:
+  - test/core/channel/call_tracer_test.cc
+  deps:
+  - grpc_test_util
+  uses_polling: false
 - name: cancel_after_accept_test
   gtest: true
   build: test

src/core/lib/channel/call_tracer.cc

@@ -20,6 +20,14 @@
 
 #include "src/core/lib/channel/call_tracer.h"
 
+#include <algorithm>
+#include <utility>
+#include <vector>
+
+#include <grpc/support/log.h>
+
+#include "src/core/lib/promise/context.h"
+
 namespace grpc_core {
 
 //
@@ -27,6 +35,7 @@ namespace grpc_core {
 //
 
 namespace {
+
 ServerCallTracerFactory* g_server_call_tracer_factory_ = nullptr;
 
 const char* kServerCallTracerFactoryChannelArgName =
@@ -48,4 +57,284 @@ absl::string_view ServerCallTracerFactory::ChannelArgName() {
   return kServerCallTracerFactoryChannelArgName;
 }
 
+class DelegatingClientCallTracer : public ClientCallTracer {
+ public:
+  class DelegatingClientCallAttemptTracer
+      : public ClientCallTracer::CallAttemptTracer {
+   public:
+    explicit DelegatingClientCallAttemptTracer(
+        std::vector<CallAttemptTracer*> tracers)
+        : tracers_(std::move(tracers)) {
+      GPR_DEBUG_ASSERT(!tracers_.empty());
+    }
+    ~DelegatingClientCallAttemptTracer() override {}
+    void RecordSendInitialMetadata(
+        grpc_metadata_batch* send_initial_metadata) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordSendInitialMetadata(send_initial_metadata);
+      }
+    }
+    void RecordSendTrailingMetadata(
+        grpc_metadata_batch* send_trailing_metadata) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordSendTrailingMetadata(send_trailing_metadata);
+      }
+    }
+    void RecordSendMessage(const SliceBuffer& send_message) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordSendMessage(send_message);
+      }
+    }
+    void RecordSendCompressedMessage(
+        const SliceBuffer& send_compressed_message) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordSendCompressedMessage(send_compressed_message);
+      }
+    }
+    void RecordReceivedInitialMetadata(
+        grpc_metadata_batch* recv_initial_metadata) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordReceivedInitialMetadata(recv_initial_metadata);
+      }
+    }
+    void RecordReceivedMessage(const SliceBuffer& recv_message) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordReceivedMessage(recv_message);
+      }
+    }
+    void RecordReceivedDecompressedMessage(
+        const SliceBuffer& recv_decompressed_message) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordReceivedDecompressedMessage(recv_decompressed_message);
+      }
+    }
+    void RecordCancel(grpc_error_handle cancel_error) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordCancel(cancel_error);
+      }
+    }
+    void RecordReceivedTrailingMetadata(
+        absl::Status status, grpc_metadata_batch* recv_trailing_metadata,
+        const grpc_transport_stream_stats* transport_stream_stats) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordReceivedTrailingMetadata(status, recv_trailing_metadata,
+                                               transport_stream_stats);
+      }
+    }
+    void RecordEnd(const gpr_timespec& latency) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordEnd(latency);
+      }
+    }
+    void RecordAnnotation(absl::string_view annotation) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordAnnotation(annotation);
+      }
+    }
+    void RecordAnnotation(const Annotation& annotation) override {
+      for (auto* tracer : tracers_) {
+        tracer->RecordAnnotation(annotation);
+      }
+    }
+    std::string TraceId() override { return tracers_[0]->TraceId(); }
+    std::string SpanId() override { return tracers_[0]->SpanId(); }
+    bool IsSampled() override { return tracers_[0]->IsSampled(); }
+    bool IsDelegatingTracer() override { return true; }
+
+   private:
+    // There is no additional synchronization needed since filters/interceptors
+    // will be adding call tracers to the context and these are already
+    // synchronized through promises/call combiners (single promise running per
+    // call at any moment).
+    std::vector<CallAttemptTracer*> tracers_;
+  };
+  explicit DelegatingClientCallTracer(ClientCallTracer* tracer)
+      : tracers_{tracer} {}
+  ~DelegatingClientCallTracer() override {}
+  CallAttemptTracer* StartNewAttempt(bool is_transparent_retry) override {
+    std::vector<CallAttemptTracer*> attempt_tracers;
+    attempt_tracers.reserve(tracers_.size());
+    for (auto* tracer : tracers_) {
+      auto* attempt_tracer = tracer->StartNewAttempt(is_transparent_retry);
+      GPR_DEBUG_ASSERT(attempt_tracer != nullptr);
+      attempt_tracers.push_back(attempt_tracer);
+    }
+    return GetContext<Arena>()->ManagedNew<DelegatingClientCallAttemptTracer>(
+        std::move(attempt_tracers));
+  }
+
+  void RecordAnnotation(absl::string_view annotation) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordAnnotation(annotation);
+    }
+  }
+  void RecordAnnotation(const Annotation& annotation) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordAnnotation(annotation);
+    }
+  }
+  std::string TraceId() override { return tracers_[0]->TraceId(); }
+  std::string SpanId() override { return tracers_[0]->SpanId(); }
+  bool IsSampled() override { return tracers_[0]->IsSampled(); }
+  bool IsDelegatingTracer() override { return true; }
+
+  // There is no additional synchronization needed since filters/interceptors
+  // will be adding call tracers to the context and these are already
+  // synchronized through promises/call combiners (single promise running per
+  // call at any moment).
+  void AddTracer(ClientCallTracer* tracer) { tracers_.push_back(tracer); }
+
+ private:
+  std::vector<ClientCallTracer*> tracers_;
+};
+
+class DelegatingServerCallTracer : public ServerCallTracer {
+ public:
+  explicit DelegatingServerCallTracer(ServerCallTracer* tracer)
+      : tracers_{tracer} {}
+  ~DelegatingServerCallTracer() override {}
+  void RecordSendInitialMetadata(
+      grpc_metadata_batch* send_initial_metadata) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordSendInitialMetadata(send_initial_metadata);
+    }
+  }
+  void RecordSendTrailingMetadata(
+      grpc_metadata_batch* send_trailing_metadata) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordSendTrailingMetadata(send_trailing_metadata);
+    }
+  }
+  void RecordSendMessage(const SliceBuffer& send_message) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordSendMessage(send_message);
+    }
+  }
+  void RecordSendCompressedMessage(
+      const SliceBuffer& send_compressed_message) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordSendCompressedMessage(send_compressed_message);
+    }
+  }
+  void RecordReceivedInitialMetadata(
+      grpc_metadata_batch* recv_initial_metadata) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordReceivedInitialMetadata(recv_initial_metadata);
+    }
+  }
+  void RecordReceivedMessage(const SliceBuffer& recv_message) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordReceivedMessage(recv_message);
+    }
+  }
+  void RecordReceivedDecompressedMessage(
+      const SliceBuffer& recv_decompressed_message) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordReceivedDecompressedMessage(recv_decompressed_message);
+    }
+  }
+  void RecordCancel(grpc_error_handle cancel_error) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordCancel(cancel_error);
+    }
+  }
+  void RecordReceivedTrailingMetadata(
+      grpc_metadata_batch* recv_trailing_metadata) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordReceivedTrailingMetadata(recv_trailing_metadata);
+    }
+  }
+  void RecordEnd(const grpc_call_final_info* final_info) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordEnd(final_info);
+    }
+  }
+  void RecordAnnotation(absl::string_view annotation) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordAnnotation(annotation);
+    }
+  }
+  void RecordAnnotation(const Annotation& annotation) override {
+    for (auto* tracer : tracers_) {
+      tracer->RecordAnnotation(annotation);
+    }
+  }
+  std::string TraceId() override { return tracers_[0]->TraceId(); }
+  std::string SpanId() override { return tracers_[0]->SpanId(); }
+  bool IsSampled() override { return tracers_[0]->IsSampled(); }
+  bool IsDelegatingTracer() override { return true; }
+
+  void AddTracer(ServerCallTracer* tracer) { tracers_.push_back(tracer); }
+
+ private:
+  // The ServerCallTracerFilter will be responsible for making sure that the
+  // tracers are added in a thread-safe manner. It is imagined that the filter
+  // will just invoke the factories in the server call tracer factory list
+  // sequentially, removing the need for any synchronization.
+  std::vector<ServerCallTracer*> tracers_;
+};
+
+void AddClientCallTracerToContext(grpc_call_context_element* call_context,
+                                  ClientCallTracer* tracer) {
+  if (call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value ==
+      nullptr) {
+    // This is the first call tracer. Set it directly.
+    call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value = tracer;
+    call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].destroy =
+        nullptr;
+  } else {
+    // There was already a call tracer present.
+    auto* orig_tracer = static_cast<ClientCallTracer*>(
+        call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value);
+    if (orig_tracer->IsDelegatingTracer()) {
+      // We already created a delegating tracer. Just add the new tracer to the
+      // list.
+      static_cast<DelegatingClientCallTracer*>(orig_tracer)->AddTracer(tracer);
+    } else {
+      // Create a new delegating tracer and add the first tracer and the new
+      // tracer to the list.
+      auto* delegating_tracer =
+          GetContext<Arena>()->ManagedNew<DelegatingClientCallTracer>(
+              orig_tracer);
+      call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value =
+          delegating_tracer;
+      delegating_tracer->AddTracer(tracer);
+    }
+  }
+}
+
+void AddServerCallTracerToContext(grpc_call_context_element* call_context,
+                                  ServerCallTracer* tracer) {
+  GPR_DEBUG_ASSERT(
+      call_context[GRPC_CONTEXT_CALL_TRACER].value ==
+      call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value);
+  if (call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value ==
+      nullptr) {
+    // This is the first call tracer. Set it directly.
+    call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value = tracer;
+    call_context[GRPC_CONTEXT_CALL_TRACER].value = tracer;
+    call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].destroy =
+        nullptr;
+  } else {
+    // There was already a call tracer present.
+    auto* orig_tracer = static_cast<ServerCallTracer*>(
+        call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value);
+    if (orig_tracer->IsDelegatingTracer()) {
+      // We already created a delegating tracer. Just add the new tracer to the
+      // list.
+      static_cast<DelegatingServerCallTracer*>(orig_tracer)->AddTracer(tracer);
+    } else {
+      // Create a new delegating tracer and add the first tracer and the new
+      // tracer to the list.
+      auto* delegating_tracer =
+          GetContext<Arena>()->ManagedNew<DelegatingServerCallTracer>(
+              orig_tracer);
+      call_context[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value =
+          delegating_tracer;
+      call_context[GRPC_CONTEXT_CALL_TRACER].value = delegating_tracer;
+      delegating_tracer->AddTracer(tracer);
+    }
+  }
+}
+
 }  // namespace grpc_core

src/core/lib/channel/call_tracer.h

@@ -30,6 +30,7 @@
 
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/channel/context.h"
 #include "src/core/lib/config/core_configuration.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/resource_quota/arena.h"
@@ -76,6 +77,10 @@ class CallTracerAnnotationInterface {
   virtual std::string TraceId() = 0;
   virtual std::string SpanId() = 0;
   virtual bool IsSampled() = 0;
+  // Indicates whether this tracer is a delegating tracer or not.
+  // `DelegatingClientCallTracer`, `DelegatingClientCallAttemptTracer` and
+  // `DelegatingServerCallTracer` are the only delegating call tracers.
+  virtual bool IsDelegatingTracer() { return false; }
 };
 
 // The base class for CallAttemptTracer and ServerCallTracer.
@@ -180,6 +185,17 @@ class ServerCallTracerFactory {
 
 void RegisterServerCallTracerFilter(CoreConfiguration::Builder* builder);
 
+// Convenience functions to add call tracers to a call context. Allows setting
+// multiple call tracers to a single call. It is only valid to add client call
+// tracers before the client_channel filter sees the send_initial_metadata op.
+void AddClientCallTracerToContext(grpc_call_context_element* call_context,
+                                  ClientCallTracer* tracer);
+
+// TODO(yashykt): We want server call tracers to be registered through the
+// ServerCallTracerFactory, which has yet to be made into a list.
+void AddServerCallTracerToContext(grpc_call_context_element* call_context,
+                                  ServerCallTracer* tracer);
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_CHANNEL_CALL_TRACER_H

test/core/channel/BUILD

@@ -18,6 +18,19 @@ grpc_package(name = "test/core/channel")
 
 licenses(["notice"])
 
+grpc_cc_test(
+    name = "call_tracer_test",
+    srcs = ["call_tracer_test.cc"],
+    external_deps = ["gtest"],
+    language = "C++",
+    uses_event_engine = False,
+    uses_polling = False,
+    deps = [
+        "//:grpc",
+        "//test/core/util:grpc_test_util",
+    ],
+)
+
 grpc_cc_test(
     name = "channel_args_test",
     srcs = ["channel_args_test.cc"],

test/core/channel/call_tracer_test.cc

@@ -0,0 +1,230 @@
+//
+//
+// Copyright 2023 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+//
+
+#include "src/core/lib/channel/call_tracer.h"
+
+#include <algorithm>
+#include <memory>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include <grpc/event_engine/memory_allocator.h>
+#include <grpc/grpc.h>
+
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/promise/context.h"
+#include "src/core/lib/resource_quota/memory_quota.h"
+#include "src/core/lib/resource_quota/resource_quota.h"
+#include "test/core/util/test_config.h"
+
+namespace grpc_core {
+namespace {
+
+class FakeClientCallTracer : public ClientCallTracer {
+ public:
+  class FakeClientCallAttemptTracer
+      : public ClientCallTracer::CallAttemptTracer {
+   public:
+    explicit FakeClientCallAttemptTracer(
+        std::vector<std::string>* annotation_logger)
+        : annotation_logger_(annotation_logger) {}
+    ~FakeClientCallAttemptTracer() override {}
+    void RecordSendInitialMetadata(
+        grpc_metadata_batch* /*send_initial_metadata*/) override {}
+    void RecordSendTrailingMetadata(
+        grpc_metadata_batch* /*send_trailing_metadata*/) override {}
+    void RecordSendMessage(const SliceBuffer& /*send_message*/) override {}
+    void RecordSendCompressedMessage(
+        const SliceBuffer& /*send_compressed_message*/) override {}
+    void RecordReceivedInitialMetadata(
+        grpc_metadata_batch* /*recv_initial_metadata*/) override {}
+    void RecordReceivedMessage(const SliceBuffer& /*recv_message*/) override {}
+    void RecordReceivedDecompressedMessage(
+        const SliceBuffer& /*recv_decompressed_message*/) override {}
+    void RecordCancel(grpc_error_handle /*cancel_error*/) override {}
+    void RecordReceivedTrailingMetadata(
+        absl::Status /*status*/,
+        grpc_metadata_batch* /*recv_trailing_metadata*/,
+        const grpc_transport_stream_stats* /*transport_stream_stats*/)
+        override {}
+    void RecordEnd(const gpr_timespec& /*latency*/) override { delete this; }
+    void RecordAnnotation(absl::string_view annotation) override {
+      annotation_logger_->push_back(std::string(annotation));
+    }
+    void RecordAnnotation(const Annotation& /*annotation*/) override {}
+    std::string TraceId() override { return ""; }
+    std::string SpanId() override { return ""; }
+    bool IsSampled() override { return false; }
+
+   private:
+    std::vector<std::string>* annotation_logger_;
+  };
+
+  explicit FakeClientCallTracer(std::vector<std::string>* annotation_logger)
+      : annotation_logger_(annotation_logger) {}
+  ~FakeClientCallTracer() override {}
+  CallAttemptTracer* StartNewAttempt(bool /*is_transparent_retry*/) override {
+    return GetContext<Arena>()->ManagedNew<FakeClientCallAttemptTracer>(
+        annotation_logger_);
+  }
+
+  void RecordAnnotation(absl::string_view annotation) override {
+    annotation_logger_->push_back(std::string(annotation));
+  }
+  void RecordAnnotation(const Annotation& /*annotation*/) override {}
+  std::string TraceId() override { return ""; }
+  std::string SpanId() override { return ""; }
+  bool IsSampled() override { return false; }
+
+ private:
+  std::vector<std::string>* annotation_logger_;
+};
+
+class FakeServerCallTracer : public ServerCallTracer {
+ public:
+  explicit FakeServerCallTracer(std::vector<std::string>* annotation_logger)
+      : annotation_logger_(annotation_logger) {}
+  ~FakeServerCallTracer() override {}
+  void RecordSendInitialMetadata(
+      grpc_metadata_batch* /*send_initial_metadata*/) override {}
+  void RecordSendTrailingMetadata(
+      grpc_metadata_batch* /*send_trailing_metadata*/) override {}
+  void RecordSendMessage(const SliceBuffer& /*send_message*/) override {}
+  void RecordSendCompressedMessage(
+      const SliceBuffer& /*send_compressed_message*/) override {}
+  void RecordReceivedInitialMetadata(
+      grpc_metadata_batch* /*recv_initial_metadata*/) override {}
+  void RecordReceivedMessage(const SliceBuffer& /*recv_message*/) override {}
+  void RecordReceivedDecompressedMessage(
+      const SliceBuffer& /*recv_decompressed_message*/) override {}
+  void RecordCancel(grpc_error_handle /*cancel_error*/) override {}
+  void RecordReceivedTrailingMetadata(
+      grpc_metadata_batch* /*recv_trailing_metadata*/) override {}
+  void RecordEnd(const grpc_call_final_info* /*final_info*/) override {}
+  void RecordAnnotation(absl::string_view annotation) override {
+    annotation_logger_->push_back(std::string(annotation));
+  }
+  void RecordAnnotation(const Annotation& /*annotation*/) override {}
+  std::string TraceId() override { return ""; }
+  std::string SpanId() override { return ""; }
+  bool IsSampled() override { return false; }
+
+ private:
+  std::vector<std::string>* annotation_logger_;
+};
+
+class CallTracerTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    memory_allocator_ = new MemoryAllocator(
+        ResourceQuota::Default()->memory_quota()->CreateMemoryAllocator(
+            "test"));
+    arena_ = Arena::Create(1024, memory_allocator_);
+  }
+
+  void TearDown() override {
+    arena_->Destroy();
+    delete memory_allocator_;
+  }
+
+  MemoryAllocator* memory_allocator_ = nullptr;
+  Arena* arena_ = nullptr;
+  grpc_call_context_element context_[GRPC_CONTEXT_COUNT] = {};
+  std::vector<std::string> annotation_logger_;
+};
+
+TEST_F(CallTracerTest, BasicClientCallTracer) {
+  FakeClientCallTracer client_call_tracer(&annotation_logger_);
+  AddClientCallTracerToContext(context_, &client_call_tracer);
+  static_cast<CallTracerAnnotationInterface*>(
+      context_[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value)
+      ->RecordAnnotation("Test");
+  EXPECT_EQ(annotation_logger_, std::vector<std::string>{"Test"});
+}
+
+TEST_F(CallTracerTest, MultipleClientCallTracers) {
+  promise_detail::Context<Arena> arena_ctx(arena_);
+  FakeClientCallTracer client_call_tracer1(&annotation_logger_);
+  FakeClientCallTracer client_call_tracer2(&annotation_logger_);
+  FakeClientCallTracer client_call_tracer3(&annotation_logger_);
+  AddClientCallTracerToContext(context_, &client_call_tracer1);
+  AddClientCallTracerToContext(context_, &client_call_tracer2);
+  AddClientCallTracerToContext(context_, &client_call_tracer3);
+  static_cast<CallTracerAnnotationInterface*>(
+      context_[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value)
+      ->RecordAnnotation("Test");
+  EXPECT_EQ(annotation_logger_,
+            std::vector<std::string>({"Test", "Test", "Test"}));
+}
+
+TEST_F(CallTracerTest, MultipleClientCallAttemptTracers) {
+  promise_detail::Context<Arena> arena_ctx(arena_);
+  FakeClientCallTracer client_call_tracer1(&annotation_logger_);
+  FakeClientCallTracer client_call_tracer2(&annotation_logger_);
+  FakeClientCallTracer client_call_tracer3(&annotation_logger_);
+  AddClientCallTracerToContext(context_, &client_call_tracer1);
+  AddClientCallTracerToContext(context_, &client_call_tracer2);
+  AddClientCallTracerToContext(context_, &client_call_tracer3);
+  auto* attempt_tracer =
+      static_cast<ClientCallTracer*>(
+          context_[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value)
+          ->StartNewAttempt(true /* is_transparent_retry */);
+  attempt_tracer->RecordAnnotation("Test");
+  EXPECT_EQ(annotation_logger_,
+            std::vector<std::string>({"Test", "Test", "Test"}));
+}
+
+TEST_F(CallTracerTest, BasicServerCallTracerTest) {
+  FakeServerCallTracer server_call_tracer(&annotation_logger_);
+  AddServerCallTracerToContext(context_, &server_call_tracer);
+  static_cast<CallTracerAnnotationInterface*>(
+      context_[GRPC_CONTEXT_CALL_TRACER].value)
+      ->RecordAnnotation("Test");
+  static_cast<CallTracerAnnotationInterface*>(
+      context_[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value)
+      ->RecordAnnotation("Test");
+  EXPECT_EQ(annotation_logger_, std::vector<std::string>({"Test", "Test"}));
+}
+
+TEST_F(CallTracerTest, MultipleServerCallTracers) {
+  promise_detail::Context<Arena> arena_ctx(arena_);
+  FakeServerCallTracer server_call_tracer1(&annotation_logger_);
+  FakeServerCallTracer server_call_tracer2(&annotation_logger_);
+  FakeServerCallTracer server_call_tracer3(&annotation_logger_);
+  AddServerCallTracerToContext(context_, &server_call_tracer1);
+  AddServerCallTracerToContext(context_, &server_call_tracer2);
+  AddServerCallTracerToContext(context_, &server_call_tracer3);
+  static_cast<CallTracerAnnotationInterface*>(
+      context_[GRPC_CONTEXT_CALL_TRACER_ANNOTATION_INTERFACE].value)
+      ->RecordAnnotation("Test");
+  EXPECT_EQ(annotation_logger_,
+            std::vector<std::string>({"Test", "Test", "Test"}));
+}
+
+}  // namespace
+}  // namespace grpc_core
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  grpc::testing::TestEnvironment env(&argc, argv);
+  grpc_init();
+  auto r = RUN_ALL_TESTS();
+  grpc_shutdown();
+  return r;
+}

tools/run_tests/generated/tests.json

@@ -1301,6 +1301,30 @@
     ],
     "uses_polling": true
   },
+  {
+    "args": [],
+    "benchmark": false,
+    "ci_platforms": [
+      "linux",
+      "mac",
+      "posix",
+      "windows"
+    ],
+    "cpu_cost": 1.0,
+    "exclude_configs": [],
+    "exclude_iomgrs": [],
+    "flaky": false,
+    "gtest": true,
+    "language": "c++",
+    "name": "call_tracer_test",
+    "platforms": [
+      "linux",
+      "mac",
+      "posix",
+      "windows"
+    ],
+    "uses_polling": false
+  },
   {
     "args": [],
     "benchmark": false,