diff --git a/src/cpp/ext/csm/metadata_exchange.cc b/src/cpp/ext/csm/metadata_exchange.cc index 340fbf36fda..c4908efe85e 100644 --- a/src/cpp/ext/csm/metadata_exchange.cc +++ b/src/cpp/ext/csm/metadata_exchange.cc @@ -34,10 +34,8 @@ #include "absl/strings/strip.h" #include "absl/types/optional.h" #include "absl/types/variant.h" -#include "google/protobuf/struct.upb.h" #include "opentelemetry/sdk/resource/semantic_conventions.h" #include "upb/base/string_view.h" -#include "upb/mem/arena.hpp" #include @@ -91,7 +89,24 @@ constexpr absl::string_view kPeerCanonicalServiceAttribute = constexpr absl::string_view kGkeType = "gcp_kubernetes_engine"; constexpr absl::string_view kGceType = "gcp_compute_engine"; -enum class GcpResourceType : std::uint8_t { kGke, kGce, kUnknown }; +// A helper method that decodes the remote metadata \a slice as a protobuf +// Struct allocated on \a arena. +google_protobuf_Struct* DecodeMetadata(grpc_core::Slice slice, + upb_Arena* arena) { + // Treat an empty slice as an invalid metadata value. + if (slice.empty()) { + return nullptr; + } + // Decode the slice. + std::string decoded_metadata; + bool metadata_decoded = + absl::Base64Unescape(slice.as_string_view(), &decoded_metadata); + if (metadata_decoded) { + return google_protobuf_Struct_parse(decoded_metadata.c_str(), + decoded_metadata.size(), arena); + } + return nullptr; +} // A minimal class for helping with the information we need from the xDS // bootstrap file for GSM Observability reasons. @@ -145,13 +160,14 @@ std::string GetXdsBootstrapContents() { return ""; } -GcpResourceType StringToGcpResourceType(absl::string_view type) { +MeshLabelsIterable::GcpResourceType StringToGcpResourceType( + absl::string_view type) { if (type == kGkeType) { - return GcpResourceType::kGke; + return MeshLabelsIterable::GcpResourceType::kGke; } else if (type == kGceType) { - return GcpResourceType::kGce; + return MeshLabelsIterable::GcpResourceType::kGce; } - return GcpResourceType::kUnknown; + return MeshLabelsIterable::GcpResourceType::kUnknown; } upb_StringView AbslStrToUpbStr(absl::string_view str) { @@ -203,147 +219,96 @@ absl::string_view GetStringValueFromUpbStruct(google_protobuf_Struct* struct_pb, return "unknown"; } -class MeshLabelsIterable : public LabelsIterable { - public: - explicit MeshLabelsIterable( - const std::vector>& - local_labels, - grpc_core::Slice remote_metadata) - : local_labels_(local_labels), metadata_(std::move(remote_metadata)) {} - - absl::optional> Next() - override { - auto& struct_pb = GetDecodedMetadata(); - size_t local_labels_size = local_labels_.size(); - if (pos_ < local_labels_size) { - return local_labels_[pos_++]; - } - const size_t fixed_attribute_end = - local_labels_size + kFixedAttributes.size(); - if (pos_ < fixed_attribute_end) { - return NextFromAttributeList(struct_pb, kFixedAttributes, - local_labels_size); - } - return NextFromAttributeList(struct_pb, GetAttributesForType(remote_type_), - fixed_attribute_end); - } - - size_t Size() const override { - return local_labels_.size() + kFixedAttributes.size() + - GetAttributesForType(remote_type_).size(); - } +struct RemoteAttribute { + absl::string_view otel_attribute; + absl::string_view metadata_attribute; +}; - void ResetIteratorPosition() override { pos_ = 0; } +constexpr std::array kFixedAttributes = { + RemoteAttribute{kPeerTypeAttribute, kMetadataExchangeTypeKey}, + RemoteAttribute{kPeerCanonicalServiceAttribute, + kMetadataExchangeCanonicalServiceKey}, +}; - // Returns true if the peer sent a non-empty base64 encoded - // "x-envoy-peer-metadata" metadata. - bool GotRemoteLabels() const { - return GetDecodedMetadata().struct_pb != nullptr; - } +constexpr std::array kGkeAttributeList = { + RemoteAttribute{kPeerWorkloadNameAttribute, + kMetadataExchangeWorkloadNameKey}, + RemoteAttribute{kPeerNamespaceNameAttribute, + kMetadataExchangeNamespaceNameKey}, + RemoteAttribute{kPeerClusterNameAttribute, kMetadataExchangeClusterNameKey}, + RemoteAttribute{kPeerLocationAttribute, kMetadataExchangeLocationKey}, + RemoteAttribute{kPeerProjectIdAttribute, kMetadataExchangeProjectIdKey}, +}; - private: - struct RemoteAttribute { - absl::string_view otel_attribute; - absl::string_view metadata_attribute; - }; +constexpr std::array kGceAttributeList = { + RemoteAttribute{kPeerWorkloadNameAttribute, + kMetadataExchangeWorkloadNameKey}, + RemoteAttribute{kPeerLocationAttribute, kMetadataExchangeLocationKey}, + RemoteAttribute{kPeerProjectIdAttribute, kMetadataExchangeProjectIdKey}, +}; - struct StructPb { - upb::Arena arena; - google_protobuf_Struct* struct_pb = nullptr; - }; +absl::Span GetAttributesForType( + MeshLabelsIterable::GcpResourceType remote_type) { + switch (remote_type) { + case MeshLabelsIterable::GcpResourceType::kGke: + return kGkeAttributeList; + case MeshLabelsIterable::GcpResourceType::kGce: + return kGceAttributeList; + default: + return {}; + } +} - static constexpr std::array kFixedAttributes = { - RemoteAttribute{kPeerTypeAttribute, kMetadataExchangeTypeKey}, - RemoteAttribute{kPeerCanonicalServiceAttribute, - kMetadataExchangeCanonicalServiceKey}, - }; +absl::optional> +NextFromAttributeList(absl::Span attributes, + size_t start_index, size_t curr, + google_protobuf_Struct* decoded_metadata, + upb_Arena* arena) { + GPR_DEBUG_ASSERT(curr >= start_index); + const size_t index = curr - start_index; + if (index >= attributes.size()) return absl::nullopt; + const auto& attribute = attributes[index]; + return std::make_pair( + attribute.otel_attribute, + GetStringValueFromUpbStruct(decoded_metadata, + attribute.metadata_attribute, arena)); +} - static constexpr std::array kGkeAttributeList = { - RemoteAttribute{kPeerWorkloadNameAttribute, - kMetadataExchangeWorkloadNameKey}, - RemoteAttribute{kPeerNamespaceNameAttribute, - kMetadataExchangeNamespaceNameKey}, - RemoteAttribute{kPeerClusterNameAttribute, - kMetadataExchangeClusterNameKey}, - RemoteAttribute{kPeerLocationAttribute, kMetadataExchangeLocationKey}, - RemoteAttribute{kPeerProjectIdAttribute, kMetadataExchangeProjectIdKey}, - }; - static constexpr std::array kGceAttributeList = { - RemoteAttribute{kPeerWorkloadNameAttribute, - kMetadataExchangeWorkloadNameKey}, - RemoteAttribute{kPeerLocationAttribute, kMetadataExchangeLocationKey}, - RemoteAttribute{kPeerProjectIdAttribute, kMetadataExchangeProjectIdKey}, - }; +} // namespace - static absl::Span GetAttributesForType( - GcpResourceType remote_type) { - switch (remote_type) { - case GcpResourceType::kGke: - return kGkeAttributeList; - case GcpResourceType::kGce: - return kGceAttributeList; - default: - return {}; - } - } +// +// MeshLabelsIterable +// - absl::optional> - NextFromAttributeList(const StructPb& struct_pb, - absl::Span attributes, - size_t start_index) { - GPR_DEBUG_ASSERT(pos_ >= start_index); - const size_t index = pos_ - start_index; - if (index >= attributes.size()) return absl::nullopt; - ++pos_; - const auto& attribute = attributes[index]; - return std::make_pair(attribute.otel_attribute, - GetStringValueFromUpbStruct( - struct_pb.struct_pb, attribute.metadata_attribute, - struct_pb.arena.ptr())); +MeshLabelsIterable::MeshLabelsIterable( + const std::vector>& local_labels, + grpc_core::Slice remote_metadata) + : struct_pb_(DecodeMetadata(std::move(remote_metadata), arena_.ptr())), + local_labels_(local_labels), + remote_type_(StringToGcpResourceType(GetStringValueFromUpbStruct( + struct_pb_, kMetadataExchangeTypeKey, arena_.ptr()))) {} + +absl::optional> +MeshLabelsIterable::Next() { + size_t local_labels_size = local_labels_.size(); + if (pos_ < local_labels_size) { + return local_labels_[pos_++]; } - - StructPb& GetDecodedMetadata() const { - auto* slice = absl::get_if(&metadata_); - if (slice == nullptr) { - return absl::get(metadata_); - } - // Treat an empty slice as an invalid metadata value. - if (slice->empty()) { - metadata_ = StructPb{}; - auto& struct_pb = absl::get(metadata_); - return struct_pb; - } - std::string decoded_metadata; - bool metadata_decoded = - absl::Base64Unescape(slice->as_string_view(), &decoded_metadata); - metadata_ = StructPb{}; - auto& struct_pb = absl::get(metadata_); - if (metadata_decoded) { - struct_pb.struct_pb = google_protobuf_Struct_parse( - decoded_metadata.c_str(), decoded_metadata.size(), - struct_pb.arena.ptr()); - remote_type_ = StringToGcpResourceType(GetStringValueFromUpbStruct( - struct_pb.struct_pb, kMetadataExchangeTypeKey, - struct_pb.arena.ptr())); - } - return struct_pb; + const size_t fixed_attribute_end = + local_labels_size + kFixedAttributes.size(); + if (pos_ < fixed_attribute_end) { + return NextFromAttributeList(kFixedAttributes, local_labels_size, pos_++, + struct_pb_, arena_.ptr()); } + return NextFromAttributeList(GetAttributesForType(remote_type_), + fixed_attribute_end, pos_++, struct_pb_, + arena_.ptr()); +} - const std::vector>& local_labels_; - // Holds either the metadata slice or the decoded proto struct. - mutable absl::variant metadata_; - mutable GcpResourceType remote_type_ = GcpResourceType::kUnknown; - uint32_t pos_ = 0; -}; - -constexpr std::array - MeshLabelsIterable::kFixedAttributes; -constexpr std::array - MeshLabelsIterable::kGkeAttributeList; -constexpr std::array - MeshLabelsIterable::kGceAttributeList; - -} // namespace +size_t MeshLabelsIterable::Size() const { + return local_labels_.size() + kFixedAttributes.size() + + GetAttributesForType(remote_type_).size(); +} // Returns the mesh ID by reading and parsing the bootstrap file. Returns // "unknown" if for some reason, mesh ID could not be figured out. @@ -370,6 +335,10 @@ std::string GetMeshId() { return std::string(mesh_id); } +// +// ServiceMeshLabelsInjector +// + ServiceMeshLabelsInjector::ServiceMeshLabelsInjector( const opentelemetry::sdk::common::AttributeMap& map) { upb::Arena arena; diff --git a/src/cpp/ext/csm/metadata_exchange.h b/src/cpp/ext/csm/metadata_exchange.h index ab74b1916fa..560a07f1e97 100644 --- a/src/cpp/ext/csm/metadata_exchange.h +++ b/src/cpp/ext/csm/metadata_exchange.h @@ -27,7 +27,9 @@ #include #include "absl/strings/string_view.h" +#include "google/protobuf/struct.upb.h" #include "opentelemetry/sdk/common/attribute_utils.h" +#include "upb/mem/arena.hpp" #include "src/core/lib/slice/slice.h" #include "src/core/lib/transport/metadata_batch.h" @@ -66,11 +68,50 @@ class ServiceMeshLabelsInjector : public LabelsInjector { absl::Span>> optional_labels_span) const override; + const std::vector>& + TestOnlyLocalLabels() const { + return local_labels_; + } + + const grpc_core::Slice& TestOnlySerializedLabels() const { + return serialized_labels_to_send_; + } + private: std::vector> local_labels_; grpc_core::Slice serialized_labels_to_send_; }; +// A LabelsIterable class provided by ServiceMeshLabelsInjector. EXPOSED FOR +// TESTING PURPOSES ONLY. +class MeshLabelsIterable : public LabelsIterable { + public: + enum class GcpResourceType : std::uint8_t { kGke, kGce, kUnknown }; + + MeshLabelsIterable( + const std::vector>& + local_labels, + grpc_core::Slice remote_metadata); + + absl::optional> Next() + override; + + size_t Size() const override; + + void ResetIteratorPosition() override { pos_ = 0; } + + // Returns true if the peer sent a non-empty base64 encoded + // "x-envoy-peer-metadata" metadata. + bool GotRemoteLabels() const { return struct_pb_ != nullptr; } + + private: + upb::Arena arena_; + google_protobuf_Struct* struct_pb_ = nullptr; + const std::vector>& local_labels_; + GcpResourceType remote_type_ = GcpResourceType::kUnknown; + uint32_t pos_ = 0; +}; + // Returns the mesh ID by reading and parsing the bootstrap file. Returns // "unknown" if for some reason, mesh ID could not be figured out. // EXPOSED FOR TESTING PURPOSES ONLY. diff --git a/test/cpp/ext/csm/metadata_exchange_test.cc b/test/cpp/ext/csm/metadata_exchange_test.cc index 19667491da3..67299f73e40 100644 --- a/test/cpp/ext/csm/metadata_exchange_test.cc +++ b/test/cpp/ext/csm/metadata_exchange_test.cc @@ -43,6 +43,35 @@ namespace grpc { namespace testing { namespace { +using ::testing::ElementsAre; +using ::testing::Pair; + +opentelemetry::sdk::resource::Resource TestGkeResource() { + opentelemetry::sdk::common::AttributeMap attributes; + attributes.SetAttribute("cloud.platform", "gcp_kubernetes_engine"); + attributes.SetAttribute("k8s.pod.name", "pod"); + attributes.SetAttribute("k8s.container.name", "container"); + attributes.SetAttribute("k8s.namespace.name", "namespace"); + attributes.SetAttribute("k8s.cluster.name", "cluster"); + attributes.SetAttribute("cloud.region", "region"); + attributes.SetAttribute("cloud.account.id", "id"); + return opentelemetry::sdk::resource::Resource::Create(attributes); +} + +opentelemetry::sdk::resource::Resource TestGceResource() { + opentelemetry::sdk::common::AttributeMap attributes; + attributes.SetAttribute("cloud.platform", "gcp_compute_engine"); + attributes.SetAttribute("cloud.availability_zone", "zone"); + attributes.SetAttribute("cloud.account.id", "id"); + return opentelemetry::sdk::resource::Resource::Create(attributes); +} + +opentelemetry::sdk::resource::Resource TestUnknownResource() { + opentelemetry::sdk::common::AttributeMap attributes; + attributes.SetAttribute("cloud.platform", "random"); + return opentelemetry::sdk::resource::Resource::Create(attributes); +} + class TestScenario { public: enum class ResourceType : std::uint8_t { kGke, kGce, kUnknown }; @@ -91,32 +120,6 @@ class TestScenario { XdsBootstrapSource bootstrap_source() const { return bootstrap_source_; } private: - static opentelemetry::sdk::resource::Resource TestGkeResource() { - opentelemetry::sdk::common::AttributeMap attributes; - attributes.SetAttribute("cloud.platform", "gcp_kubernetes_engine"); - attributes.SetAttribute("k8s.pod.name", "pod"); - attributes.SetAttribute("k8s.container.name", "container"); - attributes.SetAttribute("k8s.namespace.name", "namespace"); - attributes.SetAttribute("k8s.cluster.name", "cluster"); - attributes.SetAttribute("cloud.region", "region"); - attributes.SetAttribute("cloud.account.id", "id"); - return opentelemetry::sdk::resource::Resource::Create(attributes); - } - - static opentelemetry::sdk::resource::Resource TestGceResource() { - opentelemetry::sdk::common::AttributeMap attributes; - attributes.SetAttribute("cloud.platform", "gcp_compute_engine"); - attributes.SetAttribute("cloud.availability_zone", "zone"); - attributes.SetAttribute("cloud.account.id", "id"); - return opentelemetry::sdk::resource::Resource::Create(attributes); - } - - static opentelemetry::sdk::resource::Resource TestUnknownResource() { - opentelemetry::sdk::common::AttributeMap attributes; - attributes.SetAttribute("cloud.platform", "random"); - return opentelemetry::sdk::resource::Resource::Create(attributes); - } - ResourceType type_; XdsBootstrapSource bootstrap_source_; }; @@ -163,7 +166,7 @@ class MetadataExchangeTest case TestScenario::XdsBootstrapSource::kFromFile: { ASSERT_EQ(bootstrap_file_name_, nullptr); FILE* bootstrap_file = - gpr_tmpfile("gcp_observability_config", &bootstrap_file_name_); + gpr_tmpfile("xds_bootstrap", &bootstrap_file_name_); fputs(kBootstrap, bootstrap_file); fclose(bootstrap_file); grpc_core::SetEnv("GRPC_XDS_BOOTSTRAP", bootstrap_file_name_); @@ -186,7 +189,7 @@ class MetadataExchangeTest } ~MetadataExchangeTest() override { - grpc_core::UnsetEnv("GRPC_GCP_OBSERVABILITY_CONFIG"); + grpc_core::UnsetEnv("GRPC_XDS_BOOTSTRAP_CONFIG"); grpc_core::UnsetEnv("GRPC_XDS_BOOTSTRAP"); if (bootstrap_file_name_ != nullptr) { remove(bootstrap_file_name_); @@ -418,6 +421,134 @@ TEST_P(MetadataExchangeTest, VerifyCsmServiceLabels) { "mynamespace"); } +// Creates a serialized slice with labels for metadata exchange based on \a +// resource. +grpc_core::Slice RemoteMetadataSliceFromResource( + const opentelemetry::sdk::resource::Resource& resource) { + return grpc::internal::ServiceMeshLabelsInjector(resource.GetAttributes()) + .TestOnlySerializedLabels() + .Ref(); +} + +std::vector> LabelsFromIterable( + grpc::internal::MeshLabelsIterable* iterable) { + std::vector> labels; + while (true) { + auto label = iterable->Next(); + if (!label.has_value()) break; + labels.push_back(*std::move(label)); + } + EXPECT_EQ(labels.size(), iterable->Size()); + return labels; +} + +std::string PrettyPrintLabels( + const std::vector>& + labels) { + std::vector strings; + strings.reserve(labels.size()); + for (const auto& pair : labels) { + strings.push_back( + absl::StrFormat("{\"%s\" : \"%s\"}", pair.first, pair.second)); + } + return absl::StrJoin(strings, ", "); +} + +TEST(MeshLabelsIterableTest, NoRemoteMetadata) { + std::vector> local_labels = { + {"csm.workload_canonical_service", "canonical_service"}, + {"csm.mesh_id", "mesh"}}; + grpc::internal::MeshLabelsIterable iterable(local_labels, grpc_core::Slice()); + auto labels = LabelsFromIterable(&iterable); + EXPECT_FALSE(iterable.GotRemoteLabels()); + EXPECT_THAT( + labels, + ElementsAre(Pair("csm.workload_canonical_service", "canonical_service"), + Pair("csm.mesh_id", "mesh"), + Pair("csm.remote_workload_type", "unknown"), + Pair("csm.remote_workload_canonical_service", "unknown"))) + << PrettyPrintLabels(labels); +} + +TEST(MeshLabelsIterableTest, RemoteGceTypeMetadata) { + std::vector> local_labels = { + {"csm.workload_canonical_service", "canonical_service"}, + {"csm.mesh_id", "mesh"}}; + grpc::internal::MeshLabelsIterable iterable( + local_labels, RemoteMetadataSliceFromResource(TestGceResource())); + auto labels = LabelsFromIterable(&iterable); + EXPECT_TRUE(iterable.GotRemoteLabels()); + EXPECT_THAT( + labels, + ElementsAre( + Pair("csm.workload_canonical_service", "canonical_service"), + Pair("csm.mesh_id", "mesh"), + Pair("csm.remote_workload_type", "gcp_compute_engine"), + Pair("csm.remote_workload_canonical_service", "canonical_service"), + Pair("csm.remote_workload_name", "workload"), + Pair("csm.remote_workload_location", "zone"), + Pair("csm.remote_workload_project_id", "id"))) + << PrettyPrintLabels(labels); +} + +TEST(MeshLabelsIterableTest, RemoteGkeTypeMetadata) { + std::vector> local_labels = { + {"csm.workload_canonical_service", "canonical_service"}, + {"csm.mesh_id", "mesh"}}; + grpc::internal::MeshLabelsIterable iterable( + local_labels, RemoteMetadataSliceFromResource(TestGkeResource())); + auto labels = LabelsFromIterable(&iterable); + EXPECT_TRUE(iterable.GotRemoteLabels()); + EXPECT_THAT( + labels, + ElementsAre( + Pair("csm.workload_canonical_service", "canonical_service"), + Pair("csm.mesh_id", "mesh"), + Pair("csm.remote_workload_type", "gcp_kubernetes_engine"), + Pair("csm.remote_workload_canonical_service", "canonical_service"), + Pair("csm.remote_workload_name", "workload"), + Pair("csm.remote_workload_namespace_name", "namespace"), + Pair("csm.remote_workload_cluster_name", "cluster"), + Pair("csm.remote_workload_location", "region"), + Pair("csm.remote_workload_project_id", "id"))) + << PrettyPrintLabels(labels); +} + +TEST(MeshLabelsIterableTest, RemoteUnknownTypeMetadata) { + std::vector> local_labels = { + {"csm.workload_canonical_service", "canonical_service"}, + {"csm.mesh_id", "mesh"}}; + grpc::internal::MeshLabelsIterable iterable( + local_labels, RemoteMetadataSliceFromResource(TestUnknownResource())); + auto labels = LabelsFromIterable(&iterable); + EXPECT_TRUE(iterable.GotRemoteLabels()); + EXPECT_THAT( + labels, + ElementsAre( + Pair("csm.workload_canonical_service", "canonical_service"), + Pair("csm.mesh_id", "mesh"), + Pair("csm.remote_workload_type", "random"), + Pair("csm.remote_workload_canonical_service", "canonical_service"))) + << PrettyPrintLabels(labels); +} + +TEST(MeshLabelsIterableTest, TestResetIteratorPosition) { + std::vector> local_labels = { + {"csm.workload_canonical_service", "canonical_service"}, + {"csm.mesh_id", "mesh"}}; + grpc::internal::MeshLabelsIterable iterable(local_labels, grpc_core::Slice()); + auto labels = LabelsFromIterable(&iterable); + auto expected_labels_matcher = ElementsAre( + Pair("csm.workload_canonical_service", "canonical_service"), + Pair("csm.mesh_id", "mesh"), Pair("csm.remote_workload_type", "unknown"), + Pair("csm.remote_workload_canonical_service", "unknown")); + EXPECT_THAT(labels, expected_labels_matcher) << PrettyPrintLabels(labels); + // Resetting the iterable should return the entire list again. + iterable.ResetIteratorPosition(); + labels = LabelsFromIterable(&iterable); + EXPECT_THAT(labels, expected_labels_matcher) << PrettyPrintLabels(labels); +} + INSTANTIATE_TEST_SUITE_P( MetadataExchange, MetadataExchangeTest, ::testing::Values(